From 6a1c7666c4e8e974b6b085af4137d64db3ab8b79 Mon Sep 17 00:00:00 2001 From: Ku6epXBOCTuK Date: Fri, 4 Sep 2026 06:49:56 +0500 Subject: [PATCH] feat: manual review results flow --- apps/eval/src/report-html.mjs | 67 ++++++++--- apps/eval/src/report.mjs | 32 ++++-- apps/eval/src/review.mjs | 205 ++++++++++++++++++++++++++++++++++ apps/eval/src/run.mjs | 9 +- apps/eval/src/runner.mjs | 53 --------- package.json | 5 +- 6 files changed, 288 insertions(+), 83 deletions(-) create mode 100644 apps/eval/src/review.mjs diff --git a/apps/eval/src/report-html.mjs b/apps/eval/src/report-html.mjs index c525622..a6f0aae 100644 --- a/apps/eval/src/report-html.mjs +++ b/apps/eval/src/report-html.mjs @@ -44,16 +44,20 @@ function summaryTable(report) { const g = perModel[m]; const scen = report.models[m].scenarios; const acc = (s) => scen[s].accuracy; - return ` - - ${modelShort(m)} + const row = ` ${acc("control")}% ${acc("blind")}% ${acc("mentor")}% ${g.accuracy}% + ${g.reviewed} / ${g.totalRows} ${g.duckUsed} ${g.toolCalls} ${g.avgGenTokens} + `; + return ` + + ${modelShort(m)} + ${row} `; }) .join(""); @@ -67,6 +71,7 @@ function summaryTable(report) { Слепая Помощник Средняя acc + Размечено Уток Вызовов ср. токены out @@ -81,13 +86,15 @@ function scenarioCard(key, agg) { const pct = agg.accuracy; const color = pctColor(pct); const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0; + const pendingBadge = agg.pending > 0 ? `неразмечено: ${agg.pending}` : ""; const modelBars = Object.entries(agg.byModel) .map(([m, g]) => { const c = pctColor(g.accuracy); + const barW = g.reviewed ? g.accuracy : 0; return `
- ${modelShort(m)} -
+ ${modelShort(m)}${g.reviewed ? "" : " (?)"} +
${g.accuracy}%
`; }) @@ -99,11 +106,11 @@ function scenarioCard(key, agg) {
${esc(SCENARIO_LABELS[key])}
${esc(SCENARIO_DESCS[key])}
-
${pct}%
+
${pct}%${pendingBadge}
-
Верных${agg.correct} / ${agg.tasks}
+
Верных${agg.correct} / ${agg.reviewed}размечено из ${agg.tasks}
Утка (MCP)${duckPct}%в ${agg.duckUsed} из ${agg.tasks} задач
quack / задачу${agg.avgCalls}вызовов в среднем
@@ -112,14 +119,16 @@ function scenarioCard(key, agg) { } function rowHtml(r) { - const cls = r.correct ? "ok" : "no"; + const pending = r.correct === null || r.correct === undefined; + const cls = pending ? "pend" : r.correct ? "ok" : "no"; + const result = pending ? "?" : r.correct ? "✓" : "✗"; const duck = r.duckUsed ? "🦆" : "—"; return ` ${modelShort(r.model)} ${esc(r.id)} ${esc(SCENARIO_LABELS[r.scenario])} - ${r.correct ? "✓" : "✗"} + ${result} ${duck} (${r.toolCalls}) ${nl2br(r.question)} ${esc(r.expected)} @@ -154,6 +163,18 @@ export function buildHtml(report) { ); const rows = perTask.map(rowHtml).join(""); const models = Object.keys(report.aggregate.perModel); + let totalPending = 0, totalReviewed = 0; + for (const rep of Object.values(report.models)) { + for (const s of SCENARIOS) { + if (rep.scenarios[s]) { + totalReviewed += rep.scenarios[s].reviewed || 0; + totalPending += rep.scenarios[s].pending || 0; + } + } + } + const reviewNote = totalPending > 0 + ? `
⚠ Не все ответы размечены: ${totalReviewed} проверено, ${totalPending} ожидают ревью (показаны символом «?»). Запустите node src/review.mjs для проставления меток.
` + : ""; const modelOptions = models .map((m) => ``) .join(""); @@ -195,6 +216,7 @@ export function buildHtml(report) { .scenario-title { font-size:16px; font-weight:700; } .scenario-desc { color:var(--muted); font-size:12px; margin-top:4px; } .accuracy { font-size:32px; font-weight:800; line-height:1; } + .pending-badge { display:block; font-size:11px; font-weight:600; color:var(--muted); margin-top:4px; } .bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; } .bar-fill { height:100%; border-radius:99px; transition:width .4s; } .metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; } @@ -209,6 +231,9 @@ export function buildHtml(report) { .agg-model-name { color:var(--muted); white-space:nowrap; } .agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; } .prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } + .review-note { margin:14px 0; padding:10px 14px; background:rgba(245,158,11,.12); border:1px solid rgba(245,158,11,.4); + border-radius:10px; color:#fbbf24; font-size:13px; } + .review-note code { background:var(--panel2); padding:1px 6px; border-radius:5px; } .prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; } .prompts summary::-webkit-details-marker { display:none; } .prompts summary::before { content:"▸ "; color:var(--muted); } @@ -230,7 +255,8 @@ export function buildHtml(report) { tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; } .scenario-row.ok .r-result { color:var(--ok); font-weight:800; } .scenario-row.no .r-result { color:var(--no); font-weight:800; } - .r-model { font-weight:600; white-space:nowrap; } + .scenario-row.pend .r-result { color:var(--muted); font-weight:800; } + .scenario-row.pend { opacity:.75; } .r-model { font-weight:600; white-space:nowrap; } .r-id { font-weight:700; color:var(--muted); } .r-s { white-space:nowrap; } .r-duck { text-align:center; white-space:nowrap; } @@ -255,6 +281,8 @@ export function buildHtml(report) {

Сводка по моделям

${summaryTable(report)} + ${reviewNote} +

Агрегат по сценариям (все модели)

${aggCards}
@@ -321,14 +349,27 @@ export function buildHtml(report) { async function main() { const args = process.argv.slice(2); - const inFile = args.find((a) => a.startsWith("--in="))?.slice(5) || "out/report.json"; - const outFile = args.find((a) => a.startsWith("--out="))?.slice(6) || "out/report.html"; + let inFile = null, outFile = "out/report.html"; + for (let i = 0; i < args.length; i += 1) { + const a = args[i]; + const next = () => args[i + 1]; + if (a === "--in") inFile = next(), i += 1; + else if (a.startsWith("--in=")) inFile = a.slice(5); + else if (a === "--out") outFile = next(), i += 1; + else if (a.startsWith("--out=")) outFile = a.slice(6); + } + + if (!inFile) { + const reviewed = path.join("out", "report.reviewed.json"); + const base = path.join("out", "report.json"); + inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base)); + } const report = JSON.parse(await readFile(inFile, "utf8")); const html = buildHtml(report); await mkdir(path.dirname(outFile), { recursive: true }); await writeFile(outFile, html, "utf8"); - console.log(`HTML report written: ${path.resolve(outFile)}`); + console.log(`HTML report written: ${path.resolve(outFile)} (source: ${inFile})`); } if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) { diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.mjs index d595f63..23268c9 100644 --- a/apps/eval/src/report.mjs +++ b/apps/eval/src/report.mjs @@ -1,5 +1,3 @@ -import { isCorrect } from "./runner.mjs"; - function round(x, d = 2) { if (!Number.isFinite(x)) return x; const f = 10 ** d; @@ -13,13 +11,16 @@ function avg(arr) { export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "blind", "mentor"]; const byScenario = {}; - for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; + for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; for (const row of rows) { const s = row.scenario; if (!byScenario[s]) continue; byScenario[s].total += 1; - if (row.correct) byScenario[s].correct += 1; + if (row.correct !== null && row.correct !== undefined) { + byScenario[s].reviewed += 1; + if (row.correct) byScenario[s].correct += 1; + } if (row.duckUsed) byScenario[s].duckUsed += 1; byScenario[s].toolCalls += row.toolCalls ?? 0; byScenario[s].promptTokens.push(row.promptTokens ?? 0); @@ -31,10 +32,13 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = { const summaries = {}; for (const s of scenarios) { const g = byScenario[s]; + const denom = g.reviewed || 1; summaries[s] = { tasks: g.total, + reviewed: g.reviewed, correct: g.correct, - accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0, + accuracy: g.reviewed ? round((g.correct / g.reviewed) * 100, 1) : 0, + pending: g.total - g.reviewed, duckUsed: g.duckUsed, toolCalls: g.toolCalls, avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0, @@ -72,27 +76,30 @@ const SCENARIOS = ["control", "blind", "mentor"]; export function buildAggregate(modelsReport) { const perScenario = {}; for (const s of SCENARIOS) { - let total = 0, correct = 0, duckUsed = 0, toolCalls = 0; - const gen = []; + let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0; const byModel = {}; for (const [name, rep] of Object.entries(modelsReport)) { const g = rep.scenarios[s]; if (!g) continue; total += g.tasks; + reviewed += g.reviewed; correct += g.correct; duckUsed += g.duckUsed; toolCalls += g.toolCalls; byModel[name] = { tasks: g.tasks, + reviewed: g.reviewed, correct: g.correct, - accuracy: g.accuracy, + accuracy: g.reviewed ? g.accuracy : 0, duckUsed: g.duckUsed, }; } perScenario[s] = { tasks: total, + reviewed, correct, - accuracy: total ? round((correct / total) * 100, 1) : 0, + accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0, + pending: total - reviewed, duckUsed, toolCalls, avgCalls: total ? round(toolCalls / total, 2) : 0, @@ -104,22 +111,25 @@ export function buildAggregate(modelsReport) { for (const [name, rep] of Object.entries(modelsReport)) { let duckUsed = 0, toolCalls = 0; const gen = []; + let reviewed = 0, correct = 0; for (const s of SCENARIOS) { const g = rep.scenarios[s]; duckUsed += g.duckUsed; toolCalls += g.toolCalls; + reviewed += g.reviewed; + correct += g.correct; for (const row of rep.perTask.filter((r) => r.scenario === s)) { gen.push(row.genTokens ?? 0); } } const taskCount = rep.scenarios.control?.tasks ?? 0; - const correct = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].correct || 0), 0); const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0); perModel[name] = { tasks: taskCount, totalRows, + reviewed, correct, - accuracy: totalRows ? round((correct / totalRows) * 100, 1) : 0, + accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0, duckUsed, toolCalls, avgGenTokens: round(avg(gen)), diff --git a/apps/eval/src/review.mjs b/apps/eval/src/review.mjs new file mode 100644 index 0000000..9749abd --- /dev/null +++ b/apps/eval/src/review.mjs @@ -0,0 +1,205 @@ +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import path from "node:path"; +import readline from "node:readline"; +import { buildReport, buildAggregate } from "./report.mjs"; + +const SCENARIO_LABELS = { + control: "Без утки (контроль)", + blind: "Слепая утка", + mentor: "Утка-помощник", +}; + +function esc(s) { + return String(s ?? "") + .replace(/&/g, "&") + .replace(//g, ">"); +} + +function parseArgs(argv) { + const args = { + report: null, + out: null, + reviewFile: null, + model: null, + scenario: null, + limit: null, + }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + const next = () => argv[i + 1]; + if (a === "--report") args.report = next(), i += 1; + else if (a === "--out") args.out = next(), i += 1; + else if (a === "--review") args.reviewFile = next(), i += 1; + else if (a === "--model") args.model = next(), i += 1; + else if (a === "--scenario") args.scenario = next(), i += 1; + else if (a === "--limit") args.limit = Number(next()), i += 1; + else if (a === "--re-ask") args.reAsk = true; + } + return args; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const reportFile = args.report || path.join("out", "report.json"); + const outFile = args.out || path.join("out", "report.reviewed.json"); + const reviewFile = args.reviewFile || path.join("out", "review.json"); + + const report = JSON.parse(await readFile(reportFile, "utf8")); + + let decisions = {}; + try { + decisions = JSON.parse(await readFile(reviewFile, "utf8")); + } catch { + /* no prior review */ + } + + const entries = []; + for (const [model, rep] of Object.entries(report.models)) { + for (const row of rep.perTask) { + if (args.model && model !== args.model) continue; + if (args.scenario && row.scenario !== args.scenario) continue; + entries.push({ model, taskId: row.id, scenario: row.scenario, row }); + } + } + if (args.limit && entries.length > args.limit) entries.length = args.limit; + + const key = (e) => `${e.model}|${e.taskId}|${e.scenario}`; + + const isTTY = !!process.stdin.isTTY; + + let batchLines = []; + if (!isTTY) { + const input = await new Promise((res, rej) => { + let data = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (c) => (data += c)); + process.stdin.on("end", () => res(data)); + process.stdin.on("error", rej); + process.stdin.resume(); + }); + batchLines = input.split(/\r?\n/).map((l) => l.trim().toLowerCase()); + } + + let batchIdx = 0; + const nextInput = async () => { + if (isTTY) { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const ans = await new Promise((res) => rl.question("", res)); + rl.close(); + return ans.trim().toLowerCase(); + } + return batchLines[batchIdx++] ?? ""; + }; + + async function askOne(e, nextInput, decisions) { + const k = key(e); + const prev = decisions[k]; + if (prev !== undefined) { + console.log(`\n[${e.row.correct === prev ? " " : "*"}] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]} (ранее: ${prev ? "✓" : "✗"})`); + } else { + console.log(`\n[?] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}`); + } + + console.log("────────────────────────────────────────────"); + console.log(` Вопрос: ${e.row.question}`); + console.log(` Ожидаемо: ${e.row.expected}`); + + console.log(` --- ответ модели ---`); + const lines = String(e.row.response).split("\n"); + for (const l of lines) console.log(` ${l}`); + + const cur = e.row.correct === true ? "y" : e.row.correct === false ? "n" : null; + const hint = cur ? " [Enter] оставить" : ""; + process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `); + const a = (await nextInput()) || ""; + + let decision = null; + if (a === "y" || a === "д") decision = true; + else if (a === "n" || a === "н") decision = false; + else if (a === "" && cur) decision = e.row.correct; + + if (decision == null) { + skipped += 1; + return 0; + } + + if (prev !== decision) changed += 1; + decisions[k] = decision; + e.row.correct = decision; + return 1; + } + + let done = 0; + let skipped = 0; + let changed = 0; + + const reAsk = !!args.reAsk; + + for (const e of entries) { + const k = key(e); + const prev = decisions[k]; + + if (prev === undefined || reAsk) { + done += await askOne(e, nextInput, decisions); + continue; + } + + if (e.row.correct !== prev) changed += 1; + e.row.correct = prev; + } + console.log(`\nГотово. Решено: ${done}, пропущено: ${skipped}, сменено решений: ${changed}`); + + await mkdir(path.dirname(reviewFile), { recursive: true }); + await writeFile(reviewFile, JSON.stringify(decisions, null, 2), "utf8"); + console.log(`Решения сохранены: ${path.resolve(reviewFile)}`); + + for (const [model, rep] of Object.entries(report.models)) { + report.models[model] = { + ...rep, + scenarios: buildReport({ + model, + mcpUrl: report.mcpUrl, + tasks: [], + rows: rep.perTask.map((r) => ({ + id: r.id, + scenario: r.scenario, + correct: r.correct, + duckUsed: r.duckUsed, + toolCalls: r.toolCalls, + promptTokens: r.promptTokens, + genTokens: r.genTokens, + duckTokens: r.duckTokens, + response: r.response, + expected: r.expected, + question: r.question, + })), + }).scenarios, + }; + } + + report.aggregate = buildAggregate(report.models); + report.reviewedAt = new Date().toISOString(); + + await mkdir(path.dirname(outFile), { recursive: true }); + await writeFile(outFile, JSON.stringify(report, null, 2), "utf8"); + console.log(`Отчёт с ревью сохранён: ${path.resolve(outFile)}`); + + console.log("\n==== Ревью-сводка ===="); + for (const [model, rep] of Object.entries(report.models)) { + const g = report.aggregate.perModel[model]; + const scen = rep.scenarios; + console.log( + `${model} | ` + + ["control", "blind", "mentor"] + .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`) + .join(" ") + + ` всего rev=${g.reviewed}` + ); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.mjs index 0724113..9239931 100644 --- a/apps/eval/src/run.mjs +++ b/apps/eval/src/run.mjs @@ -3,7 +3,7 @@ import path from "node:path"; import { Ollama } from "../lib/ollama.mjs"; import { McpClient } from "../lib/mcpClient.mjs"; import { runScenario } from "./runner.mjs"; -import { isCorrect, PROMPTS } from "./runner.mjs"; +import { PROMPTS } from "./runner.mjs"; import { buildReport, buildAggregate } from "./report.mjs"; let logTarget = null; @@ -62,7 +62,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { rows.push({ id: task.id, scenario, - correct: false, + correct: null, duckUsed: false, toolCalls: 0, promptTokens: 0, @@ -74,7 +74,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { }); continue; } - const correct = isCorrect(r.response, task.answer); + const correct = null; rows.push({ id: task.id, scenario, @@ -88,10 +88,9 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { expected: task.answer, question: task.question, }); - const mark = correct ? "OK " : "XX "; const duck = r.duckUsed ? "duck" : "n/a "; const ms = Date.now() - t0; - await log(` [${task.id}/${scenario}] ${mark} duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`); + await log(` [${task.id}/${scenario}] duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`); } } return rows; diff --git a/apps/eval/src/runner.mjs b/apps/eval/src/runner.mjs index 143a87f..954a0d7 100644 --- a/apps/eval/src/runner.mjs +++ b/apps/eval/src/runner.mjs @@ -42,59 +42,6 @@ export const PROMPTS = { const MAX_DUCK_TURNS = 6; -function normalize(s) { - return String(s ?? "") - .toLowerCase() - .replace(/[^a-zа-яё0-9/]/g, "") - .replace(/ё/g, "е") - .trim(); -} - -function extractFinalNumber(text) { - const t = String(text ?? ""); - const re = /(?:ответ|answer|otvet|итог|итак|final)\s*[:：—]?\s*/gi; - let segment = t; - let m; - while ((m = re.exec(t)) !== null) { - if (m[0].trim()) segment = t.slice(re.lastIndex); - } - const digits = segment.match(/\d+/g) || []; - return digits.length ? digits[digits.length - 1] : null; -} - -function isNumeric(s) { - return /^\d+(?:[.,]\d+)?$/.test(String(s).trim()); -} - -export function isCorrect(predicted, expected) { - const e = String(expected ?? "").trim(); - const p = String(predicted ?? ""); - if (!e || !p) return false; - - // Дробный ответ (например "1/8") — сравниваем вхождение нормализованной дроби. - if (/\d+\s*\/\s*\d+/.test(e)) { - const eNorm = normalize(e); - const pNorm = normalize(p); - return pNorm.includes(eNorm); - } - - // Числовой ответ — извлекаем финальное число и сравниваем точно. - if (isNumeric(e)) { - const cand = extractFinalNumber(p); - if (cand == null) return false; - return normalizeNum(cand) === normalizeNum(e); - } - - // Текстовый ответ — нормализованное вхождение слова. - const eNorm = normalize(e); - const pNorm = normalize(p); - return pNorm.includes(eNorm); -} - -function normalizeNum(n) { - return String(n).replace(/[.,]/g, "").replace(/^0+(?=\d)/, ""); -} - function toolDefinition() { return DUCK_TOOL; } diff --git a/package.json b/package.json index 6a74f76..c6f0f74 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,10 @@ "dev-mcp": "pnpm --dir apps/mcp dev", "typecheck-mcp": "pnpm --dir apps/mcp exec tsc --noEmit", "deploy-mcp": "vercel --cwd apps/mcp --prod --yes", - "deploy-frontend": "vercel --cwd apps/frontend --prod --yes" + "deploy-frontend": "vercel --cwd apps/frontend --prod --yes", + "eval:run": "pnpm --dir apps/eval exec node src/run.mjs", + "eval:review": "pnpm --dir apps/eval exec node src/review.mjs", + "eval:html": "pnpm --dir apps/eval exec node src/report-html.mjs" }, "license": "ISC", "devEngines": {