import { readFile, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; import readline from "node:readline"; import type { ReportRoot, ScenarioName, PerTaskRow } from "@duck/types"; import { SCENARIO_NAMES } from "@duck/types"; import { buildReport, buildAggregate } from "./report.js"; const SCENARIO_LABELS: Record = { control: "Без утки (контроль)", thinking: "Думать вслух", blind: "Слепая утка", mentor: "Утка-помощник", }; interface ReviewEntry { model: string; taskId: string; scenario: ScenarioName; row: PerTaskRow; } interface ReviewArgs { report: string | null; out: string | null; reviewFile: string | null; model: string | null; scenario: string | null; limit: number | null; reAsk?: boolean; } function parseArgs(argv: string[]): ReviewArgs { const args: ReviewArgs = { report: null, out: null, reviewFile: null, model: null, scenario: null, limit: null, }; for (let i = 0; i < argv.length; i += 1) { const a = argv[i]; const next = () => argv[i + 1]; if (a === "--report") args.report = next(), i += 1; else if (a === "--out") args.out = next(), i += 1; else if (a === "--review") args.reviewFile = next(), i += 1; else if (a === "--model") args.model = next(), i += 1; else if (a === "--scenario") args.scenario = next(), i += 1; else if (a === "--limit") args.limit = Number(next()), i += 1; else if (a === "--re-ask") args.reAsk = true; } return args; } async function main(): Promise { const args = parseArgs(process.argv.slice(2)); const reportFile = args.report || path.join("out", "report.json"); const outFile = args.out || path.join("out", "report.reviewed.json"); const reviewFile = args.reviewFile || path.join("out", "review.json"); const report: ReportRoot = JSON.parse(await readFile(reportFile, "utf8")); let decisions: Record = {}; try { decisions = JSON.parse(await readFile(reviewFile, "utf8")); } catch { /* no prior review */ } const entries: ReviewEntry[] = []; for (const [model, rep] of Object.entries(report.models)) { for (const row of rep.perTask) { if (args.model && model !== args.model) continue; if (args.scenario && row.scenario !== args.scenario) continue; entries.push({ model, taskId: row.id, scenario: row.scenario, row }); } } if (args.limit && entries.length > args.limit) entries.length = args.limit; const key = (e: ReviewEntry) => `${e.model}|${e.taskId}|${e.scenario}`; const isTTY = !!process.stdin.isTTY; let batchLines: string[] = []; if (!isTTY) { const input = await new Promise((res, rej) => { let data = ""; process.stdin.setEncoding("utf8"); process.stdin.on("data", (c) => (data += c)); process.stdin.on("end", () => res(data)); process.stdin.on("error", rej); process.stdin.resume(); }); batchLines = input.split(/\r?\n/).map((l) => l.trim().toLowerCase()); } let batchIdx = 0; const nextInput = async (): Promise => { if (isTTY) { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); const ans = await new Promise((res) => rl.question("", res)); rl.close(); return ans.trim().toLowerCase(); } return batchLines[batchIdx++] ?? ""; }; let done = 0; let skipped = 0; let changed = 0; async function askOne(e: ReviewEntry): Promise { const k = key(e); const prev = decisions[k]; if (prev !== undefined) { console.log(`\n[${e.row.correct === prev ? " " : "*"}] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]} (ранее: ${prev ? "✓" : "✗"})`); } else { console.log(`\n[?] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}`); } console.log("────────────────────────────────────────────"); console.log(` Вопрос: ${e.row.question}`); console.log(` Ожидаемо: ${e.row.expected}`); console.log(` --- ответ модели ---`); const lines = String(e.row.response).split("\n"); for (const l of lines) console.log(` ${l}`); const cur = e.row.correct === true ? "y" : e.row.correct === false ? "n" : null; const hint = cur ? " [Enter] оставить" : ""; process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `); const a = (await nextInput()) || ""; let decision: boolean | null = null; if (a === "y" || a === "д") decision = true; else if (a === "n" || a === "н") decision = false; else if (a === "" && cur) decision = e.row.correct; if (decision == null) { skipped += 1; return 0; } if (prev !== decision) changed += 1; decisions[k] = decision; e.row.correct = decision; return 1; } const reAsk = !!args.reAsk; for (const e of entries) { const k = key(e); const prev = decisions[k]; if (prev === undefined || reAsk) { done += await askOne(e); continue; } if (e.row.correct !== prev) changed += 1; e.row.correct = prev; } console.log(`\nГотово. Решено: ${done}, пропущено: ${skipped}, сменено решений: ${changed}`); await mkdir(path.dirname(reviewFile), { recursive: true }); await writeFile(reviewFile, JSON.stringify(decisions, null, 2), "utf8"); console.log(`Решения сохранены: ${path.resolve(reviewFile)}`); for (const [model, rep] of Object.entries(report.models)) { report.models[model] = { ...rep, scenarios: buildReport({ model, mcpUrl: report.mcpUrl, rows: rep.perTask.map((r) => ({ id: r.id, scenario: r.scenario, correct: r.correct, duckUsed: r.duckUsed, toolCalls: r.toolCalls, promptTokens: r.promptTokens, genTokens: r.genTokens, duckTokens: r.duckTokens, response: r.response, expected: r.expected, question: r.question, })), }).scenarios, }; } report.aggregate = buildAggregate(report.models); report.reviewedAt = new Date().toISOString(); await mkdir(path.dirname(outFile), { recursive: true }); await writeFile(outFile, JSON.stringify(report, null, 2), "utf8"); console.log(`Отчёт с ревью сохранён: ${path.resolve(outFile)}`); console.log("\n==== Ревью-сводка ===="); for (const [model, rep] of Object.entries(report.models)) { const g = report.aggregate.perModel[model]; const scen = rep.scenarios; console.log( `${model} | ` + SCENARIO_NAMES .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`) .join(" ") + ` всего rev=${g.reviewed}` ); } } main().catch((err) => { console.error(err); process.exit(1); });