feat: move runner to typescript, refactor

2026-09-04 16:29:42 +05:00
parent d63cb77887
commit b9f98ea4b6
12 changed files with 649 additions and 263 deletions
@@ -0,0 +1,217 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import path from "node:path";
+import readline from "node:readline";
+import type { ReportRoot, ScenarioName, PerTaskRow } from "@duck/types";
+import { SCENARIO_NAMES } from "@duck/types";
+import { buildReport, buildAggregate } from "./report.js";
+
+const SCENARIO_LABELS: Record<ScenarioName, string> = {
+  control: "Без утки (контроль)",
+  thinking: "Думать вслух",
+  blind: "Слепая утка",
+  mentor: "Утка-помощник",
+};
+
+interface ReviewEntry {
+  model: string;
+  taskId: string;
+  scenario: ScenarioName;
+  row: PerTaskRow;
+}
+
+interface ReviewArgs {
+  report: string | null;
+  out: string | null;
+  reviewFile: string | null;
+  model: string | null;
+  scenario: string | null;
+  limit: number | null;
+  reAsk?: boolean;
+}
+
+function parseArgs(argv: string[]): ReviewArgs {
+  const args: ReviewArgs = {
+    report: null,
+    out: null,
+    reviewFile: null,
+    model: null,
+    scenario: null,
+    limit: null,
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    const next = () => argv[i + 1];
+    if (a === "--report") args.report = next(), i += 1;
+    else if (a === "--out") args.out = next(), i += 1;
+    else if (a === "--review") args.reviewFile = next(), i += 1;
+    else if (a === "--model") args.model = next(), i += 1;
+    else if (a === "--scenario") args.scenario = next(), i += 1;
+    else if (a === "--limit") args.limit = Number(next()), i += 1;
+    else if (a === "--re-ask") args.reAsk = true;
+  }
+  return args;
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+  const reportFile = args.report || path.join("out", "report.json");
+  const outFile = args.out || path.join("out", "report.reviewed.json");
+  const reviewFile = args.reviewFile || path.join("out", "review.json");
+
+  const report: ReportRoot = JSON.parse(await readFile(reportFile, "utf8"));
+
+  let decisions: Record<string, boolean> = {};
+  try {
+    decisions = JSON.parse(await readFile(reviewFile, "utf8"));
+  } catch {
+    /* no prior review */
+  }
+
+  const entries: ReviewEntry[] = [];
+  for (const [model, rep] of Object.entries(report.models)) {
+    for (const row of rep.perTask) {
+      if (args.model && model !== args.model) continue;
+      if (args.scenario && row.scenario !== args.scenario) continue;
+      entries.push({ model, taskId: row.id, scenario: row.scenario, row });
+    }
+  }
+  if (args.limit && entries.length > args.limit) entries.length = args.limit;
+
+  const key = (e: ReviewEntry) => `${e.model}|${e.taskId}|${e.scenario}`;
+
+  const isTTY = !!process.stdin.isTTY;
+
+  let batchLines: string[] = [];
+  if (!isTTY) {
+    const input = await new Promise<string>((res, rej) => {
+      let data = "";
+      process.stdin.setEncoding("utf8");
+      process.stdin.on("data", (c) => (data += c));
+      process.stdin.on("end", () => res(data));
+      process.stdin.on("error", rej);
+      process.stdin.resume();
+    });
+    batchLines = input.split(/\r?\n/).map((l) => l.trim().toLowerCase());
+  }
+
+  let batchIdx = 0;
+  const nextInput = async (): Promise<string> => {
+    if (isTTY) {
+      const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+      const ans = await new Promise<string>((res) => rl.question("", res));
+      rl.close();
+      return ans.trim().toLowerCase();
+    }
+    return batchLines[batchIdx++] ?? "";
+  };
+
+  let done = 0;
+  let skipped = 0;
+  let changed = 0;
+
+  async function askOne(e: ReviewEntry): Promise<number> {
+    const k = key(e);
+    const prev = decisions[k];
+    if (prev !== undefined) {
+      console.log(`\n[${e.row.correct === prev ? " " : "*"}] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}  (ранее: ${prev ? "✓" : "✗"})`);
+    } else {
+      console.log(`\n[?] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}`);
+    }
+
+    console.log("────────────────────────────────────────────");
+    console.log(`  Вопрос:   ${e.row.question}`);
+    console.log(`  Ожидаемо: ${e.row.expected}`);
+
+    console.log(`  --- ответ модели ---`);
+    const lines = String(e.row.response).split("\n");
+    for (const l of lines) console.log(`  ${l}`);
+
+    const cur = e.row.correct === true ? "y" : e.row.correct === false ? "n" : null;
+    const hint = cur ? "  [Enter] оставить" : "";
+    process.stdout.write(`  [y] верно  [n] неверно  [e] пропуск${hint}\n  > `);
+    const a = (await nextInput()) || "";
+
+    let decision: boolean | null = null;
+    if (a === "y" || a === "д") decision = true;
+    else if (a === "n" || a === "н") decision = false;
+    else if (a === "" && cur) decision = e.row.correct;
+
+    if (decision == null) {
+      skipped += 1;
+      return 0;
+    }
+
+    if (prev !== decision) changed += 1;
+    decisions[k] = decision;
+    e.row.correct = decision;
+    return 1;
+  }
+
+  const reAsk = !!args.reAsk;
+
+  for (const e of entries) {
+    const k = key(e);
+    const prev = decisions[k];
+
+    if (prev === undefined || reAsk) {
+      done += await askOne(e);
+      continue;
+    }
+
+    if (e.row.correct !== prev) changed += 1;
+    e.row.correct = prev;
+  }
+  console.log(`\nГотово. Решено: ${done}, пропущено: ${skipped}, сменено решений: ${changed}`);
+
+  await mkdir(path.dirname(reviewFile), { recursive: true });
+  await writeFile(reviewFile, JSON.stringify(decisions, null, 2), "utf8");
+  console.log(`Решения сохранены: ${path.resolve(reviewFile)}`);
+
+  for (const [model, rep] of Object.entries(report.models)) {
+    report.models[model] = {
+      ...rep,
+      scenarios: buildReport({
+        model,
+        mcpUrl: report.mcpUrl,
+        rows: rep.perTask.map((r) => ({
+          id: r.id,
+          scenario: r.scenario,
+          correct: r.correct,
+          duckUsed: r.duckUsed,
+          toolCalls: r.toolCalls,
+          promptTokens: r.promptTokens,
+          genTokens: r.genTokens,
+          duckTokens: r.duckTokens,
+          response: r.response,
+          expected: r.expected,
+          question: r.question,
+        })),
+      }).scenarios,
+    };
+  }
+
+  report.aggregate = buildAggregate(report.models);
+  report.reviewedAt = new Date().toISOString();
+
+  await mkdir(path.dirname(outFile), { recursive: true });
+  await writeFile(outFile, JSON.stringify(report, null, 2), "utf8");
+  console.log(`Отчёт с ревью сохранён: ${path.resolve(outFile)}`);
+
+  console.log("\n==== Ревью-сводка ====");
+  for (const [model, rep] of Object.entries(report.models)) {
+    const g = report.aggregate.perModel[model];
+    const scen = rep.scenarios;
+    console.log(
+      `${model}  |  ` +
+        SCENARIO_NAMES
+          .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`)
+          .join("  ") +
+        `  всего rev=${g.reviewed}`
+    );
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});