rubber-duck-mcp/apps/eval/src/review.ts

import { readFile, writeFile, mkdir } from "node:fs/promises";
import path from "node:path";
import readline from "node:readline";
import type { ReportRoot, ScenarioName, PerTaskRow } from "@duck/types";
import { SCENARIO_NAMES } from "@duck/types";
import { buildReport, buildAggregate } from "./report.js";

const SCENARIO_LABELS: Record<ScenarioName, string> = {
  control: "Без утки (контроль)",
  thinking: "Думать вслух",
  blind: "Слепая утка",
  mentor: "Утка-помощник",
};

interface ReviewEntry {
  model: string;
  taskId: string;
  scenario: ScenarioName;
  row: PerTaskRow;
}

interface ReviewArgs {
  report: string | null;
  out: string | null;
  reviewFile: string | null;
  model: string | null;
  scenario: string | null;
  limit: number | null;
  reAsk?: boolean;
}

function parseArgs(argv: string[]): ReviewArgs {
  const args: ReviewArgs = {
    report: null,
    out: null,
    reviewFile: null,
    model: null,
    scenario: null,
    limit: null,
  };
  for (let i = 0; i < argv.length; i += 1) {
    const a = argv[i];
    const next = () => argv[i + 1];
    if (a === "--report") args.report = next(), i += 1;
    else if (a === "--out") args.out = next(), i += 1;
    else if (a === "--review") args.reviewFile = next(), i += 1;
    else if (a === "--model") args.model = next(), i += 1;
    else if (a === "--scenario") args.scenario = next(), i += 1;
    else if (a === "--limit") args.limit = Number(next()), i += 1;
    else if (a === "--re-ask") args.reAsk = true;
  }
  return args;
}

async function main(): Promise<void> {
  const args = parseArgs(process.argv.slice(2));
  const reportFile = args.report || path.join("out", "report.json");
  const outFile = args.out || path.join("out", "report.reviewed.json");
  const reviewFile = args.reviewFile || path.join("out", "review.json");

  const report: ReportRoot = JSON.parse(await readFile(reportFile, "utf8"));

  let decisions: Record<string, boolean> = {};
  try {
    decisions = JSON.parse(await readFile(reviewFile, "utf8"));
  } catch {
    /* no prior review */
  }

  const entries: ReviewEntry[] = [];
  for (const [model, rep] of Object.entries(report.models)) {
    for (const row of rep.perTask) {
      if (args.model && model !== args.model) continue;
      if (args.scenario && row.scenario !== args.scenario) continue;
      entries.push({ model, taskId: row.id, scenario: row.scenario, row });
    }
  }
  if (args.limit && entries.length > args.limit) entries.length = args.limit;

  const key = (e: ReviewEntry) => `${e.model}|${e.taskId}|${e.scenario}`;

  const isTTY = !!process.stdin.isTTY;

  let batchLines: string[] = [];
  if (!isTTY) {
    const input = await new Promise<string>((res, rej) => {
      let data = "";
      process.stdin.setEncoding("utf8");
      process.stdin.on("data", (c) => (data += c));
      process.stdin.on("end", () => res(data));
      process.stdin.on("error", rej);
      process.stdin.resume();
    });
    batchLines = input.split(/\r?\n/).map((l) => l.trim().toLowerCase());
  }

  let batchIdx = 0;
  const nextInput = async (): Promise<string> => {
    if (isTTY) {
      const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
      const ans = await new Promise<string>((res) => rl.question("", res));
      rl.close();
      return ans.trim().toLowerCase();
    }
    return batchLines[batchIdx++] ?? "";
  };

  let done = 0;
  let skipped = 0;
  let changed = 0;

  async function askOne(e: ReviewEntry): Promise<number> {
    const k = key(e);
    const prev = decisions[k];
    if (prev !== undefined) {
      console.log(`\n[${e.row.correct === prev ? " " : "*"}] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}  (ранее: ${prev ? "✓" : "✗"})`);
    } else {
      console.log(`\n[?] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}`);
    }

    console.log("────────────────────────────────────────────");
    console.log(`  Вопрос:   ${e.row.question}`);
    console.log(`  Ожидаемо: ${e.row.expected}`);

    console.log(`  --- ответ модели ---`);
    const lines = String(e.row.response).split("\n");
    for (const l of lines) console.log(`  ${l}`);

    const cur = e.row.correct === true ? "y" : e.row.correct === false ? "n" : null;
    const hint = cur ? "  [Enter] оставить" : "";
    process.stdout.write(`  [y] верно  [n] неверно  [e] пропуск${hint}\n  > `);
    const a = (await nextInput()) || "";

    let decision: boolean | null = null;
    if (a === "y" || a === "д") decision = true;
    else if (a === "n" || a === "н") decision = false;
    else if (a === "" && cur) decision = e.row.correct;

    if (decision == null) {
      skipped += 1;
      return 0;
    }

    if (prev !== decision) changed += 1;
    decisions[k] = decision;
    e.row.correct = decision;
    return 1;
  }

  const reAsk = !!args.reAsk;

  for (const e of entries) {
    const k = key(e);
    const prev = decisions[k];

    if (prev === undefined || reAsk) {
      done += await askOne(e);
      continue;
    }

    if (e.row.correct !== prev) changed += 1;
    e.row.correct = prev;
  }
  console.log(`\nГотово. Решено: ${done}, пропущено: ${skipped}, сменено решений: ${changed}`);

  await mkdir(path.dirname(reviewFile), { recursive: true });
  await writeFile(reviewFile, JSON.stringify(decisions, null, 2), "utf8");
  console.log(`Решения сохранены: ${path.resolve(reviewFile)}`);

  for (const [model, rep] of Object.entries(report.models)) {
    report.models[model] = {
      ...rep,
      scenarios: buildReport({
        model,
        mcpUrl: report.mcpUrl,
        rows: rep.perTask.map((r) => ({
          id: r.id,
          scenario: r.scenario,
          correct: r.correct,
          duckUsed: r.duckUsed,
          toolCalls: r.toolCalls,
          promptTokens: r.promptTokens,
          genTokens: r.genTokens,
          duckTokens: r.duckTokens,
          response: r.response,
          expected: r.expected,
          question: r.question,
        })),
      }).scenarios,
    };
  }

  report.aggregate = buildAggregate(report.models);
  report.reviewedAt = new Date().toISOString();

  await mkdir(path.dirname(outFile), { recursive: true });
  await writeFile(outFile, JSON.stringify(report, null, 2), "utf8");
  console.log(`Отчёт с ревью сохранён: ${path.resolve(outFile)}`);

  console.log("\n==== Ревью-сводка ====");
  for (const [model, rep] of Object.entries(report.models)) {
    const g = report.aggregate.perModel[model];
    const scen = rep.scenarios;
    console.log(
      `${model}  |  ` +
        SCENARIO_NAMES
          .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`)
          .join("  ") +
        `  всего rev=${g.reviewed}`
    );
  }
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});