diff --git a/apps/eval/src/report-html.mjs b/apps/eval/src/report-html.mjs
index c525622..a6f0aae 100644
--- a/apps/eval/src/report-html.mjs
+++ b/apps/eval/src/report-html.mjs
@@ -44,16 +44,20 @@ function summaryTable(report) {
const g = perModel[m];
const scen = report.models[m].scenarios;
const acc = (s) => scen[s].accuracy;
- return `
-
- ${modelShort(m)}
+ const row = `
${acc("control")}%
${acc("blind")}%
${acc("mentor")}%
${g.accuracy}%
+ ${g.reviewed} / ${g.totalRows}
${g.duckUsed}
${g.toolCalls}
${g.avgGenTokens}
+ `;
+ return `
+
+ ${modelShort(m)}
+ ${row}
`;
})
.join("");
@@ -67,6 +71,7 @@ function summaryTable(report) {
Слепая
Помощник
Средняя acc
+ Размечено
Уток
Вызовов
ср. токены out
@@ -81,13 +86,15 @@ function scenarioCard(key, agg) {
const pct = agg.accuracy;
const color = pctColor(pct);
const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0;
+ const pendingBadge = agg.pending > 0 ? `неразмечено: ${agg.pending} ` : "";
const modelBars = Object.entries(agg.byModel)
.map(([m, g]) => {
const c = pctColor(g.accuracy);
+ const barW = g.reviewed ? g.accuracy : 0;
return `
-
${modelShort(m)}
-
+
${modelShort(m)}${g.reviewed ? "" : " (?) "}
+
${g.accuracy}%
${esc(SCENARIO_LABELS[key])}
${esc(SCENARIO_DESCS[key])}
- ${pct}%
+ ${pct}%${pendingBadge}
-
Верных ${agg.correct} / ${agg.tasks}
+
Верных ${agg.correct} / ${agg.reviewed} размечено из ${agg.tasks}
Утка (MCP) ${duckPct}% в ${agg.duckUsed} из ${agg.tasks} задач
quack / задачу ${agg.avgCalls} вызовов в среднем
${modelShort(r.model)}
${esc(r.id)}
${esc(SCENARIO_LABELS[r.scenario])}
- ${r.correct ? "✓" : "✗"}
+ ${result}
${duck} (${r.toolCalls})
${nl2br(r.question)}
${esc(r.expected)}
@@ -154,6 +163,18 @@ export function buildHtml(report) {
);
const rows = perTask.map(rowHtml).join("");
const models = Object.keys(report.aggregate.perModel);
+ let totalPending = 0, totalReviewed = 0;
+ for (const rep of Object.values(report.models)) {
+ for (const s of SCENARIOS) {
+ if (rep.scenarios[s]) {
+ totalReviewed += rep.scenarios[s].reviewed || 0;
+ totalPending += rep.scenarios[s].pending || 0;
+ }
+ }
+ }
+ const reviewNote = totalPending > 0
+ ? `⚠ Не все ответы размечены: ${totalReviewed} проверено, ${totalPending} ожидают ревью (показаны символом «?»). Запустите node src/review.mjs для проставления меток.
`
+ : "";
const modelOptions = models
.map((m) => `${modelShort(m)} `)
.join("");
@@ -195,6 +216,7 @@ export function buildHtml(report) {
.scenario-title { font-size:16px; font-weight:700; }
.scenario-desc { color:var(--muted); font-size:12px; margin-top:4px; }
.accuracy { font-size:32px; font-weight:800; line-height:1; }
+ .pending-badge { display:block; font-size:11px; font-weight:600; color:var(--muted); margin-top:4px; }
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
.bar-fill { height:100%; border-radius:99px; transition:width .4s; }
.metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; }
@@ -209,6 +231,9 @@ export function buildHtml(report) {
.agg-model-name { color:var(--muted); white-space:nowrap; }
.agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; }
.prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
+ .review-note { margin:14px 0; padding:10px 14px; background:rgba(245,158,11,.12); border:1px solid rgba(245,158,11,.4);
+ border-radius:10px; color:#fbbf24; font-size:13px; }
+ .review-note code { background:var(--panel2); padding:1px 6px; border-radius:5px; }
.prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
.prompts summary::-webkit-details-marker { display:none; }
.prompts summary::before { content:"▸ "; color:var(--muted); }
@@ -230,7 +255,8 @@ export function buildHtml(report) {
tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; }
.scenario-row.ok .r-result { color:var(--ok); font-weight:800; }
.scenario-row.no .r-result { color:var(--no); font-weight:800; }
- .r-model { font-weight:600; white-space:nowrap; }
+ .scenario-row.pend .r-result { color:var(--muted); font-weight:800; }
+ .scenario-row.pend { opacity:.75; } .r-model { font-weight:600; white-space:nowrap; }
.r-id { font-weight:700; color:var(--muted); }
.r-s { white-space:nowrap; }
.r-duck { text-align:center; white-space:nowrap; }
@@ -255,6 +281,8 @@ export function buildHtml(report) {
Сводка по моделям
${summaryTable(report)}
+ ${reviewNote}
+
Агрегат по сценариям (все модели)
${aggCards}
@@ -321,14 +349,27 @@ export function buildHtml(report) {
async function main() {
const args = process.argv.slice(2);
- const inFile = args.find((a) => a.startsWith("--in="))?.slice(5) || "out/report.json";
- const outFile = args.find((a) => a.startsWith("--out="))?.slice(6) || "out/report.html";
+ let inFile = null, outFile = "out/report.html";
+ for (let i = 0; i < args.length; i += 1) {
+ const a = args[i];
+ const next = () => args[i + 1];
+ if (a === "--in") inFile = next(), i += 1;
+ else if (a.startsWith("--in=")) inFile = a.slice(5);
+ else if (a === "--out") outFile = next(), i += 1;
+ else if (a.startsWith("--out=")) outFile = a.slice(6);
+ }
+
+ if (!inFile) {
+ const reviewed = path.join("out", "report.reviewed.json");
+ const base = path.join("out", "report.json");
+ inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base));
+ }
const report = JSON.parse(await readFile(inFile, "utf8"));
const html = buildHtml(report);
await mkdir(path.dirname(outFile), { recursive: true });
await writeFile(outFile, html, "utf8");
- console.log(`HTML report written: ${path.resolve(outFile)}`);
+ console.log(`HTML report written: ${path.resolve(outFile)} (source: ${inFile})`);
}
if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) {
diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.mjs
index d595f63..23268c9 100644
--- a/apps/eval/src/report.mjs
+++ b/apps/eval/src/report.mjs
@@ -1,5 +1,3 @@
-import { isCorrect } from "./runner.mjs";
-
function round(x, d = 2) {
if (!Number.isFinite(x)) return x;
const f = 10 ** d;
@@ -13,13 +11,16 @@ function avg(arr) {
export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "blind", "mentor"];
const byScenario = {};
- for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
+ for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
for (const row of rows) {
const s = row.scenario;
if (!byScenario[s]) continue;
byScenario[s].total += 1;
- if (row.correct) byScenario[s].correct += 1;
+ if (row.correct !== null && row.correct !== undefined) {
+ byScenario[s].reviewed += 1;
+ if (row.correct) byScenario[s].correct += 1;
+ }
if (row.duckUsed) byScenario[s].duckUsed += 1;
byScenario[s].toolCalls += row.toolCalls ?? 0;
byScenario[s].promptTokens.push(row.promptTokens ?? 0);
@@ -31,10 +32,13 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
const summaries = {};
for (const s of scenarios) {
const g = byScenario[s];
+ const denom = g.reviewed || 1;
summaries[s] = {
tasks: g.total,
+ reviewed: g.reviewed,
correct: g.correct,
- accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0,
+ accuracy: g.reviewed ? round((g.correct / g.reviewed) * 100, 1) : 0,
+ pending: g.total - g.reviewed,
duckUsed: g.duckUsed,
toolCalls: g.toolCalls,
avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0,
@@ -72,27 +76,30 @@ const SCENARIOS = ["control", "blind", "mentor"];
export function buildAggregate(modelsReport) {
const perScenario = {};
for (const s of SCENARIOS) {
- let total = 0, correct = 0, duckUsed = 0, toolCalls = 0;
- const gen = [];
+ let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0;
const byModel = {};
for (const [name, rep] of Object.entries(modelsReport)) {
const g = rep.scenarios[s];
if (!g) continue;
total += g.tasks;
+ reviewed += g.reviewed;
correct += g.correct;
duckUsed += g.duckUsed;
toolCalls += g.toolCalls;
byModel[name] = {
tasks: g.tasks,
+ reviewed: g.reviewed,
correct: g.correct,
- accuracy: g.accuracy,
+ accuracy: g.reviewed ? g.accuracy : 0,
duckUsed: g.duckUsed,
};
}
perScenario[s] = {
tasks: total,
+ reviewed,
correct,
- accuracy: total ? round((correct / total) * 100, 1) : 0,
+ accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0,
+ pending: total - reviewed,
duckUsed,
toolCalls,
avgCalls: total ? round(toolCalls / total, 2) : 0,
@@ -104,22 +111,25 @@ export function buildAggregate(modelsReport) {
for (const [name, rep] of Object.entries(modelsReport)) {
let duckUsed = 0, toolCalls = 0;
const gen = [];
+ let reviewed = 0, correct = 0;
for (const s of SCENARIOS) {
const g = rep.scenarios[s];
duckUsed += g.duckUsed;
toolCalls += g.toolCalls;
+ reviewed += g.reviewed;
+ correct += g.correct;
for (const row of rep.perTask.filter((r) => r.scenario === s)) {
gen.push(row.genTokens ?? 0);
}
}
const taskCount = rep.scenarios.control?.tasks ?? 0;
- const correct = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].correct || 0), 0);
const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0);
perModel[name] = {
tasks: taskCount,
totalRows,
+ reviewed,
correct,
- accuracy: totalRows ? round((correct / totalRows) * 100, 1) : 0,
+ accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0,
duckUsed,
toolCalls,
avgGenTokens: round(avg(gen)),
diff --git a/apps/eval/src/review.mjs b/apps/eval/src/review.mjs
new file mode 100644
index 0000000..9749abd
--- /dev/null
+++ b/apps/eval/src/review.mjs
@@ -0,0 +1,205 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import path from "node:path";
+import readline from "node:readline";
+import { buildReport, buildAggregate } from "./report.mjs";
+
+const SCENARIO_LABELS = {
+ control: "Без утки (контроль)",
+ blind: "Слепая утка",
+ mentor: "Утка-помощник",
+};
+
+function esc(s) {
+ return String(s ?? "")
+ .replace(/&/g, "&")
+ .replace(//g, ">");
+}
+
+function parseArgs(argv) {
+ const args = {
+ report: null,
+ out: null,
+ reviewFile: null,
+ model: null,
+ scenario: null,
+ limit: null,
+ };
+ for (let i = 0; i < argv.length; i += 1) {
+ const a = argv[i];
+ const next = () => argv[i + 1];
+ if (a === "--report") args.report = next(), i += 1;
+ else if (a === "--out") args.out = next(), i += 1;
+ else if (a === "--review") args.reviewFile = next(), i += 1;
+ else if (a === "--model") args.model = next(), i += 1;
+ else if (a === "--scenario") args.scenario = next(), i += 1;
+ else if (a === "--limit") args.limit = Number(next()), i += 1;
+ else if (a === "--re-ask") args.reAsk = true;
+ }
+ return args;
+}
+
+async function main() {
+ const args = parseArgs(process.argv.slice(2));
+ const reportFile = args.report || path.join("out", "report.json");
+ const outFile = args.out || path.join("out", "report.reviewed.json");
+ const reviewFile = args.reviewFile || path.join("out", "review.json");
+
+ const report = JSON.parse(await readFile(reportFile, "utf8"));
+
+ let decisions = {};
+ try {
+ decisions = JSON.parse(await readFile(reviewFile, "utf8"));
+ } catch {
+ /* no prior review */
+ }
+
+ const entries = [];
+ for (const [model, rep] of Object.entries(report.models)) {
+ for (const row of rep.perTask) {
+ if (args.model && model !== args.model) continue;
+ if (args.scenario && row.scenario !== args.scenario) continue;
+ entries.push({ model, taskId: row.id, scenario: row.scenario, row });
+ }
+ }
+ if (args.limit && entries.length > args.limit) entries.length = args.limit;
+
+ const key = (e) => `${e.model}|${e.taskId}|${e.scenario}`;
+
+ const isTTY = !!process.stdin.isTTY;
+
+ let batchLines = [];
+ if (!isTTY) {
+ const input = await new Promise((res, rej) => {
+ let data = "";
+ process.stdin.setEncoding("utf8");
+ process.stdin.on("data", (c) => (data += c));
+ process.stdin.on("end", () => res(data));
+ process.stdin.on("error", rej);
+ process.stdin.resume();
+ });
+ batchLines = input.split(/\r?\n/).map((l) => l.trim().toLowerCase());
+ }
+
+ let batchIdx = 0;
+ const nextInput = async () => {
+ if (isTTY) {
+ const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+ const ans = await new Promise((res) => rl.question("", res));
+ rl.close();
+ return ans.trim().toLowerCase();
+ }
+ return batchLines[batchIdx++] ?? "";
+ };
+
+ async function askOne(e, nextInput, decisions) {
+ const k = key(e);
+ const prev = decisions[k];
+ if (prev !== undefined) {
+ console.log(`\n[${e.row.correct === prev ? " " : "*"}] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]} (ранее: ${prev ? "✓" : "✗"})`);
+ } else {
+ console.log(`\n[?] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}`);
+ }
+
+ console.log("────────────────────────────────────────────");
+ console.log(` Вопрос: ${e.row.question}`);
+ console.log(` Ожидаемо: ${e.row.expected}`);
+
+ console.log(` --- ответ модели ---`);
+ const lines = String(e.row.response).split("\n");
+ for (const l of lines) console.log(` ${l}`);
+
+ const cur = e.row.correct === true ? "y" : e.row.correct === false ? "n" : null;
+ const hint = cur ? " [Enter] оставить" : "";
+ process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `);
+ const a = (await nextInput()) || "";
+
+ let decision = null;
+ if (a === "y" || a === "д") decision = true;
+ else if (a === "n" || a === "н") decision = false;
+ else if (a === "" && cur) decision = e.row.correct;
+
+ if (decision == null) {
+ skipped += 1;
+ return 0;
+ }
+
+ if (prev !== decision) changed += 1;
+ decisions[k] = decision;
+ e.row.correct = decision;
+ return 1;
+ }
+
+ let done = 0;
+ let skipped = 0;
+ let changed = 0;
+
+ const reAsk = !!args.reAsk;
+
+ for (const e of entries) {
+ const k = key(e);
+ const prev = decisions[k];
+
+ if (prev === undefined || reAsk) {
+ done += await askOne(e, nextInput, decisions);
+ continue;
+ }
+
+ if (e.row.correct !== prev) changed += 1;
+ e.row.correct = prev;
+ }
+ console.log(`\nГотово. Решено: ${done}, пропущено: ${skipped}, сменено решений: ${changed}`);
+
+ await mkdir(path.dirname(reviewFile), { recursive: true });
+ await writeFile(reviewFile, JSON.stringify(decisions, null, 2), "utf8");
+ console.log(`Решения сохранены: ${path.resolve(reviewFile)}`);
+
+ for (const [model, rep] of Object.entries(report.models)) {
+ report.models[model] = {
+ ...rep,
+ scenarios: buildReport({
+ model,
+ mcpUrl: report.mcpUrl,
+ tasks: [],
+ rows: rep.perTask.map((r) => ({
+ id: r.id,
+ scenario: r.scenario,
+ correct: r.correct,
+ duckUsed: r.duckUsed,
+ toolCalls: r.toolCalls,
+ promptTokens: r.promptTokens,
+ genTokens: r.genTokens,
+ duckTokens: r.duckTokens,
+ response: r.response,
+ expected: r.expected,
+ question: r.question,
+ })),
+ }).scenarios,
+ };
+ }
+
+ report.aggregate = buildAggregate(report.models);
+ report.reviewedAt = new Date().toISOString();
+
+ await mkdir(path.dirname(outFile), { recursive: true });
+ await writeFile(outFile, JSON.stringify(report, null, 2), "utf8");
+ console.log(`Отчёт с ревью сохранён: ${path.resolve(outFile)}`);
+
+ console.log("\n==== Ревью-сводка ====");
+ for (const [model, rep] of Object.entries(report.models)) {
+ const g = report.aggregate.perModel[model];
+ const scen = rep.scenarios;
+ console.log(
+ `${model} | ` +
+ ["control", "blind", "mentor"]
+ .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`)
+ .join(" ") +
+ ` всего rev=${g.reviewed}`
+ );
+ }
+}
+
+main().catch((err) => {
+ console.error(err);
+ process.exit(1);
+});
diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.mjs
index 0724113..9239931 100644
--- a/apps/eval/src/run.mjs
+++ b/apps/eval/src/run.mjs
@@ -3,7 +3,7 @@ import path from "node:path";
import { Ollama } from "../lib/ollama.mjs";
import { McpClient } from "../lib/mcpClient.mjs";
import { runScenario } from "./runner.mjs";
-import { isCorrect, PROMPTS } from "./runner.mjs";
+import { PROMPTS } from "./runner.mjs";
import { buildReport, buildAggregate } from "./report.mjs";
let logTarget = null;
@@ -62,7 +62,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
rows.push({
id: task.id,
scenario,
- correct: false,
+ correct: null,
duckUsed: false,
toolCalls: 0,
promptTokens: 0,
@@ -74,7 +74,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
});
continue;
}
- const correct = isCorrect(r.response, task.answer);
+ const correct = null;
rows.push({
id: task.id,
scenario,
@@ -88,10 +88,9 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
expected: task.answer,
question: task.question,
});
- const mark = correct ? "OK " : "XX ";
const duck = r.duckUsed ? "duck" : "n/a ";
const ms = Date.now() - t0;
- await log(` [${task.id}/${scenario}] ${mark} duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`);
+ await log(` [${task.id}/${scenario}] duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`);
}
}
return rows;
diff --git a/apps/eval/src/runner.mjs b/apps/eval/src/runner.mjs
index 143a87f..954a0d7 100644
--- a/apps/eval/src/runner.mjs
+++ b/apps/eval/src/runner.mjs
@@ -42,59 +42,6 @@ export const PROMPTS = {
const MAX_DUCK_TURNS = 6;
-function normalize(s) {
- return String(s ?? "")
- .toLowerCase()
- .replace(/[^a-zа-яё0-9/]/g, "")
- .replace(/ё/g, "е")
- .trim();
-}
-
-function extractFinalNumber(text) {
- const t = String(text ?? "");
- const re = /(?:ответ|answer|otvet|итог|итак|final)\s*[:：—]?\s*/gi;
- let segment = t;
- let m;
- while ((m = re.exec(t)) !== null) {
- if (m[0].trim()) segment = t.slice(re.lastIndex);
- }
- const digits = segment.match(/\d+/g) || [];
- return digits.length ? digits[digits.length - 1] : null;
-}
-
-function isNumeric(s) {
- return /^\d+(?:[.,]\d+)?$/.test(String(s).trim());
-}
-
-export function isCorrect(predicted, expected) {
- const e = String(expected ?? "").trim();
- const p = String(predicted ?? "");
- if (!e || !p) return false;
-
- // Дробный ответ (например "1/8") — сравниваем вхождение нормализованной дроби.
- if (/\d+\s*\/\s*\d+/.test(e)) {
- const eNorm = normalize(e);
- const pNorm = normalize(p);
- return pNorm.includes(eNorm);
- }
-
- // Числовой ответ — извлекаем финальное число и сравниваем точно.
- if (isNumeric(e)) {
- const cand = extractFinalNumber(p);
- if (cand == null) return false;
- return normalizeNum(cand) === normalizeNum(e);
- }
-
- // Текстовый ответ — нормализованное вхождение слова.
- const eNorm = normalize(e);
- const pNorm = normalize(p);
- return pNorm.includes(eNorm);
-}
-
-function normalizeNum(n) {
- return String(n).replace(/[.,]/g, "").replace(/^0+(?=\d)/, "");
-}
-
function toolDefinition() {
return DUCK_TOOL;
}
diff --git a/package.json b/package.json
index 6a74f76..c6f0f74 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,10 @@
"dev-mcp": "pnpm --dir apps/mcp dev",
"typecheck-mcp": "pnpm --dir apps/mcp exec tsc --noEmit",
"deploy-mcp": "vercel --cwd apps/mcp --prod --yes",
- "deploy-frontend": "vercel --cwd apps/frontend --prod --yes"
+ "deploy-frontend": "vercel --cwd apps/frontend --prod --yes",
+ "eval:run": "pnpm --dir apps/eval exec node src/run.mjs",
+ "eval:review": "pnpm --dir apps/eval exec node src/review.mjs",
+ "eval:html": "pnpm --dir apps/eval exec node src/report-html.mjs"
},
"license": "ISC",
"devEngines": {