Ku6epXBOCTuK/rubber-duck-mcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: manual review results flow

Browse Source
This commit is contained in:
Ku6epXBOCTuK
2026-09-04 06:49:56 +05:00
parent f4f48edf14
commit 6a1c7666c4
6 changed files with 288 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/eval/src/report-html.mjs
+54 -13
View File
@@ -44,16 +44,20 @@ function summaryTable(report) {
const g = perModel[m];
const scen = report.models[m].scenarios;
const acc = (s) => scen[s].accuracy;
return `
<tr>
<td class="pm-name" data-model="${esc(m)}">${modelShort(m)}</td>
const row = `
<td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td>
<td class="pm-num" data-sc="blind" style="color:${pctColor(acc("blind"))}">${acc("blind")}%</td>
<td class="pm-num" data-sc="mentor" style="color:${pctColor(acc("mentor"))}">${acc("mentor")}%</td>
<td class="pm-num">${g.accuracy}%</td>
<td class="pm-num">${g.reviewed}<span class="muted"> / ${g.totalRows}</span></td>
<td class="pm-num">${g.duckUsed}</td>
<td class="pm-num">${g.toolCalls}</td>
<td class="pm-num">${g.avgGenTokens}</td>
`;
return `
<tr>
<td class="pm-name" data-model="${esc(m)}">${modelShort(m)}</td>
${row}
</tr>`;
})
.join("");
@@ -67,6 +71,7 @@ function summaryTable(report) {
<th>Слепая</th>
<th>Помощник</th>
<th>Средняя acc</th>
<th>Размечено</th>
<th>Уток</th>
<th>Вызовов</th>
<th>ср. токены out</th>
@@ -81,13 +86,15 @@ function scenarioCard(key, agg) {
const pct = agg.accuracy;
const color = pctColor(pct);
const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0;
const pendingBadge = agg.pending > 0 ? `<span class="pending-badge">неразмечено: ${agg.pending}</span>` : "";
const modelBars = Object.entries(agg.byModel)
.map(([m, g]) => {
const c = pctColor(g.accuracy);
const barW = g.reviewed ? g.accuracy : 0;
return `
<div class="agg-model">
<span class="agg-model-name">${modelShort(m)}</span>
<div class="bar"><div class="bar-fill" style="width:${g.accuracy}%;background:${c}"></div></div>
<span class="agg-model-name">${modelShort(m)}${g.reviewed ? "" : " <span class='muted'>(?)</span>"}</span>
<div class="bar"><div class="bar-fill" style="width:${barW}%;background:${c}"></div></div>
<span class="agg-model-val" style="color:${c}">${g.accuracy}%</span>
</div>`;
})
@@ -99,11 +106,11 @@ function scenarioCard(key, agg) {
<div class="scenario-title">${esc(SCENARIO_LABELS[key])}</div>
<div class="scenario-desc">${esc(SCENARIO_DESCS[key])}</div>
</div>
<div class="accuracy" style="color:${color}">${pct}%</div>
<div class="accuracy" style="color:${color}">${pct}%${pendingBadge}</div>
</div>
<div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
<div class="metrics">
<div class="metric"><span class="m-label">Верных</span><span class="m-val">${agg.correct} <span class="m-mod">/ ${agg.tasks}</span></span></div>
<div class="metric"><span class="m-label">Верных</span><span class="m-val">${agg.correct} <span class="m-mod">/ ${agg.reviewed}</span></span><span class="m-sub">размечено из ${agg.tasks}</span></div>
<div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${agg.duckUsed} из ${agg.tasks} задач</span></div>
<div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${agg.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
</div>
@@ -112,14 +119,16 @@ function scenarioCard(key, agg) {
}
function rowHtml(r) {
const cls = r.correct ? "ok" : "no";
const pending = r.correct === null || r.correct === undefined;
const cls = pending ? "pend" : r.correct ? "ok" : "no";
const result = pending ? "?" : r.correct ? "✓" : "✗";
const duck = r.duckUsed ? "🦆" : "—";
return `
<tr class="scenario-row ${cls}" data-scenario="${r.scenario}" data-model="${esc(r.model)}">
<td class="r-model">${modelShort(r.model)}</td>
<td class="r-id">${esc(r.id)}</td>
<td class="r-s">${esc(SCENARIO_LABELS[r.scenario])}</td>
<td class="r-result">${r.correct ? "✓" : "✗"}</td>
<td class="r-result">${result}</td>
<td class="r-duck">${duck} <span class="muted">(${r.toolCalls})</span></td>
<td class="r-question">${nl2br(r.question)}</td>
<td class="r-expected">${esc(r.expected)}</td>
@@ -154,6 +163,18 @@ export function buildHtml(report) {
);
const rows = perTask.map(rowHtml).join("");
const models = Object.keys(report.aggregate.perModel);
let totalPending = 0, totalReviewed = 0;
for (const rep of Object.values(report.models)) {
for (const s of SCENARIOS) {
if (rep.scenarios[s]) {
totalReviewed += rep.scenarios[s].reviewed || 0;
totalPending += rep.scenarios[s].pending || 0;
}
}
}
const reviewNote = totalPending > 0
? `<div class="review-note">⚠ Не все ответы размечены: <b>${totalReviewed}</b> проверено, <b>${totalPending}</b> ожидают ревью (показаны символом «?»). Запустите <code>node src/review.mjs</code> для проставления меток.</div>`
: "";
const modelOptions = models
.map((m) => `<option value="${esc(m)}">${modelShort(m)}</option>`)
.join("");
@@ -195,6 +216,7 @@ export function buildHtml(report) {
.scenario-title { font-size:16px; font-weight:700; }
.scenario-desc { color:var(--muted); font-size:12px; margin-top:4px; }
.accuracy { font-size:32px; font-weight:800; line-height:1; }
.pending-badge { display:block; font-size:11px; font-weight:600; color:var(--muted); margin-top:4px; }
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
.bar-fill { height:100%; border-radius:99px; transition:width .4s; }
.metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; }
@@ -209,6 +231,9 @@ export function buildHtml(report) {
.agg-model-name { color:var(--muted); white-space:nowrap; }
.agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; }
.prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
.review-note { margin:14px 0; padding:10px 14px; background:rgba(245,158,11,.12); border:1px solid rgba(245,158,11,.4);
border-radius:10px; color:#fbbf24; font-size:13px; }
.review-note code { background:var(--panel2); padding:1px 6px; border-radius:5px; }
.prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
.prompts summary::-webkit-details-marker { display:none; }
.prompts summary::before { content:"▸ "; color:var(--muted); }
@@ -230,7 +255,8 @@ export function buildHtml(report) {
tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; }
.scenario-row.ok .r-result { color:var(--ok); font-weight:800; }
.scenario-row.no .r-result { color:var(--no); font-weight:800; }
.r-model { font-weight:600; white-space:nowrap; }
.scenario-row.pend .r-result { color:var(--muted); font-weight:800; }
.scenario-row.pend { opacity:.75; } .r-model { font-weight:600; white-space:nowrap; }
.r-id { font-weight:700; color:var(--muted); }
.r-s { white-space:nowrap; }
.r-duck { text-align:center; white-space:nowrap; }
@@ -255,6 +281,8 @@ export function buildHtml(report) {
<h2>Сводка по моделям</h2>
${summaryTable(report)}
${reviewNote}
<h2>Агрегат по сценариям (все модели)</h2>
<div class="cards">${aggCards}</div>
@@ -321,14 +349,27 @@ export function buildHtml(report) {
async function main() {
const args = process.argv.slice(2);
const inFile = args.find((a) => a.startsWith("--in="))?.slice(5) || "out/report.json";
const outFile = args.find((a) => a.startsWith("--out="))?.slice(6) || "out/report.html";
let inFile = null, outFile = "out/report.html";
for (let i = 0; i < args.length; i += 1) {
const a = args[i];
const next = () => args[i + 1];
if (a === "--in") inFile = next(), i += 1;
else if (a.startsWith("--in=")) inFile = a.slice(5);
else if (a === "--out") outFile = next(), i += 1;
else if (a.startsWith("--out=")) outFile = a.slice(6);
}
if (!inFile) {
const reviewed = path.join("out", "report.reviewed.json");
const base = path.join("out", "report.json");
inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base));
}
const report = JSON.parse(await readFile(inFile, "utf8"));
const html = buildHtml(report);
await mkdir(path.dirname(outFile), { recursive: true });
await writeFile(outFile, html, "utf8");
console.log(`HTML report written: ${path.resolve(outFile)}`);
console.log(`HTML report written: ${path.resolve(outFile)} (source: ${inFile})`);
}
if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) {
apps/eval/src/report.mjs
+21 -11
View File
@@ -1,5 +1,3 @@
import { isCorrect } from "./runner.mjs";
function round(x, d = 2) {
if (!Number.isFinite(x)) return x;
const f = 10 ** d;
@@ -13,13 +11,16 @@ function avg(arr) {
export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "blind", "mentor"];
const byScenario = {};
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
for (const row of rows) {
const s = row.scenario;
if (!byScenario[s]) continue;
byScenario[s].total += 1;
if (row.correct) byScenario[s].correct += 1;
if (row.correct !== null && row.correct !== undefined) {
byScenario[s].reviewed += 1;
if (row.correct) byScenario[s].correct += 1;
}
if (row.duckUsed) byScenario[s].duckUsed += 1;
byScenario[s].toolCalls += row.toolCalls ?? 0;
byScenario[s].promptTokens.push(row.promptTokens ?? 0);
@@ -31,10 +32,13 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
const summaries = {};
for (const s of scenarios) {
const g = byScenario[s];
const denom = g.reviewed || 1;
summaries[s] = {
tasks: g.total,
reviewed: g.reviewed,
correct: g.correct,
accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0,
accuracy: g.reviewed ? round((g.correct / g.reviewed) * 100, 1) : 0,
pending: g.total - g.reviewed,
duckUsed: g.duckUsed,
toolCalls: g.toolCalls,
avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0,
@@ -72,27 +76,30 @@ const SCENARIOS = ["control", "blind", "mentor"];
export function buildAggregate(modelsReport) {
const perScenario = {};
for (const s of SCENARIOS) {
let total = 0, correct = 0, duckUsed = 0, toolCalls = 0;
const gen = [];
let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0;
const byModel = {};
for (const [name, rep] of Object.entries(modelsReport)) {
const g = rep.scenarios[s];
if (!g) continue;
total += g.tasks;
reviewed += g.reviewed;
correct += g.correct;
duckUsed += g.duckUsed;
toolCalls += g.toolCalls;
byModel[name] = {
tasks: g.tasks,
reviewed: g.reviewed,
correct: g.correct,
accuracy: g.accuracy,
accuracy: g.reviewed ? g.accuracy : 0,
duckUsed: g.duckUsed,
};
}
perScenario[s] = {
tasks: total,
reviewed,
correct,
accuracy: total ? round((correct / total) * 100, 1) : 0,
accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0,
pending: total - reviewed,
duckUsed,
toolCalls,
avgCalls: total ? round(toolCalls / total, 2) : 0,
@@ -104,22 +111,25 @@ export function buildAggregate(modelsReport) {
for (const [name, rep] of Object.entries(modelsReport)) {
let duckUsed = 0, toolCalls = 0;
const gen = [];
let reviewed = 0, correct = 0;
for (const s of SCENARIOS) {
const g = rep.scenarios[s];
duckUsed += g.duckUsed;
toolCalls += g.toolCalls;
reviewed += g.reviewed;
correct += g.correct;
for (const row of rep.perTask.filter((r) => r.scenario === s)) {
gen.push(row.genTokens ?? 0);
}
}
const taskCount = rep.scenarios.control?.tasks ?? 0;
const correct = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].correct || 0), 0);
const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0);
perModel[name] = {
tasks: taskCount,
totalRows,
reviewed,
correct,
accuracy: totalRows ? round((correct / totalRows) * 100, 1) : 0,
accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0,
duckUsed,
toolCalls,
avgGenTokens: round(avg(gen)),
apps/eval/src/review.mjs
+205
View File
@@ -0,0 +1,205 @@
import { readFile, writeFile, mkdir } from "node:fs/promises";
import path from "node:path";
import readline from "node:readline";
import { buildReport, buildAggregate } from "./report.mjs";
const SCENARIO_LABELS = {
control: "Без утки (контроль)",
blind: "Слепая утка",
mentor: "Утка-помощник",
};
function esc(s) {
return String(s ?? "")
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
function parseArgs(argv) {
const args = {
report: null,
out: null,
reviewFile: null,
model: null,
scenario: null,
limit: null,
};
for (let i = 0; i < argv.length; i += 1) {
const a = argv[i];
const next = () => argv[i + 1];
if (a === "--report") args.report = next(), i += 1;
else if (a === "--out") args.out = next(), i += 1;
else if (a === "--review") args.reviewFile = next(), i += 1;
else if (a === "--model") args.model = next(), i += 1;
else if (a === "--scenario") args.scenario = next(), i += 1;
else if (a === "--limit") args.limit = Number(next()), i += 1;
else if (a === "--re-ask") args.reAsk = true;
}
return args;
}
async function main() {
const args = parseArgs(process.argv.slice(2));
const reportFile = args.report || path.join("out", "report.json");
const outFile = args.out || path.join("out", "report.reviewed.json");
const reviewFile = args.reviewFile || path.join("out", "review.json");
const report = JSON.parse(await readFile(reportFile, "utf8"));
let decisions = {};
try {
decisions = JSON.parse(await readFile(reviewFile, "utf8"));
} catch {
/* no prior review */
}
const entries = [];
for (const [model, rep] of Object.entries(report.models)) {
for (const row of rep.perTask) {
if (args.model && model !== args.model) continue;
if (args.scenario && row.scenario !== args.scenario) continue;
entries.push({ model, taskId: row.id, scenario: row.scenario, row });
}
}
if (args.limit && entries.length > args.limit) entries.length = args.limit;
const key = (e) => `${e.model}|${e.taskId}|${e.scenario}`;
const isTTY = !!process.stdin.isTTY;
let batchLines = [];
if (!isTTY) {
const input = await new Promise((res, rej) => {
let data = "";
process.stdin.setEncoding("utf8");
process.stdin.on("data", (c) => (data += c));
process.stdin.on("end", () => res(data));
process.stdin.on("error", rej);
process.stdin.resume();
});
batchLines = input.split(/\r?\n/).map((l) => l.trim().toLowerCase());
}
let batchIdx = 0;
const nextInput = async () => {
if (isTTY) {
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
const ans = await new Promise((res) => rl.question("", res));
rl.close();
return ans.trim().toLowerCase();
}
return batchLines[batchIdx++] ?? "";
};
async function askOne(e, nextInput, decisions) {
const k = key(e);
const prev = decisions[k];
if (prev !== undefined) {
console.log(`\n[${e.row.correct === prev ? " " : "*"}] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]} (ранее: ${prev ? "✓" : "✗"})`);
} else {
console.log(`\n[?] ${e.model} / ${e.taskId} / ${SCENARIO_LABELS[e.scenario]}`);
}
console.log("────────────────────────────────────────────");
console.log(` Вопрос: ${e.row.question}`);
console.log(` Ожидаемо: ${e.row.expected}`);
console.log(` --- ответ модели ---`);
const lines = String(e.row.response).split("\n");
for (const l of lines) console.log(` ${l}`);
const cur = e.row.correct === true ? "y" : e.row.correct === false ? "n" : null;
const hint = cur ? " [Enter] оставить" : "";
process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `);
const a = (await nextInput()) || "";
let decision = null;
if (a === "y" || a === "д") decision = true;
else if (a === "n" || a === "н") decision = false;
else if (a === "" && cur) decision = e.row.correct;
if (decision == null) {
skipped += 1;
return 0;
}
if (prev !== decision) changed += 1;
decisions[k] = decision;
e.row.correct = decision;
return 1;
}
let done = 0;
let skipped = 0;
let changed = 0;
const reAsk = !!args.reAsk;
for (const e of entries) {
const k = key(e);
const prev = decisions[k];
if (prev === undefined || reAsk) {
done += await askOne(e, nextInput, decisions);
continue;
}
if (e.row.correct !== prev) changed += 1;
e.row.correct = prev;
}
console.log(`\nГотово. Решено: ${done}, пропущено: ${skipped}, сменено решений: ${changed}`);
await mkdir(path.dirname(reviewFile), { recursive: true });
await writeFile(reviewFile, JSON.stringify(decisions, null, 2), "utf8");
console.log(`Решения сохранены: ${path.resolve(reviewFile)}`);
for (const [model, rep] of Object.entries(report.models)) {
report.models[model] = {
...rep,
scenarios: buildReport({
model,
mcpUrl: report.mcpUrl,
tasks: [],
rows: rep.perTask.map((r) => ({
id: r.id,
scenario: r.scenario,
correct: r.correct,
duckUsed: r.duckUsed,
toolCalls: r.toolCalls,
promptTokens: r.promptTokens,
genTokens: r.genTokens,
duckTokens: r.duckTokens,
response: r.response,
expected: r.expected,
question: r.question,
})),
}).scenarios,
};
}
report.aggregate = buildAggregate(report.models);
report.reviewedAt = new Date().toISOString();
await mkdir(path.dirname(outFile), { recursive: true });
await writeFile(outFile, JSON.stringify(report, null, 2), "utf8");
console.log(`Отчёт с ревью сохранён: ${path.resolve(outFile)}`);
console.log("\n==== Ревью-сводка ====");
for (const [model, rep] of Object.entries(report.models)) {
const g = report.aggregate.perModel[model];
const scen = rep.scenarios;
console.log(
`${model} | ` +
["control", "blind", "mentor"]
.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`)
.join(" ") +
` всего rev=${g.reviewed}`
);
}
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
apps/eval/src/run.mjs
+4 -5
View File
@@ -3,7 +3,7 @@ import path from "node:path";
import { Ollama } from "../lib/ollama.mjs";
import { McpClient } from "../lib/mcpClient.mjs";
import { runScenario } from "./runner.mjs";
import { isCorrect, PROMPTS } from "./runner.mjs";
import { PROMPTS } from "./runner.mjs";
import { buildReport, buildAggregate } from "./report.mjs";
let logTarget = null;
@@ -62,7 +62,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
rows.push({
id: task.id,
scenario,
correct: false,
correct: null,
duckUsed: false,
toolCalls: 0,
promptTokens: 0,
@@ -74,7 +74,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
});
continue;
}
const correct = isCorrect(r.response, task.answer);
const correct = null;
rows.push({
id: task.id,
scenario,
@@ -88,10 +88,9 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
expected: task.answer,
question: task.question,
});
const mark = correct ? "OK " : "XX ";
const duck = r.duckUsed ? "duck" : "n/a ";
const ms = Date.now() - t0;
await log(` [${task.id}/${scenario}] ${mark} duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`);
await log(` [${task.id}/${scenario}] duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`);
}
}
return rows;
apps/eval/src/runner.mjs
-53
View File
@@ -42,59 +42,6 @@ export const PROMPTS = {
const MAX_DUCK_TURNS = 6;
function normalize(s) {
return String(s ?? "")
.toLowerCase()
.replace(/[^a-zа-яё0-9/]/g, "")
.replace(/ё/g, "е")
.trim();
}
function extractFinalNumber(text) {
const t = String(text ?? "");
const re = /(?:ответ|answer|otvet|итог|итак|final)\s*[:：—]?\s*/gi;
let segment = t;
let m;
while ((m = re.exec(t)) !== null) {
if (m[0].trim()) segment = t.slice(re.lastIndex);
}
const digits = segment.match(/\d+/g) || [];
return digits.length ? digits[digits.length - 1] : null;
}
function isNumeric(s) {
return /^\d+(?:[.,]\d+)?$/.test(String(s).trim());
}
export function isCorrect(predicted, expected) {
const e = String(expected ?? "").trim();
const p = String(predicted ?? "");
if (!e || !p) return false;
// Дробный ответ (например "1/8") — сравниваем вхождение нормализованной дроби.
if (/\d+\s*\/\s*\d+/.test(e)) {
const eNorm = normalize(e);
const pNorm = normalize(p);
return pNorm.includes(eNorm);
}
// Числовой ответ — извлекаем финальное число и сравниваем точно.
if (isNumeric(e)) {
const cand = extractFinalNumber(p);
if (cand == null) return false;
return normalizeNum(cand) === normalizeNum(e);
}
// Текстовый ответ — нормализованное вхождение слова.
const eNorm = normalize(e);
const pNorm = normalize(p);
return pNorm.includes(eNorm);
}
function normalizeNum(n) {
return String(n).replace(/[.,]/g, "").replace(/^0+(?=\d)/, "");
}
function toolDefinition() {
return DUCK_TOOL;
}
package.json
+4 -1
View File
@@ -9,7 +9,10 @@
"dev-mcp": "pnpm --dir apps/mcp dev",
"typecheck-mcp": "pnpm --dir apps/mcp exec tsc --noEmit",
"deploy-mcp": "vercel --cwd apps/mcp --prod --yes",
"deploy-frontend": "vercel --cwd apps/frontend --prod --yes"
"deploy-frontend": "vercel --cwd apps/frontend --prod --yes",
"eval:run": "pnpm --dir apps/eval exec node src/run.mjs",
"eval:review": "pnpm --dir apps/eval exec node src/review.mjs",
"eval:html": "pnpm --dir apps/eval exec node src/report-html.mjs"
},
"license": "ISC",
"devEngines": {