feat: add sveltekit pages, generate reports

This commit is contained in:
2026-09-04 18:02:28 +05:00
parent 20bd9cfe88
commit a643e6e4a3
15 changed files with 2675 additions and 411 deletions
-1
View File
@@ -5,7 +5,6 @@
"scripts": {
"run": "tsx src/run.ts",
"review": "tsx src/review.ts",
"report-html": "tsx src/report-html.ts",
"typecheck": "tsc --noEmit"
},
"license": "ISC",
-390
View File
@@ -1,390 +0,0 @@
import { readFile, writeFile, mkdir } from "node:fs/promises";
import path from "node:path";
import type { ReportRoot, ScenarioName, ScenarioSummary, AggregateScenario, PerTaskRow } from "@duck/types";
import { SCENARIO_NAMES } from "@duck/types";
const SCENARIO_LABELS: Record<ScenarioName, string> = {
control: "Без утки (контроль)",
thinking: "Думать вслух",
blind: "Слепая утка",
mentor: "Утка-помощник",
};
const SCENARIO_DESCS: Record<ScenarioName, string> = {
control: "Модель решает задачу напрямую, без инструментов.",
thinking: "Модель выписывает рассуждения вслух, без дука.",
blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.",
mentor: "Модель работает в паре, зная, что ответ будет только «quack».",
};
function esc(s: unknown): string {
return String(s ?? "")
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
function nl2br(s: string): string {
return esc(s).replace(/\n/g, "<br>");
}
function pctColor(p: number): string {
return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444";
}
function modelShort(name: string): string {
const base = String(name).split(/:/)[0];
const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : "";
return esc(base + ver);
}
interface RowHtmlItem extends PerTaskRow {
model: string;
}
function summaryTable(report: ReportRoot): string {
const perModel = report.aggregate.perModel;
const models = Object.keys(perModel);
const rows = models
.map((m) => {
const g = perModel[m];
const scen = report.models[m].scenarios;
const acc = (s: ScenarioName) => scen[s].accuracy;
const row = `
<td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td>
<td class="pm-num" data-sc="thinking" style="color:${pctColor(acc("thinking"))}">${acc("thinking")}%</td>
<td class="pm-num" data-sc="blind" style="color:${pctColor(acc("blind"))}">${acc("blind")}%</td>
<td class="pm-num" data-sc="mentor" style="color:${pctColor(acc("mentor"))}">${acc("mentor")}%</td>
<td class="pm-num">${g.accuracy}%</td>
<td class="pm-num">${g.reviewed}<span class="muted"> / ${g.totalRows}</span></td>
<td class="pm-num">${g.duckUsed}</td>
<td class="pm-num">${g.toolCalls}</td>
<td class="pm-num">${g.avgGenTokens}</td>
`;
return `
<tr>
<td class="pm-name" data-model="${esc(m)}">${modelShort(m)}</td>
${row}
</tr>`;
})
.join("");
return `
<div class="summary-shell">
<table class="summary">
<thead>
<tr>
<th>Модель</th>
<th>Контроль</th>
<th>Мысли</th>
<th>Слепая</th>
<th>Помощник</th>
<th>Средняя acc</th>
<th>Размечено</th>
<th>Утка</th>
<th>Вызовов</th>
<th>ср. токены out</th>
</tr>
</thead>
<tbody>${rows}</tbody>
</table>
</div>`;
}
function scenarioCard(key: ScenarioName, agg: AggregateScenario): string {
const pct = agg.accuracy;
const color = pctColor(pct);
const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0;
const pendingBadge = agg.pending > 0 ? `<span class="pending-badge">неразмечено: ${agg.pending}</span>` : "";
const modelBars = Object.entries(agg.byModel)
.map(([m, g]) => {
const c = pctColor(g.accuracy);
const barW = g.reviewed ? g.accuracy : 0;
return `
<div class="agg-model">
<span class="agg-model-name">${modelShort(m)}${g.reviewed ? "" : " <span class='muted'>(?)</span>"}</span>
<div class="bar"><div class="bar-fill" style="width:${barW}%;background:${c}"></div></div>
<span class="agg-model-val" style="color:${c}">${g.accuracy}%</span>
</div>`;
})
.join("");
return `
<div class="card scenario" data-scenario="${key}">
<div class="scenario-head">
<div>
<div class="scenario-title">${esc(SCENARIO_LABELS[key])}</div>
<div class="scenario-desc">${esc(SCENARIO_DESCS[key])}</div>
</div>
<div class="accuracy" style="color:${color}">${pct}%${pendingBadge}</div>
</div>
<div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
<div class="metrics">
<div class="metric"><span class="m-label">Верных</span><span class="m-val">${agg.correct} <span class="m-mod">/ ${agg.reviewed}</span></span><span class="m-sub">размечено из ${agg.tasks}</span></div>
<div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${agg.duckUsed} из ${agg.tasks} задач</span></div>
<div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${agg.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
</div>
<div class="agg-models">${modelBars}</div>
</div>`;
}
function rowHtml(r: RowHtmlItem): string {
const pending = r.correct === null || r.correct === undefined;
const cls = pending ? "pend" : r.correct ? "ok" : "no";
const result = pending ? "?" : r.correct ? "✓" : "✗";
const duck = r.duckUsed ? "🦆" : "—";
return `
<tr class="scenario-row ${cls}" data-scenario="${r.scenario}" data-model="${esc(r.model)}">
<td class="r-model">${modelShort(r.model)}</td>
<td class="r-id">${esc(r.id)}</td>
<td class="r-s">${esc(SCENARIO_LABELS[r.scenario])}</td>
<td class="r-result">${result}</td>
<td class="r-duck">${duck} <span class="muted">(${r.toolCalls})</span></td>
<td class="r-question">${nl2br(r.question)}</td>
<td class="r-expected">${esc(r.expected)}</td>
<td class="r-tokens">${r.promptTokens}<span class="muted"> / </span>${r.genTokens}</td>
<td class="r-answer">${nl2br(r.response)}</td>
</tr>`;
}
function promptBlock(report: ReportRoot): string {
const p = report.prompts;
const items = (SCENARIO_NAMES as readonly ScenarioName[])
.filter((k): k is ScenarioName => !!p[k])
.map(
(k) => `
<div class="prompt-item">
<div class="prompt-name">${esc(SCENARIO_LABELS[k])}</div>
<div class="prompt-text">${esc(p[k])}</div>
</div>`
)
.join("");
if (!items) return "";
return `
<details class="prompts" open>
<summary>Промпты (то, что передаётся модели)</summary>
<div class="prompts-body">${items}</div>
</details>`;
}
export function buildHtml(report: ReportRoot): string {
const perTask: RowHtmlItem[] = Object.entries(report.models).flatMap(([model, rep]) =>
rep.perTask.map((r) => ({ ...r, model }))
);
const rows = perTask.map(rowHtml).join("");
const models = Object.keys(report.aggregate.perModel);
let totalPending = 0, totalReviewed = 0;
for (const rep of Object.values(report.models)) {
for (const s of SCENARIO_NAMES) {
if (rep.scenarios[s]) {
totalReviewed += rep.scenarios[s].reviewed || 0;
totalPending += rep.scenarios[s].pending || 0;
}
}
}
const reviewNote = totalPending > 0
? `<div class="review-note">⚠ Не все ответы размечены: <b>${totalReviewed}</b> проверено, <b>${totalPending}</b> ожидают ревью (показаны символом «?»). Запустите <code>node src/review.mjs</code> для проставления меток.</div>`
: "";
const modelOptions = models
.map((m) => `<option value="${esc(m)}">${modelShort(m)}</option>`)
.join("");
const aggCards = (SCENARIO_NAMES as readonly ScenarioName[])
.filter((k): k is ScenarioName => !!report.aggregate.perScenario[k])
.map((k) => scenarioCard(k, report.aggregate.perScenario[k]))
.join("");
return `<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Rubber Duck MCP — отчёт тестирования</title>
<style>
:root {
--bg:#0f172a; --panel:#1e293b; --panel2:#273449; --text:#e2e8f0;
--muted:#94a3b8; --ok:#10b981; --no:#f87171; --border:#334155;
}
* { box-sizing:border-box; }
body { margin:0; font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;
background:var(--bg); color:var(--text); line-height:1.5; }
.wrap { max-width:1280px; margin:0 auto; padding:24px 20px 60px; }
header h1 { margin:0 0 4px; font-size:22px; }
.meta { color:var(--muted); font-size:13px; }
.meta b { color:var(--text); font-weight:600; }
h2 { font-size:17px; margin:26px 0 12px; }
.summary-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; }
table.summary { width:100%; border-collapse:collapse; font-size:13px; }
table.summary th { background:var(--panel2); text-align:left; padding:10px 14px;
font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; }
table.summary td { padding:10px 14px; border-top:1px solid var(--border); }
.pm-name { font-weight:700; white-space:nowrap; }
.pm-num { text-align:right; font-variant-numeric:tabular-nums; }
.cards { display:grid; grid-template-columns:repeat(4,1fr); gap:16px; margin:0 0 8px; }
@media (max-width:1100px){ .cards { grid-template-columns:repeat(2,1fr); } }
@media (max-width:640px){ .cards { grid-template-columns:1fr; } }
.card { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; }
.scenario-head { display:flex; justify-content:space-between; align-items:flex-start; gap:12px; }
.scenario-title { font-size:16px; font-weight:700; }
.scenario-desc { color:var(--muted); font-size:12px; margin-top:4px; }
.accuracy { font-size:32px; font-weight:800; line-height:1; }
.pending-badge { display:block; font-size:11px; font-weight:600; color:var(--muted); margin-top:4px; }
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
.bar-fill { height:100%; border-radius:99px; transition:width .4s; }
.metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; }
.metric { background:var(--panel2); border-radius:8px; padding:10px 12px; }
.m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.05em; white-space:nowrap; }
.m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; }
.m-mod { font-size:14px; font-weight:600; color:var(--muted); }
.m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
.agg-models { margin-top:14px; display:grid; gap:8px; }
.agg-model { display:grid; grid-template-columns:auto 1fr auto; align-items:center; gap:10px; font-size:12px; }
.agg-model .bar { margin:0; }
.agg-model-name { color:var(--muted); white-space:nowrap; }
.agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; }
.prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
.review-note { margin:14px 0; padding:10px 14px; background:rgba(245,158,11,.12); border:1px solid rgba(245,158,11,.4);
border-radius:10px; color:#fbbf24; font-size:13px; }
.review-note code { background:var(--panel2); padding:1px 6px; border-radius:5px; }
.prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
.prompts summary::-webkit-details-marker { display:none; }
.prompts summary::before { content:"▸ "; color:var(--muted); }
.prompts[open] summary::before { content:"▾ "; }
.prompts-body { padding:0 16px 14px; display:grid; gap:12px; }
.prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; }
.prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; }
.prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; }
.filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
.filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
.filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
.filters select { background:var(--panel); color:var(--text); border:1px solid var(--border);
padding:7px 10px; border-radius:8px; font-size:13px; }
.tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; }
table { width:100%; border-collapse:collapse; font-size:13px; }
thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px;
font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; }
tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; }
.scenario-row.ok .r-result { color:var(--ok); font-weight:800; }
.scenario-row.no .r-result { color:var(--no); font-weight:800; }
.scenario-row.pend .r-result { color:var(--muted); font-weight:800; }
.scenario-row.pend { opacity:.75; } .r-model { font-weight:600; white-space:nowrap; }
.r-id { font-weight:700; color:var(--muted); }
.r-s { white-space:nowrap; }
.r-duck { text-align:center; white-space:nowrap; }
.r-question { max-width:300px; color:var(--text); }
.r-answer { max-width:380px; }
.muted { color:var(--muted); }
.hidden { display:none !important; }
footer { margin-top:20px; color:var(--muted); font-size:12px; }
</style>
</head>
<body>
<div class="wrap">
<header>
<h1>🦆 Rubber Duck MCP — отчёт тестирования</h1>
<div class="meta">
MCP: <b>${esc(report.mcpUrl)}</b> &nbsp;·&nbsp;
Моделей: <b>${models.length}</b> &nbsp;·&nbsp;
Дата: <b>${esc(report.generatedAt)}</b>
</div>
</header>
<h2>Сводка по моделям</h2>
${summaryTable(report)}
${reviewNote}
<h2>Агрегат по сценариям (все модели)</h2>
<div class="cards">${aggCards}</div>
${promptBlock(report)}
<div class="filters" id="filters">
<label style="font-size:12px;color:var(--muted)">Модель:</label>
<select id="modelSel">
<option value="all">Все модели</option>
${modelOptions}
</select>
<button data-filter="all" class="active">Все сценарии</button>
<button data-filter="control">Без утки</button>
<button data-filter="thinking">Мысли</button>
<button data-filter="blind">Слепая утка</button>
<button data-filter="mentor">Утка-помощник</button>
<span style="margin-left:auto;color:var(--muted);font-size:12px" id="count"></span>
</div>
<div class="tbl-shell">
<table>
<thead>
<tr>
<th>Модель</th><th>#</th><th>Сценарий</th><th>✓</th><th>Утка</th>
<th>Вопрос</th><th>Ожид.</th><th>in/out</th><th>Ответ модели</th>
</tr>
</thead>
<tbody>${rows}</tbody>
</table>
</div>
<footer>Генерировано локальным харнессом apps/eval. temperature=0. Прогон по нескольким локальным моделям через Ollama.</footer>
</div>
<script>
const buttons = document.querySelectorAll('#filters button');
const modelSel = document.getElementById('modelSel');
const rows2 = document.querySelectorAll('tbody .scenario-row');
const count = document.getElementById('count');
let sc = 'all';
let mdl = 'all';
function apply() {
let n = 0;
rows2.forEach(r => {
const okSc = sc === 'all' || r.dataset.scenario === sc;
const okM = mdl === 'all' || r.dataset.model === mdl;
const show = okSc && okM;
r.classList.toggle('hidden', !show);
if (show) n++;
});
count.textContent = 'показано: ' + n + ' из ' + rows2.length;
}
buttons.forEach(b => b.addEventListener('click', () => {
buttons.forEach(x => x.classList.remove('active'));
b.classList.add('active');
sc = b.dataset.filter;
apply();
}));
modelSel.addEventListener('change', () => { mdl = modelSel.value; apply(); });
apply('all');
</script>
</body>
</html>`;
}
async function main(): Promise<void> {
const args = process.argv.slice(2);
let inFile: string | null = null, outFile = "out/report.html";
for (let i = 0; i < args.length; i += 1) {
const a = args[i];
const next = () => args[i + 1];
if (a === "--in") inFile = next(), i += 1;
else if (a.startsWith("--in=")) inFile = a.slice(5);
else if (a === "--out") outFile = next(), i += 1;
else if (a.startsWith("--out=")) outFile = a.slice(6);
}
if (!inFile) {
const reviewed = path.join("out", "report.reviewed.json");
const base = path.join("out", "report.json");
inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base));
}
const report: ReportRoot = JSON.parse(await readFile(inFile, "utf8"));
const html = buildHtml(report);
await mkdir(path.dirname(outFile), { recursive: true });
await writeFile(outFile, html, "utf8");
console.log(`HTML report written: ${path.resolve(outFile)} (source: ${inFile})`);
}
if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) {
main().catch((e) => {
console.error(e);
process.exit(1);
});
}
+1 -7
View File
@@ -16,11 +16,9 @@ export interface BuildReportOpts {
model: string;
mcpUrl: string;
rows: PerTaskRow[];
meta?: Record<string, unknown>;
prompts?: Partial<Record<ScenarioName, string>>;
}
export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: BuildReportOpts): ModelReport {
export function buildReport({ model, mcpUrl, rows }: BuildReportOpts): ModelReport {
const scenarios = SCENARIO_NAMES;
const byScenario: Record<string, { total: number; correct: number; reviewed: number; duckUsed: number; toolCalls: number; promptTokens: number[]; genTokens: number[]; duckTokens: number[]; rows: PerTaskRow[] }> = {};
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
@@ -60,11 +58,8 @@ export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: Bu
}
return {
generatedAt: new Date().toISOString(),
model,
mcpUrl,
scenarios: summaries,
prompts: prompts as Record<ScenarioName, string>,
perTask: rows.map((r) => ({
id: r.id,
scenario: r.scenario,
@@ -78,7 +73,6 @@ export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: Bu
expected: r.expected,
question: r.question,
})),
meta,
};
}