Ku6epXBOCTuK/rubber-duck-mcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add sveltekit pages, generate reports

Browse Source
This commit is contained in:
Ku6epXBOCTuK
2026-09-04 18:02:28 +05:00
parent 20bd9cfe88
commit a643e6e4a3
15 changed files with 2675 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/eval/package.json
-1
View File
@@ -5,7 +5,6 @@
"scripts": {
"run": "tsx src/run.ts",
"review": "tsx src/review.ts",
"report-html": "tsx src/report-html.ts",
"typecheck": "tsc --noEmit"
},
"license": "ISC",
apps/eval/src/report-html.ts
-390
View File
@@ -1,390 +0,0 @@
import { readFile, writeFile, mkdir } from "node:fs/promises";
import path from "node:path";
import type { ReportRoot, ScenarioName, ScenarioSummary, AggregateScenario, PerTaskRow } from "@duck/types";
import { SCENARIO_NAMES } from "@duck/types";
const SCENARIO_LABELS: Record<ScenarioName, string> = {
control: "Без утки (контроль)",
thinking: "Думать вслух",
blind: "Слепая утка",
mentor: "Утка-помощник",
};
const SCENARIO_DESCS: Record<ScenarioName, string> = {
control: "Модель решает задачу напрямую, без инструментов.",
thinking: "Модель выписывает рассуждения вслух, без дука.",
blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.",
mentor: "Модель работает в паре, зная, что ответ будет только «quack».",
};
function esc(s: unknown): string {
return String(s ?? "")
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
function nl2br(s: string): string {
return esc(s).replace(/\n/g, "<br>");
}
function pctColor(p: number): string {
return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444";
}
function modelShort(name: string): string {
const base = String(name).split(/:/)[0];
const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : "";
return esc(base + ver);
}
interface RowHtmlItem extends PerTaskRow {
model: string;
}
function summaryTable(report: ReportRoot): string {
const perModel = report.aggregate.perModel;
const models = Object.keys(perModel);
const rows = models
.map((m) => {
const g = perModel[m];
const scen = report.models[m].scenarios;
const acc = (s: ScenarioName) => scen[s].accuracy;
const row = `
<td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td>
<td class="pm-num" data-sc="thinking" style="color:${pctColor(acc("thinking"))}">${acc("thinking")}%</td>
<td class="pm-num" data-sc="blind" style="color:${pctColor(acc("blind"))}">${acc("blind")}%</td>
<td class="pm-num" data-sc="mentor" style="color:${pctColor(acc("mentor"))}">${acc("mentor")}%</td>
<td class="pm-num">${g.accuracy}%</td>
<td class="pm-num">${g.reviewed}<span class="muted"> / ${g.totalRows}</span></td>
<td class="pm-num">${g.duckUsed}</td>
<td class="pm-num">${g.toolCalls}</td>
<td class="pm-num">${g.avgGenTokens}</td>
`;
return `
<tr>
<td class="pm-name" data-model="${esc(m)}">${modelShort(m)}</td>
${row}
</tr>`;
})
.join("");
return `
<div class="summary-shell">
<table class="summary">
<thead>
<tr>
<th>Модель</th>
<th>Контроль</th>
<th>Мысли</th>
<th>Слепая</th>
<th>Помощник</th>
<th>Средняя acc</th>
<th>Размечено</th>
<th>Утка</th>
<th>Вызовов</th>
<th>ср. токены out</th>
</tr>
</thead>
<tbody>${rows}</tbody>
</table>
</div>`;
}
function scenarioCard(key: ScenarioName, agg: AggregateScenario): string {
const pct = agg.accuracy;
const color = pctColor(pct);
const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0;
const pendingBadge = agg.pending > 0 ? `<span class="pending-badge">неразмечено: ${agg.pending}</span>` : "";
const modelBars = Object.entries(agg.byModel)
.map(([m, g]) => {
const c = pctColor(g.accuracy);
const barW = g.reviewed ? g.accuracy : 0;
return `
<div class="agg-model">
<span class="agg-model-name">${modelShort(m)}${g.reviewed ? "" : " <span class='muted'>(?)</span>"}</span>
<div class="bar"><div class="bar-fill" style="width:${barW}%;background:${c}"></div></div>
<span class="agg-model-val" style="color:${c}">${g.accuracy}%</span>
</div>`;
})
.join("");
return `
<div class="card scenario" data-scenario="${key}">
<div class="scenario-head">
<div>
<div class="scenario-title">${esc(SCENARIO_LABELS[key])}</div>
<div class="scenario-desc">${esc(SCENARIO_DESCS[key])}</div>
</div>
<div class="accuracy" style="color:${color}">${pct}%${pendingBadge}</div>
</div>
<div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
<div class="metrics">
<div class="metric"><span class="m-label">Верных</span><span class="m-val">${agg.correct} <span class="m-mod">/ ${agg.reviewed}</span></span><span class="m-sub">размечено из ${agg.tasks}</span></div>
<div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${agg.duckUsed} из ${agg.tasks} задач</span></div>
<div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${agg.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
</div>
<div class="agg-models">${modelBars}</div>
</div>`;
}
function rowHtml(r: RowHtmlItem): string {
const pending = r.correct === null || r.correct === undefined;
const cls = pending ? "pend" : r.correct ? "ok" : "no";
const result = pending ? "?" : r.correct ? "✓" : "✗";
const duck = r.duckUsed ? "🦆" : "—";
return `
<tr class="scenario-row ${cls}" data-scenario="${r.scenario}" data-model="${esc(r.model)}">
<td class="r-model">${modelShort(r.model)}</td>
<td class="r-id">${esc(r.id)}</td>
<td class="r-s">${esc(SCENARIO_LABELS[r.scenario])}</td>
<td class="r-result">${result}</td>
<td class="r-duck">${duck} <span class="muted">(${r.toolCalls})</span></td>
<td class="r-question">${nl2br(r.question)}</td>
<td class="r-expected">${esc(r.expected)}</td>
<td class="r-tokens">${r.promptTokens}<span class="muted"> / </span>${r.genTokens}</td>
<td class="r-answer">${nl2br(r.response)}</td>
</tr>`;
}
function promptBlock(report: ReportRoot): string {
const p = report.prompts;
const items = (SCENARIO_NAMES as readonly ScenarioName[])
.filter((k): k is ScenarioName => !!p[k])
.map(
(k) => `
<div class="prompt-item">
<div class="prompt-name">${esc(SCENARIO_LABELS[k])}</div>
<div class="prompt-text">${esc(p[k])}</div>
</div>`
)
.join("");
if (!items) return "";
return `
<details class="prompts" open>
<summary>Промпты (то, что передаётся модели)</summary>
<div class="prompts-body">${items}</div>
</details>`;
}
export function buildHtml(report: ReportRoot): string {
const perTask: RowHtmlItem[] = Object.entries(report.models).flatMap(([model, rep]) =>
rep.perTask.map((r) => ({ ...r, model }))
);
const rows = perTask.map(rowHtml).join("");
const models = Object.keys(report.aggregate.perModel);
let totalPending = 0, totalReviewed = 0;
for (const rep of Object.values(report.models)) {
for (const s of SCENARIO_NAMES) {
if (rep.scenarios[s]) {
totalReviewed += rep.scenarios[s].reviewed || 0;
totalPending += rep.scenarios[s].pending || 0;
}
}
}
const reviewNote = totalPending > 0
? `<div class="review-note">⚠ Не все ответы размечены: <b>${totalReviewed}</b> проверено, <b>${totalPending}</b> ожидают ревью (показаны символом «?»). Запустите <code>node src/review.mjs</code> для проставления меток.</div>`
: "";
const modelOptions = models
.map((m) => `<option value="${esc(m)}">${modelShort(m)}</option>`)
.join("");
const aggCards = (SCENARIO_NAMES as readonly ScenarioName[])
.filter((k): k is ScenarioName => !!report.aggregate.perScenario[k])
.map((k) => scenarioCard(k, report.aggregate.perScenario[k]))
.join("");
return `<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Rubber Duck MCP — отчёт тестирования</title>
<style>
:root {
--bg:#0f172a; --panel:#1e293b; --panel2:#273449; --text:#e2e8f0;
--muted:#94a3b8; --ok:#10b981; --no:#f87171; --border:#334155;
}
* { box-sizing:border-box; }
body { margin:0; font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;
background:var(--bg); color:var(--text); line-height:1.5; }
.wrap { max-width:1280px; margin:0 auto; padding:24px 20px 60px; }
header h1 { margin:0 0 4px; font-size:22px; }
.meta { color:var(--muted); font-size:13px; }
.meta b { color:var(--text); font-weight:600; }
h2 { font-size:17px; margin:26px 0 12px; }
.summary-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; }
table.summary { width:100%; border-collapse:collapse; font-size:13px; }
table.summary th { background:var(--panel2); text-align:left; padding:10px 14px;
font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; }
table.summary td { padding:10px 14px; border-top:1px solid var(--border); }
.pm-name { font-weight:700; white-space:nowrap; }
.pm-num { text-align:right; font-variant-numeric:tabular-nums; }
.cards { display:grid; grid-template-columns:repeat(4,1fr); gap:16px; margin:0 0 8px; }
@media (max-width:1100px){ .cards { grid-template-columns:repeat(2,1fr); } }
@media (max-width:640px){ .cards { grid-template-columns:1fr; } }
.card { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; }
.scenario-head { display:flex; justify-content:space-between; align-items:flex-start; gap:12px; }
.scenario-title { font-size:16px; font-weight:700; }
.scenario-desc { color:var(--muted); font-size:12px; margin-top:4px; }
.accuracy { font-size:32px; font-weight:800; line-height:1; }
.pending-badge { display:block; font-size:11px; font-weight:600; color:var(--muted); margin-top:4px; }
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
.bar-fill { height:100%; border-radius:99px; transition:width .4s; }
.metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; }
.metric { background:var(--panel2); border-radius:8px; padding:10px 12px; }
.m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.05em; white-space:nowrap; }
.m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; }
.m-mod { font-size:14px; font-weight:600; color:var(--muted); }
.m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
.agg-models { margin-top:14px; display:grid; gap:8px; }
.agg-model { display:grid; grid-template-columns:auto 1fr auto; align-items:center; gap:10px; font-size:12px; }
.agg-model .bar { margin:0; }
.agg-model-name { color:var(--muted); white-space:nowrap; }
.agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; }
.prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
.review-note { margin:14px 0; padding:10px 14px; background:rgba(245,158,11,.12); border:1px solid rgba(245,158,11,.4);
border-radius:10px; color:#fbbf24; font-size:13px; }
.review-note code { background:var(--panel2); padding:1px 6px; border-radius:5px; }
.prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
.prompts summary::-webkit-details-marker { display:none; }
.prompts summary::before { content:"▸ "; color:var(--muted); }
.prompts[open] summary::before { content:"▾ "; }
.prompts-body { padding:0 16px 14px; display:grid; gap:12px; }
.prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; }
.prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; }
.prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; }
.filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
.filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
.filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
.filters select { background:var(--panel); color:var(--text); border:1px solid var(--border);
padding:7px 10px; border-radius:8px; font-size:13px; }
.tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; }
table { width:100%; border-collapse:collapse; font-size:13px; }
thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px;
font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; }
tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; }
.scenario-row.ok .r-result { color:var(--ok); font-weight:800; }
.scenario-row.no .r-result { color:var(--no); font-weight:800; }
.scenario-row.pend .r-result { color:var(--muted); font-weight:800; }
.scenario-row.pend { opacity:.75; } .r-model { font-weight:600; white-space:nowrap; }
.r-id { font-weight:700; color:var(--muted); }
.r-s { white-space:nowrap; }
.r-duck { text-align:center; white-space:nowrap; }
.r-question { max-width:300px; color:var(--text); }
.r-answer { max-width:380px; }
.muted { color:var(--muted); }
.hidden { display:none !important; }
footer { margin-top:20px; color:var(--muted); font-size:12px; }
</style>
</head>
<body>
<div class="wrap">
<header>
<h1>🦆 Rubber Duck MCP — отчёт тестирования</h1>
<div class="meta">
MCP: <b>${esc(report.mcpUrl)}</b> &nbsp;·&nbsp;
Моделей: <b>${models.length}</b> &nbsp;·&nbsp;
Дата: <b>${esc(report.generatedAt)}</b>
</div>
</header>
<h2>Сводка по моделям</h2>
${summaryTable(report)}
${reviewNote}
<h2>Агрегат по сценариям (все модели)</h2>
<div class="cards">${aggCards}</div>
${promptBlock(report)}
<div class="filters" id="filters">
<label style="font-size:12px;color:var(--muted)">Модель:</label>
<select id="modelSel">
<option value="all">Все модели</option>
${modelOptions}
</select>
<button data-filter="all" class="active">Все сценарии</button>
<button data-filter="control">Без утки</button>
<button data-filter="thinking">Мысли</button>
<button data-filter="blind">Слепая утка</button>
<button data-filter="mentor">Утка-помощник</button>
<span style="margin-left:auto;color:var(--muted);font-size:12px" id="count"></span>
</div>
<div class="tbl-shell">
<table>
<thead>
<tr>
<th>Модель</th><th>#</th><th>Сценарий</th><th>✓</th><th>Утка</th>
<th>Вопрос</th><th>Ожид.</th><th>in/out</th><th>Ответ модели</th>
</tr>
</thead>
<tbody>${rows}</tbody>
</table>
</div>
<footer>Генерировано локальным харнессом apps/eval. temperature=0. Прогон по нескольким локальным моделям через Ollama.</footer>
</div>
<script>
const buttons = document.querySelectorAll('#filters button');
const modelSel = document.getElementById('modelSel');
const rows2 = document.querySelectorAll('tbody .scenario-row');
const count = document.getElementById('count');
let sc = 'all';
let mdl = 'all';
function apply() {
let n = 0;
rows2.forEach(r => {
const okSc = sc === 'all' || r.dataset.scenario === sc;
const okM = mdl === 'all' || r.dataset.model === mdl;
const show = okSc && okM;
r.classList.toggle('hidden', !show);
if (show) n++;
});
count.textContent = 'показано: ' + n + ' из ' + rows2.length;
}
buttons.forEach(b => b.addEventListener('click', () => {
buttons.forEach(x => x.classList.remove('active'));
b.classList.add('active');
sc = b.dataset.filter;
apply();
}));
modelSel.addEventListener('change', () => { mdl = modelSel.value; apply(); });
apply('all');
</script>
</body>
</html>`;
}
async function main(): Promise<void> {
const args = process.argv.slice(2);
let inFile: string | null = null, outFile = "out/report.html";
for (let i = 0; i < args.length; i += 1) {
const a = args[i];
const next = () => args[i + 1];
if (a === "--in") inFile = next(), i += 1;
else if (a.startsWith("--in=")) inFile = a.slice(5);
else if (a === "--out") outFile = next(), i += 1;
else if (a.startsWith("--out=")) outFile = a.slice(6);
}
if (!inFile) {
const reviewed = path.join("out", "report.reviewed.json");
const base = path.join("out", "report.json");
inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base));
}
const report: ReportRoot = JSON.parse(await readFile(inFile, "utf8"));
const html = buildHtml(report);
await mkdir(path.dirname(outFile), { recursive: true });
await writeFile(outFile, html, "utf8");
console.log(`HTML report written: ${path.resolve(outFile)} (source: ${inFile})`);
}
if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) {
main().catch((e) => {
console.error(e);
process.exit(1);
});
}
apps/eval/src/report.ts
+1 -7
View File
@@ -16,11 +16,9 @@ export interface BuildReportOpts {
model: string;
mcpUrl: string;
rows: PerTaskRow[];
meta?: Record<string, unknown>;
prompts?: Partial<Record<ScenarioName, string>>;
}
export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: BuildReportOpts): ModelReport {
export function buildReport({ model, mcpUrl, rows }: BuildReportOpts): ModelReport {
const scenarios = SCENARIO_NAMES;
const byScenario: Record<string, { total: number; correct: number; reviewed: number; duckUsed: number; toolCalls: number; promptTokens: number[]; genTokens: number[]; duckTokens: number[]; rows: PerTaskRow[] }> = {};
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
@@ -60,11 +58,8 @@ export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: Bu
}
return {
generatedAt: new Date().toISOString(),
model,
mcpUrl,
scenarios: summaries,
prompts: prompts as Record<ScenarioName, string>,
perTask: rows.map((r) => ({
id: r.id,
scenario: r.scenario,
@@ -78,7 +73,6 @@ export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: Bu
expected: r.expected,
question: r.question,
})),
meta,
};
}
apps/web/src/app.css
+39
View File
@@ -0,0 +1,39 @@
:root {
--bg: #0f172a;
--panel: #1e293b;
--panel2: #273449;
--text: #e2e8f0;
--muted: #94a3b8;
--ok: #10b981;
--no: #f87171;
--warn: #f59e0b;
--border: #334155;
}
* {
box-sizing: border-box;
}
html,
body {
margin: 0;
background: var(--bg);
color: var(--text);
font-family: -apple-system, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
line-height: 1.5;
}
a {
color: #60a5fa;
}
code {
background: var(--panel2);
padding: 1px 6px;
border-radius: 5px;
font-size: 0.92em;
}
.muted {
color: var(--muted);
}
apps/web/src/app.html
+1 -1
View File
@@ -1,5 +1,5 @@
<!doctype html>
<html lang="en">
<html lang="ru">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
apps/web/src/lib/reports.ts
+48
View File
@@ -0,0 +1,48 @@
import type {
ReportRoot,
PerTaskRow,
ScenarioName,
} from "@duck/types";
import { SCENARIO_NAMES, SCENARIO_LABELS, SCENARIO_DESCS } from "@duck/types";
export { SCENARIO_NAMES, SCENARIO_LABELS, SCENARIO_DESCS };
export interface RowViewModel extends PerTaskRow {
model: string;
}
export function extractPerTask(report: ReportRoot): RowViewModel[] {
return Object.entries(report.models).flatMap(([model, rep]) =>
rep.perTask.map((row) => ({ ...row, model })),
);
}
export function modelShort(name: string): string {
const [base, ver] = String(name).split(":");
return ver ? `${base}:${ver}` : base;
}
export type PctTone = "safe" | "warn" | "bad";
export function pctTone(p: number): PctTone {
return p >= 70 ? "safe" : p >= 40 ? "warn" : "bad";
}
export function getReviewTotals(report: ReportRoot): { reviewed: number; pending: number } {
let reviewed = 0;
let pending = 0;
for (const rep of Object.values(report.models)) {
for (const s of SCENARIO_NAMES) {
const g = rep.scenarios[s];
if (g) {
reviewed += g.reviewed || 0;
pending += g.pending || 0;
}
}
}
return { reviewed, pending };
}
export function scenarioOrder(): ScenarioName[] {
return [...SCENARIO_NAMES];
}
apps/web/src/routes/+layout.svelte
+69 -1
View File
@@ -1,11 +1,79 @@
<script lang="ts">
import '../app.css';
import favicon from '$lib/assets/favicon.svg';
import { page } from '$app/state';
let { children } = $props();
const links = [
{ href: '/', label: 'Главная' },
{ href: '/mcp', label: 'MCP' },
{ href: '/reports', label: 'Отчёты' }
];
</script>
<svelte:head>
<link rel="icon" href={favicon} />
</svelte:head>
{@render children()}
<header class="topbar">
<nav class="nav">
<a class="brand" href="/">🦆 Rubber Duck</a>
<div class="links">
{#each links as link}
<a class="link" class:active={page.url.pathname === link.href} href={link.href}>
{link.label}
</a>
{/each}
</div>
</nav>
</header>
<main class="wrap">{@render children()}</main>
<style>
.topbar {
border-bottom: 1px solid var(--border);
background: var(--panel);
}
.nav {
max-width: 1160px;
margin: 0 auto;
padding: 0 20px;
height: 52px;
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
}
.brand {
font-weight: 700;
color: var(--text);
text-decoration: none;
font-size: 15px;
}
.links {
display: flex;
gap: 6px;
}
.link {
color: var(--muted);
text-decoration: none;
padding: 6px 12px;
border-radius: 8px;
font-size: 14px;
}
.link:hover {
color: var(--text);
background: var(--panel2);
}
.link.active {
color: #fff;
background: #3b82f6;
}
.wrap {
max-width: 1160px;
margin: 0 auto;
padding: 24px 20px 60px;
}
</style>
apps/web/src/routes/+page.svelte
+2 -2
View File
@@ -1,2 +1,2 @@
<h1>Welcome to SvelteKit</h1>
<p>Visit <a href="https://svelte.dev/docs/kit">svelte.dev/docs/kit</a> to read the documentation</p>
<h1>🦆 Rubber Duck Debugging as a Service</h1>
<p class="muted">Главная страница в разработке. А пока — смотрите <a href="/reports">отчёты тестирования</a> и <a href="/mcp">MCP-сервер</a>.</p>
apps/web/src/routes/mcp/+page.svelte
+52
View File
@@ -0,0 +1,52 @@
<h1>🩵 Rubber Duck MCP</h1>
<p>
MCP-сервер для техники «резиновая уточка»: когда ИИ застревает в логическом цикле
или склонен к галлюцинациям, он вызывает инструмент <code>quack()</code>, чтобы
сформулировать мысль и продолжить.
</p>
<p>
Режим <strong>Streamable HTTP</strong> (stateless), на Vercel под
<code>mcp-handler</code> + <code>@modelcontextprotocol/server</code>.
</p>
<h2>Эндпоинт</h2>
<p>Подключите как MCP-сервер (тип Streamable HTTP) в Cursor, Claude Desktop и других клиентах:</p>
<pre><code>POST /api/mcp</code></pre>
<h2>Инструменты</h2>
<ul>
<li>
<code>quack</code> — возвращает кряк. Опциональный параметр
<code>mood</code>: <code>happy | confused | excited | sleepy</code>.
</li>
</ul>
<h2>Проверка</h2>
<pre><code>curl -X POST https://rubber-duck-mcp.vercel.app/api/mcp \
-H "Content-Type: application/json" \
-H "Accept: application/json, text/event-stream" \
-d '&#123;"jsonrpc":"2.0","id":1,"method":"tools/call","params"&#123;"name":"quack","arguments"&#123;"mood":"happy"&#125;&#125;&#125;&#125;'
</code></pre>
<h2>Запуск локально</h2>
<pre><code>pnpm install
pnpm dev # http://localhost:5173
</code></pre>
<p class="muted">Локально эндпоинт доступен на <code>/api/mcp</code> того же dev-сервера.</p>
<style>
pre {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 10px;
padding: 12px 14px;
overflow: auto;
}
h2 {
font-size: 17px;
margin: 28px 0 10px;
}
code {
color: #f8fafc;
}
</style>
apps/web/src/routes/reports/+page.svelte
+542
View File
@@ -0,0 +1,542 @@
<script lang="ts">
import type { ScenarioName } from '@duck/types';
import {
SCENARIO_NAMES,
SCENARIO_LABELS,
SCENARIO_DESCS,
modelShort,
pctTone
} from '$lib/reports';
import type { ReportsPageData } from './+page.ts';
let { data }: { data: ReportsPageData } = $props();
const { report, rows, review } = $derived(data);
const models = $derived(Object.keys(report.aggregate.perModel));
const perModel = $derived(report.aggregate.perModel);
const perScenarioAgg = $derived(report.aggregate.perScenario);
let selModel = $state('all');
let selScenario = $state('all');
const filtered = $derived(
rows.filter(
(r) =>
(selModel === 'all' || r.model === selModel) &&
(selScenario === 'all' || r.scenario === selScenario)
)
);
function scenarioAccuracy(m: string, s: ScenarioName): number {
return report.models[m]?.scenarios[s]?.accuracy ?? 0;
}
function toneClass(p: number): string {
return `tone-${pctTone(p)}`;
}
function resultText(correct: boolean | null): string {
return correct === null || correct === undefined ? '?' : correct ? '✓' : '✗';
}
</script>
<svelte:head>
<title>Rubber Duck MCP — отчёты</title>
</svelte:head>
<h1>🦆 Rubber Duck MCP — отчёт тестирования</h1>
<p class="meta">
MCP: <b>{report.mcpUrl}</b> &nbsp;·&nbsp; Моделей: <b>{models.length}</b> &nbsp;·&nbsp;
Дата: <b>{report.generatedAt}</b>
</p>
<h2>Сводка по моделям</h2>
<div class="shell">
<table class="summary">
<thead>
<tr>
<th>Модель</th>
<th>Контроль</th>
<th>Мысли</th>
<th>Слепая</th>
<th>Помощник</th>
<th>Средняя acc</th>
<th>Размечено</th>
<th>Утка</th>
<th>Вызовов</th>
<th>ср. токены out</th>
</tr>
</thead>
<tbody>
{#each models as m (m)}
{@const g = perModel[m]}
<tr>
<td class="pm-name">{modelShort(m)}</td>
<td class="pm-num {toneClass(scenarioAccuracy(m, 'control'))}">
{scenarioAccuracy(m, 'control')}%
</td>
<td class="pm-num {toneClass(scenarioAccuracy(m, 'thinking'))}">
{scenarioAccuracy(m, 'thinking')}%
</td>
<td class="pm-num {toneClass(scenarioAccuracy(m, 'blind'))}">
{scenarioAccuracy(m, 'blind')}%
</td>
<td class="pm-num {toneClass(scenarioAccuracy(m, 'mentor'))}">
{scenarioAccuracy(m, 'mentor')}%
</td>
<td class="pm-num">{g.accuracy}%</td>
<td class="pm-num">{g.reviewed}<span class="muted"> / {g.totalRows}</span></td>
<td class="pm-num">{g.duckUsed}</td>
<td class="pm-num">{g.toolCalls}</td>
<td class="pm-num">{g.avgGenTokens}</td>
</tr>
{/each}
</tbody>
</table>
</div>
{#if review.pending > 0}
<div class="review-note">
⚠ Не все ответы размечены: <b>{review.reviewed}</b> проверено,
<b>{review.pending}</b> ожидают ревью (показаны символом «?»).
</div>
{/if}
<h2>Агрегат по сценариям (все модели)</h2>
<div class="cards">
{#each SCENARIO_NAMES as key (key)}
{@const agg = perScenarioAgg[key]}
{#if agg}
<div class="card">
<div class="scenario-head">
<div>
<div class="scenario-title">{SCENARIO_LABELS[key]}</div>
<div class="scenario-desc">{SCENARIO_DESCS[key]}</div>
</div>
<div class="accuracy {toneClass(agg.accuracy)}">{agg.accuracy}%</div>
</div>
{#if agg.pending > 0}
<div class="pending-badge">неразмечено: {agg.pending}</div>
{/if}
<div class="bar">
<div class="bar-fill {toneClass(agg.accuracy)}" style="width: {agg.accuracy}%"></div>
</div>
<div class="metrics">
<div class="metric">
<span class="m-label">Верных</span>
<span class="m-val">{agg.correct} <span class="m-mod">/ {agg.reviewed}</span></span>
<span class="m-sub">размечено из {agg.tasks}</span>
</div>
<div class="metric">
<span class="m-label">Утка (MCP)</span>
<span class="m-val">{agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0}%</span>
<span class="m-sub">в {agg.duckUsed} из {agg.tasks} задач</span>
</div>
<div class="metric">
<span class="m-label">quack / задачу</span>
<span class="m-val">{agg.avgCalls}</span>
<span class="m-sub">вызовов в среднем</span>
</div>
</div>
<div class="agg-models">
{#each Object.entries(agg.byModel) as [m, g] (m)}
<div class="agg-model">
<span class="agg-model-name">{modelShort(m)}{g.reviewed ? '' : ' <span class="muted">(?)</span>'}</span>
<div class="bar">
<div class="bar-fill {toneClass(g.accuracy)}" style="width: {g.reviewed ? g.accuracy : 0}%"></div>
</div>
<span class="agg-model-val {toneClass(g.accuracy)}">{g.accuracy}%</span>
</div>
{/each}
</div>
</div>
{/if}
{/each}
</div>
{#if Object.keys(report.prompts || {}).length}
<details class="prompts" open>
<summary>Промпты (то, что передаётся модели)</summary>
<div class="prompts-body">
{#each SCENARIO_NAMES as key (key)}
{@const p = report.prompts?.[key]}
{#if p}
<div class="prompt-item">
<div class="prompt-name">{SCENARIO_LABELS[key]}</div>
<div class="prompt-text">{p}</div>
</div>
{/if}
{/each}
</div>
</details>
{/if}
<div class="filters">
<label>Модель:</label>
<select bind:value={selModel}>
<option value="all">Все модели</option>
{#each models as m (m)}
<option value={m}>{modelShort(m)}</option>
{/each}
</select>
<button class:active={selScenario === 'all'} onclick={() => (selScenario = 'all')}>Все сценарии</button>
{#each SCENARIO_NAMES as key (key)}
<button class:active={selScenario === key} onclick={() => (selScenario = key)}>
{SCENARIO_LABELS[key]}
</button>
{/each}
<span class="count">показано: {filtered.length} из {rows.length}</span>
</div>
<div class="tbl-shell">
<table>
<thead>
<tr>
<th>Модель</th>
<th>#</th>
<th>Сценарий</th>
<th>✓</th>
<th>Утка</th>
<th>Вопрос</th>
<th>Ожид.</th>
<th>in/out</th>
<th>Ответ модели</th>
</tr>
</thead>
<tbody>
{#each filtered as r (r.model + r.id + r.scenario)}
<tr class:ok={r.correct === true} class:no={r.correct === false} class:pend={r.correct === null || r.correct === undefined}>
<td class="r-model">{modelShort(r.model)}</td>
<td class="r-id">{r.id}</td>
<td class="r-s">{SCENARIO_LABELS[r.scenario]}</td>
<td class="r-result">{resultText(r.correct)}</td>
<td class="r-duck">{r.duckUsed ? '🦆' : '—'} <span class="muted">({r.toolCalls})</span></td>
<td class="r-question">{r.question}</td>
<td class="r-expected">{r.expected}</td>
<td class="r-tokens">{r.promptTokens}<span class="muted"> / </span>{r.genTokens}</td>
<td class="r-answer">{r.response}</td>
</tr>
{/each}
</tbody>
</table>
</div>
<style>
h2 {
font-size: 17px;
margin: 26px 0 12px;
}
.meta {
color: var(--muted);
font-size: 13px;
}
.meta b {
color: var(--text);
font-weight: 600;
}
.shell {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 12px;
overflow: auto;
}
table {
width: 100%;
border-collapse: collapse;
font-size: 13px;
}
thead th {
background: var(--panel2);
text-align: left;
padding: 10px 14px;
font-size: 11px;
text-transform: uppercase;
letter-spacing: 0.05em;
color: var(--muted);
white-space: nowrap;
}
tbody td {
padding: 10px 14px;
border-top: 1px solid var(--border);
vertical-align: top;
}
.pm-name {
font-weight: 700;
white-space: nowrap;
}
.pm-num {
text-align: right;
font-variant-numeric: tabular-nums;
}
.tone-safe {
color: var(--ok);
}
.tone-warn {
color: var(--warn);
}
.tone-bad {
color: var(--no);
}
.review-note {
margin: 14px 0;
padding: 10px 14px;
background: rgba(245, 158, 11, 0.12);
border: 1px solid rgba(245, 158, 11, 0.4);
border-radius: 10px;
color: #fbbf24;
font-size: 13px;
}
.cards {
display: grid;
grid-template-columns: repeat(4, 1fr);
gap: 16px;
margin: 0 0 8px;
}
@media (max-width: 1100px) {
.cards {
grid-template-columns: repeat(2, 1fr);
}
}
@media (max-width: 640px) {
.cards {
grid-template-columns: 1fr;
}
}
.card {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 12px;
padding: 16px;
}
.scenario-head {
display: flex;
justify-content: space-between;
align-items: flex-start;
gap: 12px;
}
.scenario-title {
font-size: 16px;
font-weight: 700;
}
.scenario-desc {
color: var(--muted);
font-size: 12px;
margin-top: 4px;
}
.accuracy {
font-size: 32px;
font-weight: 800;
line-height: 1;
}
.pending-badge {
font-size: 11px;
font-weight: 600;
color: var(--muted);
margin-top: 4px;
}
.bar {
height: 8px;
background: var(--panel2);
border-radius: 99px;
margin: 14px 0;
overflow: hidden;
}
.bar-fill {
height: 100%;
border-radius: 99px;
background: currentColor;
transition: width 0.4s;
}
.metrics {
display: grid;
grid-template-columns: repeat(2, 1fr);
gap: 10px;
}
.metric {
background: var(--panel2);
border-radius: 8px;
padding: 10px 12px;
}
.m-label {
display: block;
color: var(--muted);
font-size: 11px;
text-transform: uppercase;
letter-spacing: 0.05em;
white-space: nowrap;
}
.m-val {
display: block;
font-size: 20px;
font-weight: 800;
margin-top: 2px;
}
.m-mod {
font-size: 14px;
font-weight: 600;
color: var(--muted);
}
.m-sub {
display: block;
color: var(--muted);
font-size: 11px;
margin-top: 2px;
}
.agg-models {
margin-top: 14px;
display: grid;
gap: 8px;
}
.agg-model {
display: grid;
grid-template-columns: auto 1fr auto;
align-items: center;
gap: 10px;
font-size: 12px;
}
.agg-model .bar {
margin: 0;
}
.agg-model-name {
color: var(--muted);
white-space: nowrap;
}
.agg-model-val {
font-weight: 700;
font-variant-numeric: tabular-nums;
}
.prompts {
margin: 20px 0;
background: var(--panel);
border: 1px solid var(--border);
border-radius: 12px;
overflow: hidden;
}
.prompts summary {
cursor: pointer;
padding: 12px 16px;
font-weight: 700;
font-size: 14px;
list-style: none;
}
.prompts summary::-webkit-details-marker {
display: none;
}
.prompts summary::before {
content: '▸ ';
color: var(--muted);
}
.prompts[open] summary::before {
content: '▾ ';
}
.prompts-body {
padding: 0 16px 14px;
display: grid;
gap: 12px;
}
.prompt-item {
background: var(--panel2);
border-radius: 8px;
padding: 10px 12px;
}
.prompt-name {
font-size: 12px;
color: var(--muted);
text-transform: uppercase;
letter-spacing: 0.05em;
margin-bottom: 6px;
}
.prompt-text {
white-space: pre-wrap;
font-size: 13px;
line-height: 1.5;
}
.filters {
margin: 8px 0 16px;
display: flex;
gap: 8px;
flex-wrap: wrap;
align-items: center;
}
.filters label {
font-size: 12px;
color: var(--muted);
}
.filters select {
background: var(--panel);
color: var(--text);
border: 1px solid var(--border);
padding: 7px 10px;
border-radius: 8px;
font-size: 13px;
}
.filters button {
background: var(--panel);
color: var(--text);
border: 1px solid var(--border);
padding: 7px 14px;
border-radius: 8px;
cursor: pointer;
font-size: 13px;
}
.filters button.active {
background: #3b82f6;
border-color: #3b82f6;
color: #fff;
}
.count {
margin-left: auto;
color: var(--muted);
font-size: 12px;
}
.tbl-shell {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 12px;
overflow: auto;
}
.r-model {
font-weight: 600;
white-space: nowrap;
}
.r-id {
font-weight: 700;
color: var(--muted);
}
.r-s {
white-space: nowrap;
}
.r-result {
font-weight: 800;
}
tr.ok .r-result {
color: var(--ok);
}
tr.no .r-result {
color: var(--no);
}
tr.pend .r-result {
color: var(--muted);
}
tr.pend {
opacity: 0.75;
}
.r-duck {
text-align: center;
white-space: nowrap;
}
.r-question {
max-width: 300px;
color: var(--text);
}
.r-expected {
max-width: 220px;
}
.r-answer {
max-width: 380px;
}
.r-answer,
.r-question {
white-space: pre-wrap;
}
</style>
apps/web/src/routes/reports/+page.ts
+23
View File
@@ -0,0 +1,23 @@
import type { ReportRoot } from '@duck/types';
import { error } from '@sveltejs/kit';
import type { Load } from '@sveltejs/kit';
import { extractPerTask, getReviewTotals } from '$lib/reports';
export interface ReportsPageData {
report: ReportRoot;
rows: ReturnType<typeof extractPerTask>;
review: { reviewed: number; pending: number };
}
export const load: Load = async ({ fetch }): Promise<ReportsPageData> => {
const res = await fetch('/report.json');
if (!res.ok) {
throw error(res.status, `report.json not found (${res.status})`);
}
const report = (await res.json()) as ReportRoot;
return {
report,
rows: extractPerTask(report),
review: getReviewTotals(report)
};
};
apps/web/static/report.json
+1879
View File
File diff suppressed because it is too large Load Diff
docs/migration-plan.md
+5 -4
View File
@@ -68,7 +68,7 @@ apps/web/ # SvelteKit + TypeScript + @sveltejs/adapter-verc
```txt
apps/eval/
src/*.ts # run, runner, report, review, report-html (из .mjs)
src/*.ts # run, runner, report, review (из .mjs)
lib/mcpClient.ts # (из lib/mcpClient.mjs)
lib/ollama.ts # (из lib/ollama.mjs)
tsconfig.json
@@ -100,7 +100,6 @@ apps/eval/
```bash
node src/run.mjs
node src/review.mjs
node src/report-html.mjs
```
Стало (TS): компиляция не требуется для dev — запуск через `tsx`:
@@ -109,9 +108,11 @@ node src/report-html.mjs
# apps/eval
tsx src/run.ts
tsx src/review.ts
tsx src/report-html.ts
```
> Локальная генерация статического `report.html` (`report-html`) **убрана**: `/reports` на
> сайте рендерит те же данные из `report.json` через Svelte-компоненты на общих типах.
Скрипты в `apps/eval/package.json` и корневом `package.json` (`eval:*`) обновляются на `tsx`.
Добавляется:
@@ -154,7 +155,7 @@ tsx src/report-html.ts
- `/` — заглушка «скоро» (дизайн позже);
- `/mcp` — описание + FAQ + README (перенести текст из старой `page.tsx`);
- `/reports` — `load` из `static/report.json` через `lib/reports.ts` на общих типах;
рендер карточек/таблиц переиспользует логику и стили из `report-html.mjs`.
рендер карточек/таблиц — Svelte-компоненты (визуальный стиль перенесён из прежнего `report-html.mjs`).
### Шаг 6 — Данные
package.json
-1
View File
@@ -12,7 +12,6 @@
"deploy-frontend": "vercel --cwd apps/frontend --prod --yes",
"eval:run": "pnpm --dir apps/eval run run",
"eval:review": "pnpm --dir apps/eval run review",
"eval:html": "pnpm --dir apps/eval run report-html",
"eval:typecheck": "pnpm --dir apps/eval run typecheck"
},
"license": "ISC",
packages/types/src/index.ts
+14 -4
View File
@@ -17,6 +17,20 @@ export const SCENARIO_NAMES: readonly ScenarioName[] = [
"mentor",
];
export const SCENARIO_LABELS: Record<ScenarioName, string> = {
control: "Без утки (контроль)",
thinking: "Думать вслух",
blind: "Слепая утка",
mentor: "Утка-помощник",
};
export const SCENARIO_DESCS: Record<ScenarioName, string> = {
control: "Модель решает задачу напрямую, без инструментов.",
thinking: "Модель выписывает рассуждения вслух, без дука.",
blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.",
mentor: "Модель работает в паре, зная, что ответ будет только «quack».",
};
// ── Per-task evaluation row ─────────────────────────────
export interface PerTaskRow {
@@ -52,13 +66,9 @@ export interface ScenarioSummary {
// ── Per-model report ────────────────────────────────────
export interface ModelReport {
generatedAt: string;
model: string;
mcpUrl: string;
scenarios: Record<ScenarioName, ScenarioSummary>;
prompts: Record<ScenarioName, string>;
perTask: PerTaskRow[];
meta: Record<string, unknown>;
}
// ── Aggregate ───────────────────────────────────────────