diff --git a/apps/eval/lib/ollama.mjs b/apps/eval/lib/ollama.mjs index ddd393e..2d6bbed 100644 --- a/apps/eval/lib/ollama.mjs +++ b/apps/eval/lib/ollama.mjs @@ -1,20 +1,21 @@ const DEFAULT_URL = "http://localhost:11434"; export class Ollama { - constructor({ url = DEFAULT_URL } = {}) { + constructor({ url = DEFAULT_URL, timeoutMs = 600000 } = {}) { this.url = url.replace(/\/$/, ""); + this.timeoutMs = timeoutMs; } - async ping() { + async ping({ timeoutMs = 30000 } = {}) { if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, model: process.env.OLLAMA_MODEL }; - const res = await fetch(`${this.url}/api/tags`); + const res = await fetch(`${this.url}/api/tags`, { signal: AbortSignal.timeout(timeoutMs) }); if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`); const data = await res.json(); const models = (data.models ?? []).map((m) => m.name); return { ok: true, models }; } - async chat({ model, messages, tools, temperature = 0, numCtx = 8192 }) { + async chat({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs }) { const body = { model, messages, @@ -28,6 +29,7 @@ export class Ollama { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs ?? this.timeoutMs), }); if (!res.ok) { const t = await res.text(); diff --git a/apps/eval/src/report-html.mjs b/apps/eval/src/report-html.mjs index d61c528..c525622 100644 --- a/apps/eval/src/report-html.mjs +++ b/apps/eval/src/report-html.mjs @@ -1,6 +1,8 @@ import { readFile, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; +const SCENARIOS = ["control", "blind", "mentor"]; + const SCENARIO_LABELS = { control: "Без утки (контроль)", blind: "Слепая утка", @@ -13,8 +15,6 @@ const SCENARIO_DESCS = { mentor: "Модель обязана выписать мысли и возможные ошибки, затем проверить себя уткой.", }; -const MOOD_COLORS = { happy: "#f59e0b", confused: "#8b5cf6", excited: "#10b981", sleepy: "#64748b" }; - function esc(s) { return String(s ?? "") .replace(/&/g, "&") @@ -26,10 +26,72 @@ function nl2br(s) { return esc(s).replace(/\n/g, "
"); } -function scenarioCard(key, g) { - const pct = g.accuracy; - const color = pct >= 70 ? "#10b981" : pct >= 40 ? "#f59e0b" : "#ef4444"; - const duckPct = g.tasks ? Math.round((g.duckUsed / g.tasks) * 100) : 0; +function pctColor(p) { + return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444"; +} + +function modelShort(name) { + const base = String(name).split(/:/)[0]; + const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : ""; + return esc(base + ver); +} + +function summaryTable(report) { + const perModel = report.aggregate.perModel; + const models = Object.keys(perModel); + const rows = models + .map((m) => { + const g = perModel[m]; + const scen = report.models[m].scenarios; + const acc = (s) => scen[s].accuracy; + return ` + + ${modelShort(m)} + ${acc("control")}% + ${acc("blind")}% + ${acc("mentor")}% + ${g.accuracy}% + ${g.duckUsed} + ${g.toolCalls} + ${g.avgGenTokens} + `; + }) + .join(""); + return ` +
+ + + + + + + + + + + + + + ${rows} +
МодельКонтрольСлепаяПомощникСредняя accУтокВызововср. токены out
+
`; +} + +function scenarioCard(key, agg) { + const pct = agg.accuracy; + const color = pctColor(pct); + const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0; + const modelBars = Object.entries(agg.byModel) + .map(([m, g]) => { + const c = pctColor(g.accuracy); + return ` +
+ ${modelShort(m)} +
+ ${g.accuracy}% +
`; + }) + .join(""); return `
@@ -41,13 +103,11 @@ function scenarioCard(key, g) {
-
Верных${g.correct} / ${g.tasks}
-
Утка (MCP)${duckPct}%в ${g.duckUsed} из ${g.tasks} задач
-
quack / задачу${g.avgCalls}вызовов в среднем
-
Промпт (in)${g.avgPromptTokens}токенов
-
Ответ (out)${g.avgGenTokens}токенов
-
Рассужд.${g.avgDuckTokens}токенов
+
Верных${agg.correct} / ${agg.tasks}
+
Утка (MCP)${duckPct}%в ${agg.duckUsed} из ${agg.tasks} задач
+
quack / задачу${agg.avgCalls}вызовов в среднем
+
${modelBars}
`; } @@ -55,7 +115,8 @@ function rowHtml(r) { const cls = r.correct ? "ok" : "no"; const duck = r.duckUsed ? "🦆" : "—"; return ` - + + ${modelShort(r.model)} ${esc(r.id)} ${esc(SCENARIO_LABELS[r.scenario])} ${r.correct ? "✓" : "✗"} @@ -69,8 +130,8 @@ function rowHtml(r) { function promptBlock(report) { const p = report.prompts || {}; - const items = ["control", "blind", "mentor"] - .filter((k) => p[k] && report.scenarios[k]) + const items = SCENARIOS + .filter((k) => p[k]) .map( (k) => `
@@ -88,13 +149,18 @@ function promptBlock(report) { } export function buildHtml(report) { - const s = report.scenarios; - const cards = ["control", "blind", "mentor"] - .filter((k) => s[k]) - .map((k) => scenarioCard(k, s[k])) + const perTask = Object.entries(report.models).flatMap(([model, rep]) => + rep.perTask.map((r) => ({ ...r, model })) + ); + const rows = perTask.map(rowHtml).join(""); + const models = Object.keys(report.aggregate.perModel); + const modelOptions = models + .map((m) => ``) + .join(""); + const aggCards = SCENARIOS + .filter((k) => report.aggregate.perScenario[k]) + .map((k) => scenarioCard(k, report.aggregate.perScenario[k])) .join(""); - - const rows = report.perTask.map(rowHtml).join(""); return ` @@ -110,12 +176,20 @@ export function buildHtml(report) { * { box-sizing:border-box; } body { margin:0; font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif; background:var(--bg); color:var(--text); line-height:1.5; } - .wrap { max-width:1200px; margin:0 auto; padding:24px 20px 60px; } + .wrap { max-width:1280px; margin:0 auto; padding:24px 20px 60px; } header h1 { margin:0 0 4px; font-size:22px; } .meta { color:var(--muted); font-size:13px; } .meta b { color:var(--text); font-weight:600; } - .cards { display:grid; grid-template-columns:repeat(3,1fr); gap:16px; margin:24px 0; } - @media (max-width:900px){ .cards { grid-template-columns:1fr; } } + h2 { font-size:17px; margin:26px 0 12px; } + .summary-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; } + table.summary { width:100%; border-collapse:collapse; font-size:13px; } + table.summary th { background:var(--panel2); text-align:left; padding:10px 14px; + font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; } + table.summary td { padding:10px 14px; border-top:1px solid var(--border); } + .pm-name { font-weight:700; white-space:nowrap; } + .pm-num { text-align:right; font-variant-numeric:tabular-nums; } + .cards { display:grid; grid-template-columns:repeat(3,1fr); gap:16px; margin:0 0 8px; } + @media (max-width:1000px){ .cards { grid-template-columns:1fr; } } .card { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; } .scenario-head { display:flex; justify-content:space-between; align-items:flex-start; gap:12px; } .scenario-title { font-size:16px; font-weight:700; } @@ -129,11 +203,12 @@ export function buildHtml(report) { .m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; } .m-mod { font-size:14px; font-weight:600; color:var(--muted); } .m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; } - .filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; } - .filters button { background:var(--panel); color:var(--text); border:1px solid var(--border); - padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; } - .filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; } - .prompts { margin:0 0 20px; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } + .agg-models { margin-top:14px; display:grid; gap:8px; } + .agg-model { display:grid; grid-template-columns:auto 1fr auto; align-items:center; gap:10px; font-size:12px; } + .agg-model .bar { margin:0; } + .agg-model-name { color:var(--muted); white-space:nowrap; } + .agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; } + .prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } .prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; } .prompts summary::-webkit-details-marker { display:none; } .prompts summary::before { content:"▸ "; color:var(--muted); } @@ -142,21 +217,27 @@ export function buildHtml(report) { .prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; } .prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; } .prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; } - .tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } + .filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; } + .filters button { background:var(--panel); color:var(--text); border:1px solid var(--border); + padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; } + .filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; } + .filters select { background:var(--panel); color:var(--text); border:1px solid var(--border); + padding:7px 10px; border-radius:8px; font-size:13px; } + .tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; } table { width:100%; border-collapse:collapse; font-size:13px; } thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px; - font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); } + font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; } tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; } .scenario-row.ok .r-result { color:var(--ok); font-weight:800; } .scenario-row.no .r-result { color:var(--no); font-weight:800; } + .r-model { font-weight:600; white-space:nowrap; } .r-id { font-weight:700; color:var(--muted); } .r-s { white-space:nowrap; } .r-duck { text-align:center; white-space:nowrap; } - .r-question { max-width:340px; color:var(--text); } - .r-answer { max-width:420px; } + .r-question { max-width:300px; color:var(--text); } + .r-answer { max-width:380px; } .muted { color:var(--muted); } .hidden { display:none !important; } - .toggle-row { background:transparent; border:none; color:#3b82f6; cursor:pointer; font-size:12px; padding:0; } footer { margin-top:20px; color:var(--muted); font-size:12px; } @@ -165,19 +246,27 @@ export function buildHtml(report) {

🦆 Rubber Duck MCP — отчёт тестирования

- Модель: ${esc(report.model)}  ·  MCP: ${esc(report.mcpUrl)}  ·  - Задач: ${s.control?.tasks ?? report.perTask.length}  ·  + Моделей: ${models.length}  ·  Дата: ${esc(report.generatedAt)}
-
${cards}
+

Сводка по моделям

+ ${summaryTable(report)} + +

Агрегат по сценариям (все модели)

+
${aggCards}
${promptBlock(report)}
- + + + @@ -188,7 +277,7 @@ export function buildHtml(report) { - + @@ -196,17 +285,22 @@ export function buildHtml(report) {
#Сценарий✓УткаМодель#Сценарий✓Утка ВопросОжид.in/outОтвет модели
- +
diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.mjs index 176cb98..d595f63 100644 --- a/apps/eval/src/report.mjs +++ b/apps/eval/src/report.mjs @@ -11,8 +11,7 @@ function avg(arr) { return arr.reduce((a, b) => a + b, 0) / arr.length; } -export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { - const scenarios = ["control", "blind", "mentor"]; +export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "blind", "mentor"]; const byScenario = {}; for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; @@ -67,3 +66,65 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = { meta, }; } + +const SCENARIOS = ["control", "blind", "mentor"]; + +export function buildAggregate(modelsReport) { + const perScenario = {}; + for (const s of SCENARIOS) { + let total = 0, correct = 0, duckUsed = 0, toolCalls = 0; + const gen = []; + const byModel = {}; + for (const [name, rep] of Object.entries(modelsReport)) { + const g = rep.scenarios[s]; + if (!g) continue; + total += g.tasks; + correct += g.correct; + duckUsed += g.duckUsed; + toolCalls += g.toolCalls; + byModel[name] = { + tasks: g.tasks, + correct: g.correct, + accuracy: g.accuracy, + duckUsed: g.duckUsed, + }; + } + perScenario[s] = { + tasks: total, + correct, + accuracy: total ? round((correct / total) * 100, 1) : 0, + duckUsed, + toolCalls, + avgCalls: total ? round(toolCalls / total, 2) : 0, + byModel, + }; + } + + const perModel = {}; + for (const [name, rep] of Object.entries(modelsReport)) { + let duckUsed = 0, toolCalls = 0; + const gen = []; + for (const s of SCENARIOS) { + const g = rep.scenarios[s]; + duckUsed += g.duckUsed; + toolCalls += g.toolCalls; + for (const row of rep.perTask.filter((r) => r.scenario === s)) { + gen.push(row.genTokens ?? 0); + } + } + const taskCount = rep.scenarios.control?.tasks ?? 0; + const correct = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].correct || 0), 0); + const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0); + perModel[name] = { + tasks: taskCount, + totalRows, + correct, + accuracy: totalRows ? round((correct / totalRows) * 100, 1) : 0, + duckUsed, + toolCalls, + avgGenTokens: round(avg(gen)), + }; + } + + return { perScenario, perModel }; +} diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.mjs index bd11ae6..0724113 100644 --- a/apps/eval/src/run.mjs +++ b/apps/eval/src/run.mjs @@ -1,17 +1,44 @@ -import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises"; import path from "node:path"; import { Ollama } from "../lib/ollama.mjs"; import { McpClient } from "../lib/mcpClient.mjs"; import { runScenario } from "./runner.mjs"; import { isCorrect, PROMPTS } from "./runner.mjs"; -import { buildReport } from "./report.mjs"; +import { buildReport, buildAggregate } from "./report.mjs"; + +let logTarget = null; +async function log(msg) { + const line = `[${new Date().toISOString()}] ${msg}`; + console.log(line); + if (logTarget) { + try { + await appendFile(logTarget, line + "\n", "utf8"); + } catch { + /* keep going */ + } + } +} + +const SCENARIOS = ["control", "blind", "mentor"]; + +const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:4b", "gemma3:4b", "granite4.1:3b"]; +const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp"; function parseArgs(argv) { - const args = { scenarios: ["control", "blind", "mentor"], limit: Infinity, model: process.env.OLLAMA_MODEL || null }; + const args = { + models: [], + scenarios: SCENARIOS, + limit: Infinity, + model: process.env.OLLAMA_MODEL || null, + mcpUrl: null, + tasksFile: null, + out: null, + }; for (let i = 0; i < argv.length; i += 1) { const a = argv[i]; const next = () => argv[i + 1]; - if (a === "--model") args.model = next(), i += 1; + if (a === "--models") args.models = next().split(",").map((s) => s.trim()).filter(Boolean), i += 1; + else if (a === "--model") args.model = next(), i += 1; else if (a === "--mcp-url") args.mcpUrl = next(), i += 1; else if (a === "--scenarios") args.scenarios = next().split(","), i += 1; else if (a === "--limit") args.limit = Number(next()), i += 1; @@ -21,44 +48,17 @@ function parseArgs(argv) { return args; } -const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp"; - -async function main() { - const args = parseArgs(process.argv.slice(2)); - if (!args.model) { - console.error("Usage: node src/run.mjs --model [--mcp-url URL] [--scenarios c,b,m] [--limit N] [--tasks file] [--out file]"); - process.exit(1); - } - const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL; - const tasksFile = args.tasksFile || path.join("data", "tasks.json"); - - const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8")); - const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit); - - console.log(`\nModel: ${args.model}`); - console.log(`MCP endpoint: ${mcpUrl}`); - console.log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`); - - const ollama = new Ollama(); - const ping = await ollama.ping(); - console.log(`Ollama OK (${ping.models.length} models)`); - const modelPresent = ping.models.some((m) => m.startsWith(args.model)); - if (!modelPresent) { - console.warn(` (warning) model "${args.model}" not found among: ${ping.models.join(", ")}`); - } - - const mcp = new McpClient(mcpUrl); - const mcpTools = await mcp.listTools(); - console.log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`); - +async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { const rows = []; for (const task of tasks) { - for (const scenario of args.scenarios) { + for (const scenario of scenarios) { let r; + const t0 = Date.now(); try { - r = await runScenario({ scenario, ollama, model: args.model, task: task.question, mcp }); + r = await runScenario({ scenario, ollama, model, task: task.question, mcp }); } catch (err) { - console.error(` [${task.id}/${scenario}] FAILED: ${err.message}`); + const ms = Date.now() - t0; + await log(` [${task.id}/${scenario}] FAILED (${ms}ms): ${err.message}`); rows.push({ id: task.id, scenario, @@ -90,21 +90,83 @@ async function main() { }); const mark = correct ? "OK " : "XX "; const duck = r.duckUsed ? "duck" : "n/a "; - console.log(`${scenario.padEnd(8)} ${mark} ${task.id} duck=${duck} prompt=${r.promptTokens} gen=${r.genTokens} | ${r.response.slice(0, 70).replace(/\n/g, " ")}`); + const ms = Date.now() - t0; + await log(` [${task.id}/${scenario}] ${mark} duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`); } } + return rows; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + let models = args.models; + if (args.model && !models.length) models = [args.model]; + if (!models.length) models = DEFAULT_MODELS; + + const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL; + const tasksFile = args.tasksFile || path.join("data", "tasks.json"); + + const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8")); + const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit); - const report = buildReport({ model: args.model, mcpUrl, tasks, rows, prompts: PROMPTS }); const out = args.out || path.join("out", "report.json"); + const logFile = args.logTarget || path.join(path.dirname(out), "run.log"); + logTarget = logFile; + await mkdir(path.dirname(logFile), { recursive: true }); + await appendFile(logFile, "", "utf8").catch(() => {}); + + await log(`MCP endpoint: ${mcpUrl}`); + await log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`); + await log(`Models: ${models.join(", ")}`); + + const ollama = new Ollama(); + const ping = await ollama.ping(); + await log(`Ollama OK (${ping.models.length} models): ${ping.models.join(", ")}`); + for (const m of models) { + const present = ping.models.some((x) => x.startsWith(m)); + await log(` ${present ? "OK " : "MISSING "} ${m}`); + } + + const mcp = new McpClient(mcpUrl); + const mcpTools = await mcp.listTools(); + await log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`); + + const modelsReport = {}; + for (const model of models) { + const tStart = Date.now(); + await log(`==== Running model: ${model} ====`); + const rows = await runOneModel({ model, ollama, mcp, tasks, scenarios: args.scenarios }); + const rep = buildReport({ model, mcpUrl, tasks, rows }); + modelsReport[model] = { + model, + scenarios: rep.scenarios, + perTask: rep.perTask, + }; + await log(`==== Done ${model} in ${Math.round((Date.now() - tStart) / 1000)}s ====`); + } + + const aggregate = buildAggregate(modelsReport); + + const report = { + generatedAt: new Date().toISOString(), + mcpUrl, + prompts: PROMPTS, + models: modelsReport, + aggregate, + }; + await mkdir(path.dirname(out), { recursive: true }); await writeFile(out, JSON.stringify(report, null, 2), "utf8"); - console.log(`\nReport written: ${out}`); - console.log("\n--- Summary ---"); - for (const [s, g] of Object.entries(report.scenarios)) { - console.log( - `${s.padEnd(8)} accuracy=${String(g.accuracy).padStart(5)}% (${g.correct}/${g.tasks}) ` + - `duckUsed=${g.duckUsed} toolCalls=${g.toolCalls} ` + - `avgPrompt=${g.avgPromptTokens} avgGen=${g.avgGenTokens} avgDuck=${g.avgDuckTokens}` + await log(`Report written: ${out}`); + + await log("==== Aggregate summary ===="); + for (const [modelName, g] of Object.entries(aggregate.perModel)) { + const scen = modelsReport[modelName].scenarios; + await log( + `${modelName} | ` + + SCENARIOS.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}%`).join(" ") + + ` duck=${g.duckUsed} calls=${g.toolCalls} avgGen=${g.avgGenTokens}` ); } }