From 137f5e0488a361eba04e3d8ff2ed743b7d9d16b8 Mon Sep 17 00:00:00 2001 From: Ku6epXBOCTuK Date: Thu, 3 Sep 2026 19:42:53 +0500 Subject: [PATCH] feat: update run tests and reports flow --- apps/eval/src/report-html.mjs | 53 ++++++++++++++++++++++++++++------- apps/eval/src/report.mjs | 4 ++- apps/eval/src/run.mjs | 4 +-- apps/eval/src/runner.mjs | 10 +++++-- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/apps/eval/src/report-html.mjs b/apps/eval/src/report-html.mjs index b047dd6..d61c528 100644 --- a/apps/eval/src/report-html.mjs +++ b/apps/eval/src/report-html.mjs @@ -41,12 +41,12 @@ function scenarioCard(key, g) {
-
Верно${g.correct} / ${g.tasks}
-
Утка${duckPct}% (${g.duckUsed}/${g.tasks})
-
Вызовов${g.toolCalls}
-
Input${g.avgPromptTokens}
-
Output${g.avgGenTokens}
-
Рассужд.${g.avgDuckTokens}
+
Верных${g.correct} / ${g.tasks}
+
Утка (MCP)${duckPct}%в ${g.duckUsed} из ${g.tasks} задач
+
quack / задачу${g.avgCalls}вызовов в среднем
+
Промпт (in)${g.avgPromptTokens}токенов
+
Ответ (out)${g.avgGenTokens}токенов
+
Рассужд.${g.avgDuckTokens}токенов
`; } @@ -67,6 +67,26 @@ function rowHtml(r) { `; } +function promptBlock(report) { + const p = report.prompts || {}; + const items = ["control", "blind", "mentor"] + .filter((k) => p[k] && report.scenarios[k]) + .map( + (k) => ` +
+
${esc(SCENARIO_LABELS[k])}
+
${esc(p[k])}
+
` + ) + .join(""); + if (!items) return ""; + return ` +
+ Промпты (то, что передаётся модели) +
${items}
+
`; +} + export function buildHtml(report) { const s = report.scenarios; const cards = ["control", "blind", "mentor"] @@ -103,14 +123,25 @@ export function buildHtml(report) { .accuracy { font-size:32px; font-weight:800; line-height:1; } .bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; } .bar-fill { height:100%; border-radius:99px; transition:width .4s; } - .metrics { display:grid; grid-template-columns:repeat(3,1fr); gap:10px; } - .metric { background:var(--panel2); border-radius:8px; padding:8px; text-align:center; } - .m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.04em; } - .m-val { font-size:16px; font-weight:700; } + .metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; } + .metric { background:var(--panel2); border-radius:8px; padding:10px 12px; } + .m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.05em; white-space:nowrap; } + .m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; } + .m-mod { font-size:14px; font-weight:600; color:var(--muted); } + .m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; } .filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; } .filters button { background:var(--panel); color:var(--text); border:1px solid var(--border); padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; } .filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; } + .prompts { margin:0 0 20px; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } + .prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; } + .prompts summary::-webkit-details-marker { display:none; } + .prompts summary::before { content:"▸ "; color:var(--muted); } + .prompts[open] summary::before { content:"▾ "; } + .prompts-body { padding:0 16px 14px; display:grid; gap:12px; } + .prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; } + .prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; } + .prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; } .tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } table { width:100%; border-collapse:collapse; font-size:13px; } thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px; @@ -143,6 +174,8 @@ export function buildHtml(report) {
${cards}
+ ${promptBlock(report)} +
diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.mjs index a4021b6..176cb98 100644 --- a/apps/eval/src/report.mjs +++ b/apps/eval/src/report.mjs @@ -11,7 +11,7 @@ function avg(arr) { return arr.reduce((a, b) => a + b, 0) / arr.length; } -export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) { +export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "blind", "mentor"]; const byScenario = {}; for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; @@ -38,6 +38,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) { accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0, duckUsed: g.duckUsed, toolCalls: g.toolCalls, + avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0, avgPromptTokens: round(avg(g.promptTokens)), avgGenTokens: round(avg(g.genTokens)), avgDuckTokens: round(avg(g.duckTokens)), @@ -49,6 +50,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) { model, mcpUrl, scenarios: summaries, + prompts, perTask: rows.map((r) => ({ id: r.id, scenario: r.scenario, diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.mjs index bea1714..bd11ae6 100644 --- a/apps/eval/src/run.mjs +++ b/apps/eval/src/run.mjs @@ -3,7 +3,7 @@ import path from "node:path"; import { Ollama } from "../lib/ollama.mjs"; import { McpClient } from "../lib/mcpClient.mjs"; import { runScenario } from "./runner.mjs"; -import { isCorrect } from "./runner.mjs"; +import { isCorrect, PROMPTS } from "./runner.mjs"; import { buildReport } from "./report.mjs"; function parseArgs(argv) { @@ -94,7 +94,7 @@ async function main() { } } - const report = buildReport({ model: args.model, mcpUrl, tasks, rows }); + const report = buildReport({ model: args.model, mcpUrl, tasks, rows, prompts: PROMPTS }); const out = args.out || path.join("out", "report.json"); await mkdir(path.dirname(out), { recursive: true }); await writeFile(out, JSON.stringify(report, null, 2), "utf8"); diff --git a/apps/eval/src/runner.mjs b/apps/eval/src/runner.mjs index 01e6558..143a87f 100644 --- a/apps/eval/src/runner.mjs +++ b/apps/eval/src/runner.mjs @@ -27,15 +27,19 @@ const CONTROL_SYSTEM = const BLIND_SYSTEM = "You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " + "First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " + - "Then use the 'quack' tool so the colleague can give you a brief reaction, and wait for that reaction. " + - "Then, taking the colleague's reaction into account, complete your solution and give the final answer to the user. " + - "Note: your colleague is a rubber duck; it replies only with a short quack. Even so, saying your reasoning aloud helps you spot mistakes."; + "Wait for the colleague's reply. Then, taking that reply into account, complete your solution and give the final answer to the user."; const MENTOR_SYSTEM = "You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " + "your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " + "Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user."; +export const PROMPTS = { + control: CONTROL_SYSTEM, + blind: BLIND_SYSTEM, + mentor: MENTOR_SYSTEM, +}; + const MAX_DUCK_TURNS = 6; function normalize(s) {