feat: update run tests and reports flow

This commit is contained in:
2026-09-03 19:42:53 +05:00
parent b6b89d128a
commit 137f5e0488
4 changed files with 55 additions and 16 deletions
+43 -10
View File
@@ -41,12 +41,12 @@ function scenarioCard(key, g) {
</div> </div>
<div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div> <div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
<div class="metrics"> <div class="metrics">
<div class="metric"><span class="m-label">Верно</span><span class="m-val">${g.correct} / ${g.tasks}</span></div> <div class="metric"><span class="m-label">Верных</span><span class="m-val">${g.correct} <span class="m-mod">/ ${g.tasks}</span></span></div>
<div class="metric"><span class="m-label">Утка</span><span class="m-val">${duckPct}% (${g.duckUsed}/${g.tasks})</span></div> <div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${g.duckUsed} из ${g.tasks} задач</span></div>
<div class="metric"><span class="m-label">Вызовов</span><span class="m-val">${g.toolCalls}</span></div> <div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${g.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
<div class="metric"><span class="m-label">Input</span><span class="m-val">${g.avgPromptTokens}</span></div> <div class="metric"><span class="m-label">Промпт (in)</span><span class="m-val">${g.avgPromptTokens}</span><span class="m-sub">токенов</span></div>
<div class="metric"><span class="m-label">Output</span><span class="m-val">${g.avgGenTokens}</span></div> <div class="metric"><span class="m-label">Ответ (out)</span><span class="m-val">${g.avgGenTokens}</span><span class="m-sub">токенов</span></div>
<div class="metric"><span class="m-label">Рассужд.</span><span class="m-val">${g.avgDuckTokens}</span></div> <div class="metric"><span class="m-label">Рассужд.</span><span class="m-val">${g.avgDuckTokens}</span><span class="m-sub">токенов</span></div>
</div> </div>
</div>`; </div>`;
} }
@@ -67,6 +67,26 @@ function rowHtml(r) {
</tr>`; </tr>`;
} }
function promptBlock(report) {
const p = report.prompts || {};
const items = ["control", "blind", "mentor"]
.filter((k) => p[k] && report.scenarios[k])
.map(
(k) => `
<div class="prompt-item">
<div class="prompt-name">${esc(SCENARIO_LABELS[k])}</div>
<div class="prompt-text">${esc(p[k])}</div>
</div>`
)
.join("");
if (!items) return "";
return `
<details class="prompts" open>
<summary>Промпты (то, что передаётся модели)</summary>
<div class="prompts-body">${items}</div>
</details>`;
}
export function buildHtml(report) { export function buildHtml(report) {
const s = report.scenarios; const s = report.scenarios;
const cards = ["control", "blind", "mentor"] const cards = ["control", "blind", "mentor"]
@@ -103,14 +123,25 @@ export function buildHtml(report) {
.accuracy { font-size:32px; font-weight:800; line-height:1; } .accuracy { font-size:32px; font-weight:800; line-height:1; }
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; } .bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
.bar-fill { height:100%; border-radius:99px; transition:width .4s; } .bar-fill { height:100%; border-radius:99px; transition:width .4s; }
.metrics { display:grid; grid-template-columns:repeat(3,1fr); gap:10px; } .metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; }
.metric { background:var(--panel2); border-radius:8px; padding:8px; text-align:center; } .metric { background:var(--panel2); border-radius:8px; padding:10px 12px; }
.m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.04em; } .m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.05em; white-space:nowrap; }
.m-val { font-size:16px; font-weight:700; } .m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; }
.m-mod { font-size:14px; font-weight:600; color:var(--muted); }
.m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
.filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; } .filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
.filters button { background:var(--panel); color:var(--text); border:1px solid var(--border); .filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; } padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
.filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; } .filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
.prompts { margin:0 0 20px; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
.prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
.prompts summary::-webkit-details-marker { display:none; }
.prompts summary::before { content:"▸ "; color:var(--muted); }
.prompts[open] summary::before { content:"▾ "; }
.prompts-body { padding:0 16px 14px; display:grid; gap:12px; }
.prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; }
.prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; }
.prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; }
.tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; } .tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
table { width:100%; border-collapse:collapse; font-size:13px; } table { width:100%; border-collapse:collapse; font-size:13px; }
thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px; thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px;
@@ -143,6 +174,8 @@ export function buildHtml(report) {
<div class="cards">${cards}</div> <div class="cards">${cards}</div>
${promptBlock(report)}
<div class="filters" id="filters"> <div class="filters" id="filters">
<button data-filter="all" class="active">Все</button> <button data-filter="all" class="active">Все</button>
<button data-filter="control">Без утки</button> <button data-filter="control">Без утки</button>
+3 -1
View File
@@ -11,7 +11,7 @@ function avg(arr) {
return arr.reduce((a, b) => a + b, 0) / arr.length; return arr.reduce((a, b) => a + b, 0) / arr.length;
} }
export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) { export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {
const scenarios = ["control", "blind", "mentor"]; const scenarios = ["control", "blind", "mentor"];
const byScenario = {}; const byScenario = {};
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
@@ -38,6 +38,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0, accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0,
duckUsed: g.duckUsed, duckUsed: g.duckUsed,
toolCalls: g.toolCalls, toolCalls: g.toolCalls,
avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0,
avgPromptTokens: round(avg(g.promptTokens)), avgPromptTokens: round(avg(g.promptTokens)),
avgGenTokens: round(avg(g.genTokens)), avgGenTokens: round(avg(g.genTokens)),
avgDuckTokens: round(avg(g.duckTokens)), avgDuckTokens: round(avg(g.duckTokens)),
@@ -49,6 +50,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
model, model,
mcpUrl, mcpUrl,
scenarios: summaries, scenarios: summaries,
prompts,
perTask: rows.map((r) => ({ perTask: rows.map((r) => ({
id: r.id, id: r.id,
scenario: r.scenario, scenario: r.scenario,
+2 -2
View File
@@ -3,7 +3,7 @@ import path from "node:path";
import { Ollama } from "../lib/ollama.mjs"; import { Ollama } from "../lib/ollama.mjs";
import { McpClient } from "../lib/mcpClient.mjs"; import { McpClient } from "../lib/mcpClient.mjs";
import { runScenario } from "./runner.mjs"; import { runScenario } from "./runner.mjs";
import { isCorrect } from "./runner.mjs"; import { isCorrect, PROMPTS } from "./runner.mjs";
import { buildReport } from "./report.mjs"; import { buildReport } from "./report.mjs";
function parseArgs(argv) { function parseArgs(argv) {
@@ -94,7 +94,7 @@ async function main() {
} }
} }
const report = buildReport({ model: args.model, mcpUrl, tasks, rows }); const report = buildReport({ model: args.model, mcpUrl, tasks, rows, prompts: PROMPTS });
const out = args.out || path.join("out", "report.json"); const out = args.out || path.join("out", "report.json");
await mkdir(path.dirname(out), { recursive: true }); await mkdir(path.dirname(out), { recursive: true });
await writeFile(out, JSON.stringify(report, null, 2), "utf8"); await writeFile(out, JSON.stringify(report, null, 2), "utf8");
+7 -3
View File
@@ -27,15 +27,19 @@ const CONTROL_SYSTEM =
const BLIND_SYSTEM = const BLIND_SYSTEM =
"You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " + "You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " +
"First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " + "First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " +
"Then use the 'quack' tool so the colleague can give you a brief reaction, and wait for that reaction. " + "Wait for the colleague's reply. Then, taking that reply into account, complete your solution and give the final answer to the user.";
"Then, taking the colleague's reaction into account, complete your solution and give the final answer to the user. " +
"Note: your colleague is a rubber duck; it replies only with a short quack. Even so, saying your reasoning aloud helps you spot mistakes.";
const MENTOR_SYSTEM = const MENTOR_SYSTEM =
"You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " + "You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " +
"your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " + "your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " +
"Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user."; "Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user.";
export const PROMPTS = {
control: CONTROL_SYSTEM,
blind: BLIND_SYSTEM,
mentor: MENTOR_SYSTEM,
};
const MAX_DUCK_TURNS = 6; const MAX_DUCK_TURNS = 6;
function normalize(s) { function normalize(s) {