feat: update run tests and reports flow
This commit is contained in:
@@ -41,12 +41,12 @@ function scenarioCard(key, g) {
|
||||
</div>
|
||||
<div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
|
||||
<div class="metrics">
|
||||
<div class="metric"><span class="m-label">Верно</span><span class="m-val">${g.correct} / ${g.tasks}</span></div>
|
||||
<div class="metric"><span class="m-label">Утка</span><span class="m-val">${duckPct}% (${g.duckUsed}/${g.tasks})</span></div>
|
||||
<div class="metric"><span class="m-label">Вызовов</span><span class="m-val">${g.toolCalls}</span></div>
|
||||
<div class="metric"><span class="m-label">Input</span><span class="m-val">${g.avgPromptTokens}</span></div>
|
||||
<div class="metric"><span class="m-label">Output</span><span class="m-val">${g.avgGenTokens}</span></div>
|
||||
<div class="metric"><span class="m-label">Рассужд.</span><span class="m-val">${g.avgDuckTokens}</span></div>
|
||||
<div class="metric"><span class="m-label">Верных</span><span class="m-val">${g.correct} <span class="m-mod">/ ${g.tasks}</span></span></div>
|
||||
<div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${g.duckUsed} из ${g.tasks} задач</span></div>
|
||||
<div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${g.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
|
||||
<div class="metric"><span class="m-label">Промпт (in)</span><span class="m-val">${g.avgPromptTokens}</span><span class="m-sub">токенов</span></div>
|
||||
<div class="metric"><span class="m-label">Ответ (out)</span><span class="m-val">${g.avgGenTokens}</span><span class="m-sub">токенов</span></div>
|
||||
<div class="metric"><span class="m-label">Рассужд.</span><span class="m-val">${g.avgDuckTokens}</span><span class="m-sub">токенов</span></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
@@ -67,6 +67,26 @@ function rowHtml(r) {
|
||||
</tr>`;
|
||||
}
|
||||
|
||||
function promptBlock(report) {
|
||||
const p = report.prompts || {};
|
||||
const items = ["control", "blind", "mentor"]
|
||||
.filter((k) => p[k] && report.scenarios[k])
|
||||
.map(
|
||||
(k) => `
|
||||
<div class="prompt-item">
|
||||
<div class="prompt-name">${esc(SCENARIO_LABELS[k])}</div>
|
||||
<div class="prompt-text">${esc(p[k])}</div>
|
||||
</div>`
|
||||
)
|
||||
.join("");
|
||||
if (!items) return "";
|
||||
return `
|
||||
<details class="prompts" open>
|
||||
<summary>Промпты (то, что передаётся модели)</summary>
|
||||
<div class="prompts-body">${items}</div>
|
||||
</details>`;
|
||||
}
|
||||
|
||||
export function buildHtml(report) {
|
||||
const s = report.scenarios;
|
||||
const cards = ["control", "blind", "mentor"]
|
||||
@@ -103,14 +123,25 @@ export function buildHtml(report) {
|
||||
.accuracy { font-size:32px; font-weight:800; line-height:1; }
|
||||
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
|
||||
.bar-fill { height:100%; border-radius:99px; transition:width .4s; }
|
||||
.metrics { display:grid; grid-template-columns:repeat(3,1fr); gap:10px; }
|
||||
.metric { background:var(--panel2); border-radius:8px; padding:8px; text-align:center; }
|
||||
.m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.04em; }
|
||||
.m-val { font-size:16px; font-weight:700; }
|
||||
.metrics { display:grid; grid-template-columns:repeat(2,1fr); gap:10px; }
|
||||
.metric { background:var(--panel2); border-radius:8px; padding:10px 12px; }
|
||||
.m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.05em; white-space:nowrap; }
|
||||
.m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; }
|
||||
.m-mod { font-size:14px; font-weight:600; color:var(--muted); }
|
||||
.m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
|
||||
.filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
|
||||
.filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
|
||||
padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
|
||||
.filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
|
||||
.prompts { margin:0 0 20px; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
|
||||
.prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
|
||||
.prompts summary::-webkit-details-marker { display:none; }
|
||||
.prompts summary::before { content:"▸ "; color:var(--muted); }
|
||||
.prompts[open] summary::before { content:"▾ "; }
|
||||
.prompts-body { padding:0 16px 14px; display:grid; gap:12px; }
|
||||
.prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; }
|
||||
.prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; }
|
||||
.prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; }
|
||||
.tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
|
||||
table { width:100%; border-collapse:collapse; font-size:13px; }
|
||||
thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px;
|
||||
@@ -143,6 +174,8 @@ export function buildHtml(report) {
|
||||
|
||||
<div class="cards">${cards}</div>
|
||||
|
||||
${promptBlock(report)}
|
||||
|
||||
<div class="filters" id="filters">
|
||||
<button data-filter="all" class="active">Все</button>
|
||||
<button data-filter="control">Без утки</button>
|
||||
|
||||
@@ -11,7 +11,7 @@ function avg(arr) {
|
||||
return arr.reduce((a, b) => a + b, 0) / arr.length;
|
||||
}
|
||||
|
||||
export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
|
||||
export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {
|
||||
const scenarios = ["control", "blind", "mentor"];
|
||||
const byScenario = {};
|
||||
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
|
||||
@@ -38,6 +38,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
|
||||
accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0,
|
||||
duckUsed: g.duckUsed,
|
||||
toolCalls: g.toolCalls,
|
||||
avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0,
|
||||
avgPromptTokens: round(avg(g.promptTokens)),
|
||||
avgGenTokens: round(avg(g.genTokens)),
|
||||
avgDuckTokens: round(avg(g.duckTokens)),
|
||||
@@ -49,6 +50,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
|
||||
model,
|
||||
mcpUrl,
|
||||
scenarios: summaries,
|
||||
prompts,
|
||||
perTask: rows.map((r) => ({
|
||||
id: r.id,
|
||||
scenario: r.scenario,
|
||||
|
||||
@@ -3,7 +3,7 @@ import path from "node:path";
|
||||
import { Ollama } from "../lib/ollama.mjs";
|
||||
import { McpClient } from "../lib/mcpClient.mjs";
|
||||
import { runScenario } from "./runner.mjs";
|
||||
import { isCorrect } from "./runner.mjs";
|
||||
import { isCorrect, PROMPTS } from "./runner.mjs";
|
||||
import { buildReport } from "./report.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
@@ -94,7 +94,7 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
const report = buildReport({ model: args.model, mcpUrl, tasks, rows });
|
||||
const report = buildReport({ model: args.model, mcpUrl, tasks, rows, prompts: PROMPTS });
|
||||
const out = args.out || path.join("out", "report.json");
|
||||
await mkdir(path.dirname(out), { recursive: true });
|
||||
await writeFile(out, JSON.stringify(report, null, 2), "utf8");
|
||||
|
||||
@@ -27,15 +27,19 @@ const CONTROL_SYSTEM =
|
||||
const BLIND_SYSTEM =
|
||||
"You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " +
|
||||
"First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " +
|
||||
"Then use the 'quack' tool so the colleague can give you a brief reaction, and wait for that reaction. " +
|
||||
"Then, taking the colleague's reaction into account, complete your solution and give the final answer to the user. " +
|
||||
"Note: your colleague is a rubber duck; it replies only with a short quack. Even so, saying your reasoning aloud helps you spot mistakes.";
|
||||
"Wait for the colleague's reply. Then, taking that reply into account, complete your solution and give the final answer to the user.";
|
||||
|
||||
const MENTOR_SYSTEM =
|
||||
"You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " +
|
||||
"your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " +
|
||||
"Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user.";
|
||||
|
||||
export const PROMPTS = {
|
||||
control: CONTROL_SYSTEM,
|
||||
blind: BLIND_SYSTEM,
|
||||
mentor: MENTOR_SYSTEM,
|
||||
};
|
||||
|
||||
const MAX_DUCK_TURNS = 6;
|
||||
|
||||
function normalize(s) {
|
||||
|
||||
Reference in New Issue
Block a user