feat: add run test flow
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const SCENARIO_LABELS = {
|
||||
control: "Без утки (контроль)",
|
||||
blind: "Слепая утка",
|
||||
mentor: "Утка-помощник",
|
||||
};
|
||||
|
||||
const SCENARIO_DESCS = {
|
||||
control: "Модель решает задачу напрямую, без инструментов.",
|
||||
blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.",
|
||||
mentor: "Модель обязана выписать мысли и возможные ошибки, затем проверить себя уткой.",
|
||||
};
|
||||
|
||||
const MOOD_COLORS = { happy: "#f59e0b", confused: "#8b5cf6", excited: "#10b981", sleepy: "#64748b" };
|
||||
|
||||
function esc(s) {
|
||||
return String(s ?? "")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">");
|
||||
}
|
||||
|
||||
function nl2br(s) {
|
||||
return esc(s).replace(/\n/g, "<br>");
|
||||
}
|
||||
|
||||
function scenarioCard(key, g) {
|
||||
const pct = g.accuracy;
|
||||
const color = pct >= 70 ? "#10b981" : pct >= 40 ? "#f59e0b" : "#ef4444";
|
||||
const duckPct = g.tasks ? Math.round((g.duckUsed / g.tasks) * 100) : 0;
|
||||
return `
|
||||
<div class="card scenario" data-scenario="${key}">
|
||||
<div class="scenario-head">
|
||||
<div>
|
||||
<div class="scenario-title">${esc(SCENARIO_LABELS[key])}</div>
|
||||
<div class="scenario-desc">${esc(SCENARIO_DESCS[key])}</div>
|
||||
</div>
|
||||
<div class="accuracy" style="color:${color}">${pct}%</div>
|
||||
</div>
|
||||
<div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
|
||||
<div class="metrics">
|
||||
<div class="metric"><span class="m-label">Верно</span><span class="m-val">${g.correct} / ${g.tasks}</span></div>
|
||||
<div class="metric"><span class="m-label">Утка</span><span class="m-val">${duckPct}% (${g.duckUsed}/${g.tasks})</span></div>
|
||||
<div class="metric"><span class="m-label">Вызовов</span><span class="m-val">${g.toolCalls}</span></div>
|
||||
<div class="metric"><span class="m-label">Input</span><span class="m-val">${g.avgPromptTokens}</span></div>
|
||||
<div class="metric"><span class="m-label">Output</span><span class="m-val">${g.avgGenTokens}</span></div>
|
||||
<div class="metric"><span class="m-label">Рассужд.</span><span class="m-val">${g.avgDuckTokens}</span></div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function rowHtml(r) {
|
||||
const cls = r.correct ? "ok" : "no";
|
||||
const duck = r.duckUsed ? "🦆" : "—";
|
||||
return `
|
||||
<tr class="scenario-row ${cls}" data-scenario="${r.scenario}">
|
||||
<td class="r-id">${esc(r.id)}</td>
|
||||
<td class="r-s">${esc(SCENARIO_LABELS[r.scenario])}</td>
|
||||
<td class="r-result">${r.correct ? "✓" : "✗"}</td>
|
||||
<td class="r-duck">${duck} <span class="muted">(${r.toolCalls})</span></td>
|
||||
<td class="r-question">${nl2br(r.question)}</td>
|
||||
<td class="r-expected">${esc(r.expected)}</td>
|
||||
<td class="r-tokens">${r.promptTokens}<span class="muted"> / </span>${r.genTokens}</td>
|
||||
<td class="r-answer">${nl2br(r.response)}</td>
|
||||
</tr>`;
|
||||
}
|
||||
|
||||
export function buildHtml(report) {
|
||||
const s = report.scenarios;
|
||||
const cards = ["control", "blind", "mentor"]
|
||||
.filter((k) => s[k])
|
||||
.map((k) => scenarioCard(k, s[k]))
|
||||
.join("");
|
||||
|
||||
const rows = report.perTask.map(rowHtml).join("");
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Rubber Duck MCP — отчёт тестирования</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg:#0f172a; --panel:#1e293b; --panel2:#273449; --text:#e2e8f0;
|
||||
--muted:#94a3b8; --ok:#10b981; --no:#f87171; --border:#334155;
|
||||
}
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;
|
||||
background:var(--bg); color:var(--text); line-height:1.5; }
|
||||
.wrap { max-width:1200px; margin:0 auto; padding:24px 20px 60px; }
|
||||
header h1 { margin:0 0 4px; font-size:22px; }
|
||||
.meta { color:var(--muted); font-size:13px; }
|
||||
.meta b { color:var(--text); font-weight:600; }
|
||||
.cards { display:grid; grid-template-columns:repeat(3,1fr); gap:16px; margin:24px 0; }
|
||||
@media (max-width:900px){ .cards { grid-template-columns:1fr; } }
|
||||
.card { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; }
|
||||
.scenario-head { display:flex; justify-content:space-between; align-items:flex-start; gap:12px; }
|
||||
.scenario-title { font-size:16px; font-weight:700; }
|
||||
.scenario-desc { color:var(--muted); font-size:12px; margin-top:4px; }
|
||||
.accuracy { font-size:32px; font-weight:800; line-height:1; }
|
||||
.bar { height:8px; background:var(--panel2); border-radius:99px; margin:14px 0; overflow:hidden; }
|
||||
.bar-fill { height:100%; border-radius:99px; transition:width .4s; }
|
||||
.metrics { display:grid; grid-template-columns:repeat(3,1fr); gap:10px; }
|
||||
.metric { background:var(--panel2); border-radius:8px; padding:8px; text-align:center; }
|
||||
.m-label { display:block; color:var(--muted); font-size:11px; text-transform:uppercase; letter-spacing:.04em; }
|
||||
.m-val { font-size:16px; font-weight:700; }
|
||||
.filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
|
||||
.filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
|
||||
padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
|
||||
.filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
|
||||
.tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
|
||||
table { width:100%; border-collapse:collapse; font-size:13px; }
|
||||
thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px;
|
||||
font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); }
|
||||
tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; }
|
||||
.scenario-row.ok .r-result { color:var(--ok); font-weight:800; }
|
||||
.scenario-row.no .r-result { color:var(--no); font-weight:800; }
|
||||
.r-id { font-weight:700; color:var(--muted); }
|
||||
.r-s { white-space:nowrap; }
|
||||
.r-duck { text-align:center; white-space:nowrap; }
|
||||
.r-question { max-width:340px; color:var(--text); }
|
||||
.r-answer { max-width:420px; }
|
||||
.muted { color:var(--muted); }
|
||||
.hidden { display:none !important; }
|
||||
.toggle-row { background:transparent; border:none; color:#3b82f6; cursor:pointer; font-size:12px; padding:0; }
|
||||
footer { margin-top:20px; color:var(--muted); font-size:12px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header>
|
||||
<h1>🦆 Rubber Duck MCP — отчёт тестирования</h1>
|
||||
<div class="meta">
|
||||
Модель: <b>${esc(report.model)}</b> ·
|
||||
MCP: <b>${esc(report.mcpUrl)}</b> ·
|
||||
Задач: <b>${s.control?.tasks ?? report.perTask.length}</b> ·
|
||||
Дата: <b>${esc(report.generatedAt)}</b>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="cards">${cards}</div>
|
||||
|
||||
<div class="filters" id="filters">
|
||||
<button data-filter="all" class="active">Все</button>
|
||||
<button data-filter="control">Без утки</button>
|
||||
<button data-filter="blind">Слепая утка</button>
|
||||
<button data-filter="mentor">Утка-помощник</button>
|
||||
<span style="margin-left:auto;color:var(--muted);font-size:12px" id="count"></span>
|
||||
</div>
|
||||
|
||||
<div class="tbl-shell">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th><th>Сценарий</th><th>✓</th><th>Утка</th>
|
||||
<th>Вопрос</th><th>Ожид.</th><th>in/out</th><th>Ответ модели</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<footer>Генерировано локальным харнессом apps/eval. temperature=0, llama3.2:3b в данном прогоне.</footer>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const buttons = document.querySelectorAll('#filters button');
|
||||
const rows2 = document.querySelectorAll('tbody .scenario-row');
|
||||
const count = document.getElementById('count');
|
||||
function apply(f) {
|
||||
let n = 0;
|
||||
rows2.forEach(r => {
|
||||
const show = f === 'all' || r.dataset.scenario === f;
|
||||
r.classList.toggle('hidden', !show);
|
||||
if (show) n++;
|
||||
});
|
||||
count.textContent = 'показано: ' + n + ' из ' + rows2.length;
|
||||
}
|
||||
buttons.forEach(b => b.addEventListener('click', () => {
|
||||
buttons.forEach(x => x.classList.remove('active'));
|
||||
b.classList.add('active');
|
||||
apply(b.dataset.filter);
|
||||
}));
|
||||
apply('all');
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const inFile = args.find((a) => a.startsWith("--in="))?.slice(5) || "out/report.json";
|
||||
const outFile = args.find((a) => a.startsWith("--out="))?.slice(6) || "out/report.html";
|
||||
|
||||
const report = JSON.parse(await readFile(inFile, "utf8"));
|
||||
const html = buildHtml(report);
|
||||
await mkdir(path.dirname(outFile), { recursive: true });
|
||||
await writeFile(outFile, html, "utf8");
|
||||
console.log(`HTML report written: ${path.resolve(outFile)}`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) {
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { isCorrect } from "./runner.mjs";
|
||||
|
||||
function round(x, d = 2) {
|
||||
if (!Number.isFinite(x)) return x;
|
||||
const f = 10 ** d;
|
||||
return Math.round(x * f) / f;
|
||||
}
|
||||
|
||||
function avg(arr) {
|
||||
if (!arr.length) return 0;
|
||||
return arr.reduce((a, b) => a + b, 0) / arr.length;
|
||||
}
|
||||
|
||||
export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
|
||||
const scenarios = ["control", "blind", "mentor"];
|
||||
const byScenario = {};
|
||||
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
|
||||
|
||||
for (const row of rows) {
|
||||
const s = row.scenario;
|
||||
if (!byScenario[s]) continue;
|
||||
byScenario[s].total += 1;
|
||||
if (row.correct) byScenario[s].correct += 1;
|
||||
if (row.duckUsed) byScenario[s].duckUsed += 1;
|
||||
byScenario[s].toolCalls += row.toolCalls ?? 0;
|
||||
byScenario[s].promptTokens.push(row.promptTokens ?? 0);
|
||||
byScenario[s].genTokens.push(row.genTokens ?? 0);
|
||||
byScenario[s].duckTokens.push(row.duckTokens ?? 0);
|
||||
byScenario[s].rows.push(row);
|
||||
}
|
||||
|
||||
const summaries = {};
|
||||
for (const s of scenarios) {
|
||||
const g = byScenario[s];
|
||||
summaries[s] = {
|
||||
tasks: g.total,
|
||||
correct: g.correct,
|
||||
accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0,
|
||||
duckUsed: g.duckUsed,
|
||||
toolCalls: g.toolCalls,
|
||||
avgPromptTokens: round(avg(g.promptTokens)),
|
||||
avgGenTokens: round(avg(g.genTokens)),
|
||||
avgDuckTokens: round(avg(g.duckTokens)),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
model,
|
||||
mcpUrl,
|
||||
scenarios: summaries,
|
||||
perTask: rows.map((r) => ({
|
||||
id: r.id,
|
||||
scenario: r.scenario,
|
||||
correct: r.correct,
|
||||
duckUsed: r.duckUsed,
|
||||
toolCalls: r.toolCalls,
|
||||
promptTokens: r.promptTokens,
|
||||
genTokens: r.genTokens,
|
||||
duckTokens: r.duckTokens,
|
||||
response: r.response,
|
||||
expected: r.expected,
|
||||
question: r.question,
|
||||
})),
|
||||
meta,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { Ollama } from "../lib/ollama.mjs";
|
||||
import { McpClient } from "../lib/mcpClient.mjs";
|
||||
import { runScenario } from "./runner.mjs";
|
||||
import { isCorrect } from "./runner.mjs";
|
||||
import { buildReport } from "./report.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { scenarios: ["control", "blind", "mentor"], limit: Infinity, model: process.env.OLLAMA_MODEL || null };
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const a = argv[i];
|
||||
const next = () => argv[i + 1];
|
||||
if (a === "--model") args.model = next(), i += 1;
|
||||
else if (a === "--mcp-url") args.mcpUrl = next(), i += 1;
|
||||
else if (a === "--scenarios") args.scenarios = next().split(","), i += 1;
|
||||
else if (a === "--limit") args.limit = Number(next()), i += 1;
|
||||
else if (a === "--tasks") args.tasksFile = next(), i += 1;
|
||||
else if (a === "--out") args.out = next(), i += 1;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp";
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (!args.model) {
|
||||
console.error("Usage: node src/run.mjs --model <ollama-model> [--mcp-url URL] [--scenarios c,b,m] [--limit N] [--tasks file] [--out file]");
|
||||
process.exit(1);
|
||||
}
|
||||
const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL;
|
||||
const tasksFile = args.tasksFile || path.join("data", "tasks.json");
|
||||
|
||||
const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8"));
|
||||
const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit);
|
||||
|
||||
console.log(`\nModel: ${args.model}`);
|
||||
console.log(`MCP endpoint: ${mcpUrl}`);
|
||||
console.log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`);
|
||||
|
||||
const ollama = new Ollama();
|
||||
const ping = await ollama.ping();
|
||||
console.log(`Ollama OK (${ping.models.length} models)`);
|
||||
const modelPresent = ping.models.some((m) => m.startsWith(args.model));
|
||||
if (!modelPresent) {
|
||||
console.warn(` (warning) model "${args.model}" not found among: ${ping.models.join(", ")}`);
|
||||
}
|
||||
|
||||
const mcp = new McpClient(mcpUrl);
|
||||
const mcpTools = await mcp.listTools();
|
||||
console.log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`);
|
||||
|
||||
const rows = [];
|
||||
for (const task of tasks) {
|
||||
for (const scenario of args.scenarios) {
|
||||
let r;
|
||||
try {
|
||||
r = await runScenario({ scenario, ollama, model: args.model, task: task.question, mcp });
|
||||
} catch (err) {
|
||||
console.error(` [${task.id}/${scenario}] FAILED: ${err.message}`);
|
||||
rows.push({
|
||||
id: task.id,
|
||||
scenario,
|
||||
correct: false,
|
||||
duckUsed: false,
|
||||
toolCalls: 0,
|
||||
promptTokens: 0,
|
||||
genTokens: 0,
|
||||
duckTokens: 0,
|
||||
response: `ERR: ${err.message}`,
|
||||
expected: task.answer,
|
||||
question: task.question,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const correct = isCorrect(r.response, task.answer);
|
||||
rows.push({
|
||||
id: task.id,
|
||||
scenario,
|
||||
correct,
|
||||
duckUsed: r.duckUsed,
|
||||
toolCalls: r.toolCalls,
|
||||
promptTokens: r.promptTokens,
|
||||
genTokens: r.genTokens,
|
||||
duckTokens: r.duckTokens,
|
||||
response: r.response,
|
||||
expected: task.answer,
|
||||
question: task.question,
|
||||
});
|
||||
const mark = correct ? "OK " : "XX ";
|
||||
const duck = r.duckUsed ? "duck" : "n/a ";
|
||||
console.log(`${scenario.padEnd(8)} ${mark} ${task.id} duck=${duck} prompt=${r.promptTokens} gen=${r.genTokens} | ${r.response.slice(0, 70).replace(/\n/g, " ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
const report = buildReport({ model: args.model, mcpUrl, tasks, rows });
|
||||
const out = args.out || path.join("out", "report.json");
|
||||
await mkdir(path.dirname(out), { recursive: true });
|
||||
await writeFile(out, JSON.stringify(report, null, 2), "utf8");
|
||||
console.log(`\nReport written: ${out}`);
|
||||
console.log("\n--- Summary ---");
|
||||
for (const [s, g] of Object.entries(report.scenarios)) {
|
||||
console.log(
|
||||
`${s.padEnd(8)} accuracy=${String(g.accuracy).padStart(5)}% (${g.correct}/${g.tasks}) ` +
|
||||
`duckUsed=${g.duckUsed} toolCalls=${g.toolCalls} ` +
|
||||
`avgPrompt=${g.avgPromptTokens} avgGen=${g.avgGenTokens} avgDuck=${g.avgDuckTokens}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
import { Ollama } from "../lib/ollama.mjs";
|
||||
import { McpClient, textContentFrom } from "../lib/mcpClient.mjs";
|
||||
|
||||
const DUCK_TOOL = {
|
||||
type: "function",
|
||||
function: {
|
||||
name: "quack",
|
||||
description:
|
||||
"Explain your current reasoning, steps, doubts or possible mistakes to your rubber duck colleague, and get a short acknowledgment in return. The duck replies with a quack.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
mood: {
|
||||
type: "string",
|
||||
enum: ["happy", "confused", "excited", "sleepy"],
|
||||
description: "Mood of the duck while you explain. Omit for a random quack.",
|
||||
},
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const CONTROL_SYSTEM =
|
||||
"You are an experienced assistant. Solve the user's problem as accurately as possible. Give only the final answer.";
|
||||
|
||||
const BLIND_SYSTEM =
|
||||
"You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " +
|
||||
"First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " +
|
||||
"Then use the 'quack' tool so the colleague can give you a brief reaction, and wait for that reaction. " +
|
||||
"Then, taking the colleague's reaction into account, complete your solution and give the final answer to the user. " +
|
||||
"Note: your colleague is a rubber duck; it replies only with a short quack. Even so, saying your reasoning aloud helps you spot mistakes.";
|
||||
|
||||
const MENTOR_SYSTEM =
|
||||
"You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " +
|
||||
"your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " +
|
||||
"Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user.";
|
||||
|
||||
const MAX_DUCK_TURNS = 6;
|
||||
|
||||
function normalize(s) {
|
||||
return String(s ?? "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-zа-яё0-9/]/g, "")
|
||||
.replace(/ё/g, "е")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function extractFinalNumber(text) {
|
||||
const t = String(text ?? "");
|
||||
const re = /(?:ответ|answer|otvet|итог|итак|final)\s*[:：—]?\s*/gi;
|
||||
let segment = t;
|
||||
let m;
|
||||
while ((m = re.exec(t)) !== null) {
|
||||
if (m[0].trim()) segment = t.slice(re.lastIndex);
|
||||
}
|
||||
const digits = segment.match(/\d+/g) || [];
|
||||
return digits.length ? digits[digits.length - 1] : null;
|
||||
}
|
||||
|
||||
function isNumeric(s) {
|
||||
return /^\d+(?:[.,]\d+)?$/.test(String(s).trim());
|
||||
}
|
||||
|
||||
export function isCorrect(predicted, expected) {
|
||||
const e = String(expected ?? "").trim();
|
||||
const p = String(predicted ?? "");
|
||||
if (!e || !p) return false;
|
||||
|
||||
// Дробный ответ (например "1/8") — сравниваем вхождение нормализованной дроби.
|
||||
if (/\d+\s*\/\s*\d+/.test(e)) {
|
||||
const eNorm = normalize(e);
|
||||
const pNorm = normalize(p);
|
||||
return pNorm.includes(eNorm);
|
||||
}
|
||||
|
||||
// Числовой ответ — извлекаем финальное число и сравниваем точно.
|
||||
if (isNumeric(e)) {
|
||||
const cand = extractFinalNumber(p);
|
||||
if (cand == null) return false;
|
||||
return normalizeNum(cand) === normalizeNum(e);
|
||||
}
|
||||
|
||||
// Текстовый ответ — нормализованное вхождение слова.
|
||||
const eNorm = normalize(e);
|
||||
const pNorm = normalize(p);
|
||||
return pNorm.includes(eNorm);
|
||||
}
|
||||
|
||||
function normalizeNum(n) {
|
||||
return String(n).replace(/[.,]/g, "").replace(/^0+(?=\d)/, "");
|
||||
}
|
||||
|
||||
function toolDefinition() {
|
||||
return DUCK_TOOL;
|
||||
}
|
||||
|
||||
async function runControl(ollama, model, task) {
|
||||
const messages = [
|
||||
{ role: "system", content: CONTROL_SYSTEM },
|
||||
{ role: "user", content: task },
|
||||
];
|
||||
const r = await ollama.chat({ model, messages });
|
||||
return {
|
||||
response: r.content,
|
||||
toolCalls: 0,
|
||||
duckUsed: false,
|
||||
duckTokens: 0,
|
||||
promptTokens: r.promptEvalCount,
|
||||
genTokens: r.evalCount,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
|
||||
async function runDuck(ollama, model, task, mcp, system, scenario) {
|
||||
const messages = [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: task },
|
||||
];
|
||||
let toolCalls = 0;
|
||||
let duckUsed = false;
|
||||
let totalPrompt = 0;
|
||||
let totalGen = 0;
|
||||
let reasoningTokens = 0;
|
||||
let turns = 0;
|
||||
|
||||
for (;;) {
|
||||
turns += 1;
|
||||
const r = await ollama.chat({ model, messages, tools: [toolDefinition()] });
|
||||
totalPrompt += r.promptEvalCount;
|
||||
totalGen += r.evalCount;
|
||||
reasoningTokens += r.evalCount;
|
||||
|
||||
if (r.toolCalls && r.toolCalls.length) {
|
||||
toolCalls += r.toolCalls.length;
|
||||
const assistantMsg = { role: "assistant", content: r.content || "" };
|
||||
assistantMsg.tool_calls = r.toolCalls.map((tc) => ({
|
||||
function: { name: tc.function.name, arguments: tc.function.arguments },
|
||||
}));
|
||||
messages.push(assistantMsg);
|
||||
|
||||
for (const tc of r.toolCalls) {
|
||||
if (tc.function.name === "quack") {
|
||||
duckUsed = true;
|
||||
let args = {};
|
||||
try {
|
||||
args = tc.function.arguments ? JSON.parse(tc.function.arguments) : {};
|
||||
} catch {
|
||||
args = {};
|
||||
}
|
||||
const result = await mcp.callTool("quack", args);
|
||||
const text = textContentFrom(result) || "QUACK!";
|
||||
messages.push({ role: "tool", content: text });
|
||||
} else {
|
||||
messages.push({ role: "tool", content: "{}" });
|
||||
}
|
||||
}
|
||||
if (turns >= MAX_DUCK_TURNS) break;
|
||||
continue;
|
||||
}
|
||||
|
||||
messages.push({ role: "assistant", content: r.content || "" });
|
||||
return {
|
||||
response: r.content,
|
||||
toolCalls,
|
||||
duckUsed,
|
||||
duckTokens: reasoningTokens,
|
||||
promptTokens: totalPrompt,
|
||||
genTokens: totalGen,
|
||||
turns,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
|
||||
// loop bound hit without a final answer
|
||||
const last = messages[messages.length - 1];
|
||||
return {
|
||||
response: last?.content ?? "",
|
||||
toolCalls,
|
||||
duckUsed,
|
||||
duckTokens: reasoningTokens,
|
||||
promptTokens: totalPrompt,
|
||||
genTokens: totalGen,
|
||||
turns,
|
||||
messages,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runScenario({ scenario, ollama, model, task, mcp }) {
|
||||
if (scenario === "control") {
|
||||
return runControl(ollama, model, task);
|
||||
}
|
||||
const system = scenario === "blind" ? BLIND_SYSTEM : MENTOR_SYSTEM;
|
||||
return runDuck(ollama, model, task, mcp, system, scenario);
|
||||
}
|
||||
Reference in New Issue
Block a user