From b6b89d128a300bf4c8cf705e0bd2f9a611e8dc70 Mon Sep 17 00:00:00 2001 From: Ku6epXBOCTuK Date: Thu, 3 Sep 2026 19:12:25 +0500 Subject: [PATCH] feat: add run test flow --- apps/eval/data/tasks.json | 107 +++++++++++++++++ apps/eval/lib/mcpClient.mjs | 56 +++++++++ apps/eval/lib/ollama.mjs | 55 +++++++++ apps/eval/package.json | 14 +++ apps/eval/src/report-html.mjs | 210 ++++++++++++++++++++++++++++++++++ apps/eval/src/report.mjs | 67 +++++++++++ apps/eval/src/run.mjs | 115 +++++++++++++++++++ apps/eval/src/runner.mjs | 195 +++++++++++++++++++++++++++++++ pnpm-lock.yaml | 89 ++++++++++++++ 9 files changed, 908 insertions(+) create mode 100644 apps/eval/data/tasks.json create mode 100644 apps/eval/lib/mcpClient.mjs create mode 100644 apps/eval/lib/ollama.mjs create mode 100644 apps/eval/package.json create mode 100644 apps/eval/src/report-html.mjs create mode 100644 apps/eval/src/report.mjs create mode 100644 apps/eval/src/run.mjs create mode 100644 apps/eval/src/runner.mjs diff --git a/apps/eval/data/tasks.json b/apps/eval/data/tasks.json new file mode 100644 index 0000000..8822172 --- /dev/null +++ b/apps/eval/data/tasks.json @@ -0,0 +1,107 @@ +[ + { + "id": "t1", + "question": "На столе лежало 3 яблока. Ты взял 2 яблока. Сколько яблок осталось на столе? Ответь одним числом.", + "answer": "1" + }, + { + "id": "t2", + "question": "Фермер имеет 17 кур. Все, кроме 9, умерли. Сколько кур осталось живыми? Ответь одним числом.", + "answer": "9" + }, + { + "id": "t3", + "question": "Сколько месяцев в году имеют 28 дней? Ответь одним числом.", + "answer": "12" + }, + { + "id": "t4", + "question": "Карандаш и ручка вместе стоят 1 рубль 10 копеек. Ручка стоит на 1 рубль дороже карандаша. Сколько стоит карандаш? Ответь числом в копейках.", + "answer": "5" + }, + { + "id": "t5", + "question": "Если 5 машин за 5 минут делают 5 деталей, сколько деталей сделают 100 машин за 100 минут? Ответь одним числом.", + "answer": "2000" + }, + { + "id": "t6", + "question": "В озере растут кувшинки. Каждый день их количество удваивается. Пруд полностью покрывается за 48 дней. За сколько дней покрывается половина пруда? Ответь одним числом.", + "answer": "47" + }, + { + "id": "t7", + "question": "Числа от 1 до 9 включительно: сколько из них содержат букву «и» в русском названии? Ответь одним числом.", + "answer": "2" + }, + { + "id": "t8", + "question": "У тебя список из 12 чисел. Если удалить каждое второе число в списке, сколько чисел останется? Ответь одним числом.", + "answer": "6" + }, + { + "id": "t9", + "question": "Монетку подбросили 3 раза. Какова вероятность, что выпадет орёл все 3 раза? Ответь обыкновенной дробью.", + "answer": "1/8" + }, + { + "id": "t10", + "question": "У меня есть 10 рублей. Я потратил 3.50 на хлеб и 1.50 на молоко. Сколько сдачи осталось? Ответь числом в рублях.", + "answer": "5" + }, + { + "id": "t11", + "question": "Поезд длиной 100 метров движется со скоростью 36 км/ч. За сколько секунд он полностью проедет мимо столба? Ответь одним числом.", + "answer": "10" + }, + { + "id": "t12", + "question": "Если число увеличить на 30% и получить 78, чему было исходное число? Ответь одним числом.", + "answer": "60" + }, + { + "id": "t13", + "question": "В комнате 4 угла. В каждом углу сидит кошка. Напротив каждой кошки сидят 3 кошки. Сколько всего кошек в комнате? Ответь одним числом.", + "answer": "4" + }, + { + "id": "t14", + "question": "Периметр квадрата 28 см. Чему равна его площадь в квадратных сантиметрах? Ответь одним числом.", + "answer": "49" + }, + { + "id": "t15", + "question": "Лена вдвое старше Миши. Сумма их возрастов 36 лет. Сколько лет Мише? Ответь одним числом.", + "answer": "12" + }, + { + "id": "t16", + "question": "В шкафу 10 белых и 10 чёрных носков вперемешку. Сколько носков надо достать вслепую, чтобы гарантированно получить пару одного цвета? Ответь одним числом.", + "answer": "3" + }, + { + "id": "t17", + "question": "Если 3 курицы несут 3 яйца за 3 дня, сколько яиц снесут 6 куриц за 6 дней? Ответь одним числом.", + "answer": "12" + }, + { + "id": "t18", + "question": "Восемь минус четыре, делённое на два (8 - 4/2). Чему равно выражение? Ответь одним числом.", + "answer": "6" + }, + { + "id": "t19", + "question": "На столе 7 свечей. 3 потухли. Сколько свечей осталось на столе? Ответь одним числом.", + "answer": "7" + }, + { + "id": "t20", + "question": "У Вити 5 машинок, у Кати в 3 раза больше. Потом Катя подарила Вите столько, сколько у него было изначально. Сколько машинок стало у Кати? Ответь одним числом.", + "answer": "10" + }, + { + "id": "t21", + "question": "Сумма трёх последовательных нечётных чисел равна 27. Чему равно наибольшее из них? Ответь одним числом.", + "answer": "11" + } +] diff --git a/apps/eval/lib/mcpClient.mjs b/apps/eval/lib/mcpClient.mjs new file mode 100644 index 0000000..29adbdf --- /dev/null +++ b/apps/eval/lib/mcpClient.mjs @@ -0,0 +1,56 @@ +const ACCEPT = "application/json, text/event-stream"; + +function parseMcpResponse(text) { + const trimmed = text.trim(); + if (!trimmed) throw new Error("Empty MCP response"); + if (trimmed.startsWith("{")) { + return JSON.parse(trimmed); + } + if (trimmed.includes("event:") || trimmed.includes("data:")) { + let payload = ""; + for (const line of trimmed.split(/\r?\n/)) { + if (line.startsWith("data:")) payload += line.slice(5).trim(); + } + if (payload.startsWith("{")) return JSON.parse(payload); + } + throw new Error(`Unrecognized MCP response format: ${trimmed.slice(0, 200)}`); +} + +export class McpClient { + constructor(url) { + this.url = url; + this.seq = 0; + } + + async request(method, params = {}) { + this.seq += 1; + const res = await fetch(this.url, { + method: "POST", + headers: { "Content-Type": "application/json", Accept: ACCEPT }, + body: JSON.stringify({ jsonrpc: "2.0", id: this.seq, method, params }), + }); + if (!res.ok) { + throw new Error(`MCP HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`); + } + const rpc = parseMcpResponse(await res.text()); + if (rpc.error) throw new Error(`MCP RPC error ${rpc.error.code}: ${rpc.error.message}`); + return rpc; + } + + async listTools() { + const rpc = await this.request("tools/list"); + return rpc.result?.tools ?? []; + } + + async callTool(name, args = {}) { + const rpc = await this.request("tools/call", { name, arguments: args }); + return rpc.result; + } +} + +export function textContentFrom(result) { + if (!result?.content) return ""; + return result.content + .map((b) => (b.type === "text" ? b.text : JSON.stringify(b))) + .join(" "); +} diff --git a/apps/eval/lib/ollama.mjs b/apps/eval/lib/ollama.mjs new file mode 100644 index 0000000..ddd393e --- /dev/null +++ b/apps/eval/lib/ollama.mjs @@ -0,0 +1,55 @@ +const DEFAULT_URL = "http://localhost:11434"; + +export class Ollama { + constructor({ url = DEFAULT_URL } = {}) { + this.url = url.replace(/\/$/, ""); + } + + async ping() { + if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, model: process.env.OLLAMA_MODEL }; + const res = await fetch(`${this.url}/api/tags`); + if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`); + const data = await res.json(); + const models = (data.models ?? []).map((m) => m.name); + return { ok: true, models }; + } + + async chat({ model, messages, tools, temperature = 0, numCtx = 8192 }) { + const body = { + model, + messages, + stream: false, + options: { temperature }, + }; + if (numCtx) body.options.num_ctx = numCtx; + if (tools && tools.length) body.tools = tools; + + const res = await fetch(`${this.url}/api/chat`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const t = await res.text(); + throw new Error(`Ollama chat HTTP ${res.status}: ${t.slice(0, 300)}`); + } + const data = await res.json(); + const message = data.message ?? {}; + return { + role: message.role ?? "assistant", + content: message.content ?? "", + toolCalls: message.tool_calls ?? [], + promptEvalCount: data.prompt_eval_count ?? 0, + evalCount: data.eval_count ?? 0, + raw: data, + }; + } +} + +export async function listModels() { + const o = new Ollama(); + const res = await fetch(`${o.url}/api/tags`); + if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status})`); + const data = await res.json(); + return (data.models ?? []).map((m) => m.name); +} diff --git a/apps/eval/package.json b/apps/eval/package.json new file mode 100644 index 0000000..61d3e63 --- /dev/null +++ b/apps/eval/package.json @@ -0,0 +1,14 @@ +{ + "name": "eval", + "version": "1.0.0", + "description": "MCP rubber-duck evaluation harness", + "scripts": { + "run": "node src/run.mjs", + "report-html": "node src/report-html.mjs" + }, + "license": "ISC", + "type": "module", + "dependencies": { + "@modelcontextprotocol/client": "^2.0.0" + } +} diff --git a/apps/eval/src/report-html.mjs b/apps/eval/src/report-html.mjs new file mode 100644 index 0000000..b047dd6 --- /dev/null +++ b/apps/eval/src/report-html.mjs @@ -0,0 +1,210 @@ +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import path from "node:path"; + +const SCENARIO_LABELS = { + control: "Без утки (контроль)", + blind: "Слепая утка", + mentor: "Утка-помощник", +}; + +const SCENARIO_DESCS = { + control: "Модель решает задачу напрямую, без инструментов.", + blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.", + mentor: "Модель обязана выписать мысли и возможные ошибки, затем проверить себя уткой.", +}; + +const MOOD_COLORS = { happy: "#f59e0b", confused: "#8b5cf6", excited: "#10b981", sleepy: "#64748b" }; + +function esc(s) { + return String(s ?? "") + .replace(/&/g, "&") + .replace(//g, ">"); +} + +function nl2br(s) { + return esc(s).replace(/\n/g, "
"); +} + +function scenarioCard(key, g) { + const pct = g.accuracy; + const color = pct >= 70 ? "#10b981" : pct >= 40 ? "#f59e0b" : "#ef4444"; + const duckPct = g.tasks ? Math.round((g.duckUsed / g.tasks) * 100) : 0; + return ` +
+
+
+
${esc(SCENARIO_LABELS[key])}
+
${esc(SCENARIO_DESCS[key])}
+
+
${pct}%
+
+
+
+
Верно${g.correct} / ${g.tasks}
+
Утка${duckPct}% (${g.duckUsed}/${g.tasks})
+
Вызовов${g.toolCalls}
+
Input${g.avgPromptTokens}
+
Output${g.avgGenTokens}
+
Рассужд.${g.avgDuckTokens}
+
+
`; +} + +function rowHtml(r) { + const cls = r.correct ? "ok" : "no"; + const duck = r.duckUsed ? "🦆" : "—"; + return ` + + ${esc(r.id)} + ${esc(SCENARIO_LABELS[r.scenario])} + ${r.correct ? "✓" : "✗"} + ${duck} (${r.toolCalls}) + ${nl2br(r.question)} + ${esc(r.expected)} + ${r.promptTokens} / ${r.genTokens} + ${nl2br(r.response)} + `; +} + +export function buildHtml(report) { + const s = report.scenarios; + const cards = ["control", "blind", "mentor"] + .filter((k) => s[k]) + .map((k) => scenarioCard(k, s[k])) + .join(""); + + const rows = report.perTask.map(rowHtml).join(""); + + return ` + + + + +Rubber Duck MCP — отчёт тестирования + + + +
+
+

🦆 Rubber Duck MCP — отчёт тестирования

+
+ Модель: ${esc(report.model)}  ·  + MCP: ${esc(report.mcpUrl)}  ·  + Задач: ${s.control?.tasks ?? report.perTask.length}  ·  + Дата: ${esc(report.generatedAt)} +
+
+ +
${cards}
+ +
+ + + + + +
+ +
+ + + + + + + + ${rows} +
#Сценарий✓УткаВопросОжид.in/outОтвет модели
+
+ +
Генерировано локальным харнессом apps/eval. temperature=0, llama3.2:3b в данном прогоне.
+
+ + + +`; +} + +async function main() { + const args = process.argv.slice(2); + const inFile = args.find((a) => a.startsWith("--in="))?.slice(5) || "out/report.json"; + const outFile = args.find((a) => a.startsWith("--out="))?.slice(6) || "out/report.html"; + + const report = JSON.parse(await readFile(inFile, "utf8")); + const html = buildHtml(report); + await mkdir(path.dirname(outFile), { recursive: true }); + await writeFile(outFile, html, "utf8"); + console.log(`HTML report written: ${path.resolve(outFile)}`); +} + +if (process.argv[1] && path.resolve(process.argv[1]).includes("report-html")) { + main().catch((e) => { + console.error(e); + process.exit(1); + }); +} diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.mjs new file mode 100644 index 0000000..a4021b6 --- /dev/null +++ b/apps/eval/src/report.mjs @@ -0,0 +1,67 @@ +import { isCorrect } from "./runner.mjs"; + +function round(x, d = 2) { + if (!Number.isFinite(x)) return x; + const f = 10 ** d; + return Math.round(x * f) / f; +} + +function avg(arr) { + if (!arr.length) return 0; + return arr.reduce((a, b) => a + b, 0) / arr.length; +} + +export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) { + const scenarios = ["control", "blind", "mentor"]; + const byScenario = {}; + for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; + + for (const row of rows) { + const s = row.scenario; + if (!byScenario[s]) continue; + byScenario[s].total += 1; + if (row.correct) byScenario[s].correct += 1; + if (row.duckUsed) byScenario[s].duckUsed += 1; + byScenario[s].toolCalls += row.toolCalls ?? 0; + byScenario[s].promptTokens.push(row.promptTokens ?? 0); + byScenario[s].genTokens.push(row.genTokens ?? 0); + byScenario[s].duckTokens.push(row.duckTokens ?? 0); + byScenario[s].rows.push(row); + } + + const summaries = {}; + for (const s of scenarios) { + const g = byScenario[s]; + summaries[s] = { + tasks: g.total, + correct: g.correct, + accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0, + duckUsed: g.duckUsed, + toolCalls: g.toolCalls, + avgPromptTokens: round(avg(g.promptTokens)), + avgGenTokens: round(avg(g.genTokens)), + avgDuckTokens: round(avg(g.duckTokens)), + }; + } + + return { + generatedAt: new Date().toISOString(), + model, + mcpUrl, + scenarios: summaries, + perTask: rows.map((r) => ({ + id: r.id, + scenario: r.scenario, + correct: r.correct, + duckUsed: r.duckUsed, + toolCalls: r.toolCalls, + promptTokens: r.promptTokens, + genTokens: r.genTokens, + duckTokens: r.duckTokens, + response: r.response, + expected: r.expected, + question: r.question, + })), + meta, + }; +} diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.mjs new file mode 100644 index 0000000..bea1714 --- /dev/null +++ b/apps/eval/src/run.mjs @@ -0,0 +1,115 @@ +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import path from "node:path"; +import { Ollama } from "../lib/ollama.mjs"; +import { McpClient } from "../lib/mcpClient.mjs"; +import { runScenario } from "./runner.mjs"; +import { isCorrect } from "./runner.mjs"; +import { buildReport } from "./report.mjs"; + +function parseArgs(argv) { + const args = { scenarios: ["control", "blind", "mentor"], limit: Infinity, model: process.env.OLLAMA_MODEL || null }; + for (let i = 0; i < argv.length; i += 1) { + const a = argv[i]; + const next = () => argv[i + 1]; + if (a === "--model") args.model = next(), i += 1; + else if (a === "--mcp-url") args.mcpUrl = next(), i += 1; + else if (a === "--scenarios") args.scenarios = next().split(","), i += 1; + else if (a === "--limit") args.limit = Number(next()), i += 1; + else if (a === "--tasks") args.tasksFile = next(), i += 1; + else if (a === "--out") args.out = next(), i += 1; + } + return args; +} + +const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp"; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (!args.model) { + console.error("Usage: node src/run.mjs --model [--mcp-url URL] [--scenarios c,b,m] [--limit N] [--tasks file] [--out file]"); + process.exit(1); + } + const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL; + const tasksFile = args.tasksFile || path.join("data", "tasks.json"); + + const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8")); + const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit); + + console.log(`\nModel: ${args.model}`); + console.log(`MCP endpoint: ${mcpUrl}`); + console.log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`); + + const ollama = new Ollama(); + const ping = await ollama.ping(); + console.log(`Ollama OK (${ping.models.length} models)`); + const modelPresent = ping.models.some((m) => m.startsWith(args.model)); + if (!modelPresent) { + console.warn(` (warning) model "${args.model}" not found among: ${ping.models.join(", ")}`); + } + + const mcp = new McpClient(mcpUrl); + const mcpTools = await mcp.listTools(); + console.log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`); + + const rows = []; + for (const task of tasks) { + for (const scenario of args.scenarios) { + let r; + try { + r = await runScenario({ scenario, ollama, model: args.model, task: task.question, mcp }); + } catch (err) { + console.error(` [${task.id}/${scenario}] FAILED: ${err.message}`); + rows.push({ + id: task.id, + scenario, + correct: false, + duckUsed: false, + toolCalls: 0, + promptTokens: 0, + genTokens: 0, + duckTokens: 0, + response: `ERR: ${err.message}`, + expected: task.answer, + question: task.question, + }); + continue; + } + const correct = isCorrect(r.response, task.answer); + rows.push({ + id: task.id, + scenario, + correct, + duckUsed: r.duckUsed, + toolCalls: r.toolCalls, + promptTokens: r.promptTokens, + genTokens: r.genTokens, + duckTokens: r.duckTokens, + response: r.response, + expected: task.answer, + question: task.question, + }); + const mark = correct ? "OK " : "XX "; + const duck = r.duckUsed ? "duck" : "n/a "; + console.log(`${scenario.padEnd(8)} ${mark} ${task.id} duck=${duck} prompt=${r.promptTokens} gen=${r.genTokens} | ${r.response.slice(0, 70).replace(/\n/g, " ")}`); + } + } + + const report = buildReport({ model: args.model, mcpUrl, tasks, rows }); + const out = args.out || path.join("out", "report.json"); + await mkdir(path.dirname(out), { recursive: true }); + await writeFile(out, JSON.stringify(report, null, 2), "utf8"); + console.log(`\nReport written: ${out}`); + console.log("\n--- Summary ---"); + for (const [s, g] of Object.entries(report.scenarios)) { + console.log( + `${s.padEnd(8)} accuracy=${String(g.accuracy).padStart(5)}% (${g.correct}/${g.tasks}) ` + + `duckUsed=${g.duckUsed} toolCalls=${g.toolCalls} ` + + `avgPrompt=${g.avgPromptTokens} avgGen=${g.avgGenTokens} avgDuck=${g.avgDuckTokens}` + ); + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/apps/eval/src/runner.mjs b/apps/eval/src/runner.mjs new file mode 100644 index 0000000..01e6558 --- /dev/null +++ b/apps/eval/src/runner.mjs @@ -0,0 +1,195 @@ +import { Ollama } from "../lib/ollama.mjs"; +import { McpClient, textContentFrom } from "../lib/mcpClient.mjs"; + +const DUCK_TOOL = { + type: "function", + function: { + name: "quack", + description: + "Explain your current reasoning, steps, doubts or possible mistakes to your rubber duck colleague, and get a short acknowledgment in return. The duck replies with a quack.", + parameters: { + type: "object", + properties: { + mood: { + type: "string", + enum: ["happy", "confused", "excited", "sleepy"], + description: "Mood of the duck while you explain. Omit for a random quack.", + }, + }, + additionalProperties: false, + }, + }, +}; + +const CONTROL_SYSTEM = + "You are an experienced assistant. Solve the user's problem as accurately as possible. Give only the final answer."; + +const BLIND_SYSTEM = + "You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " + + "First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " + + "Then use the 'quack' tool so the colleague can give you a brief reaction, and wait for that reaction. " + + "Then, taking the colleague's reaction into account, complete your solution and give the final answer to the user. " + + "Note: your colleague is a rubber duck; it replies only with a short quack. Even so, saying your reasoning aloud helps you spot mistakes."; + +const MENTOR_SYSTEM = + "You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " + + "your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " + + "Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user."; + +const MAX_DUCK_TURNS = 6; + +function normalize(s) { + return String(s ?? "") + .toLowerCase() + .replace(/[^a-zа-яё0-9/]/g, "") + .replace(/ё/g, "е") + .trim(); +} + +function extractFinalNumber(text) { + const t = String(text ?? ""); + const re = /(?:ответ|answer|otvet|итог|итак|final)\s*[:：—]?\s*/gi; + let segment = t; + let m; + while ((m = re.exec(t)) !== null) { + if (m[0].trim()) segment = t.slice(re.lastIndex); + } + const digits = segment.match(/\d+/g) || []; + return digits.length ? digits[digits.length - 1] : null; +} + +function isNumeric(s) { + return /^\d+(?:[.,]\d+)?$/.test(String(s).trim()); +} + +export function isCorrect(predicted, expected) { + const e = String(expected ?? "").trim(); + const p = String(predicted ?? ""); + if (!e || !p) return false; + + // Дробный ответ (например "1/8") — сравниваем вхождение нормализованной дроби. + if (/\d+\s*\/\s*\d+/.test(e)) { + const eNorm = normalize(e); + const pNorm = normalize(p); + return pNorm.includes(eNorm); + } + + // Числовой ответ — извлекаем финальное число и сравниваем точно. + if (isNumeric(e)) { + const cand = extractFinalNumber(p); + if (cand == null) return false; + return normalizeNum(cand) === normalizeNum(e); + } + + // Текстовый ответ — нормализованное вхождение слова. + const eNorm = normalize(e); + const pNorm = normalize(p); + return pNorm.includes(eNorm); +} + +function normalizeNum(n) { + return String(n).replace(/[.,]/g, "").replace(/^0+(?=\d)/, ""); +} + +function toolDefinition() { + return DUCK_TOOL; +} + +async function runControl(ollama, model, task) { + const messages = [ + { role: "system", content: CONTROL_SYSTEM }, + { role: "user", content: task }, + ]; + const r = await ollama.chat({ model, messages }); + return { + response: r.content, + toolCalls: 0, + duckUsed: false, + duckTokens: 0, + promptTokens: r.promptEvalCount, + genTokens: r.evalCount, + messages, + }; +} + +async function runDuck(ollama, model, task, mcp, system, scenario) { + const messages = [ + { role: "system", content: system }, + { role: "user", content: task }, + ]; + let toolCalls = 0; + let duckUsed = false; + let totalPrompt = 0; + let totalGen = 0; + let reasoningTokens = 0; + let turns = 0; + + for (;;) { + turns += 1; + const r = await ollama.chat({ model, messages, tools: [toolDefinition()] }); + totalPrompt += r.promptEvalCount; + totalGen += r.evalCount; + reasoningTokens += r.evalCount; + + if (r.toolCalls && r.toolCalls.length) { + toolCalls += r.toolCalls.length; + const assistantMsg = { role: "assistant", content: r.content || "" }; + assistantMsg.tool_calls = r.toolCalls.map((tc) => ({ + function: { name: tc.function.name, arguments: tc.function.arguments }, + })); + messages.push(assistantMsg); + + for (const tc of r.toolCalls) { + if (tc.function.name === "quack") { + duckUsed = true; + let args = {}; + try { + args = tc.function.arguments ? JSON.parse(tc.function.arguments) : {}; + } catch { + args = {}; + } + const result = await mcp.callTool("quack", args); + const text = textContentFrom(result) || "QUACK!"; + messages.push({ role: "tool", content: text }); + } else { + messages.push({ role: "tool", content: "{}" }); + } + } + if (turns >= MAX_DUCK_TURNS) break; + continue; + } + + messages.push({ role: "assistant", content: r.content || "" }); + return { + response: r.content, + toolCalls, + duckUsed, + duckTokens: reasoningTokens, + promptTokens: totalPrompt, + genTokens: totalGen, + turns, + messages, + }; + } + + // loop bound hit without a final answer + const last = messages[messages.length - 1]; + return { + response: last?.content ?? "", + toolCalls, + duckUsed, + duckTokens: reasoningTokens, + promptTokens: totalPrompt, + genTokens: totalGen, + turns, + messages, + }; +} + +export async function runScenario({ scenario, ollama, model, task, mcp }) { + if (scenario === "control") { + return runControl(ollama, model, task); + } + const system = scenario === "blind" ? BLIND_SYSTEM : MENTOR_SYSTEM; + return runDuck(ollama, model, task, mcp, system, scenario); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c1a651d..2a51dc8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -207,6 +207,12 @@ importers: .: {} + apps/eval: + dependencies: + '@modelcontextprotocol/client': + specifier: ^2.0.0 + version: 2.0.0 + apps/frontend: devDependencies: vite: @@ -414,6 +420,10 @@ packages: cpu: [x64] os: [win32] + '@modelcontextprotocol/client@2.0.0': + resolution: {integrity: sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==} + engines: {node: '>=20'} + '@modelcontextprotocol/core@2.0.0': resolution: {integrity: sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==} engines: {node: '>=20'} @@ -612,6 +622,10 @@ packages: resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} engines: {node: '>=16'} + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} @@ -619,6 +633,14 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + eventsource-parser@3.1.1: + resolution: {integrity: sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -633,6 +655,12 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + jose@6.2.10: + resolution: {integrity: sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==} + lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -744,6 +772,10 @@ packages: sass: optional: true + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -751,6 +783,10 @@ packages: resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} engines: {node: '>=12'} + pkce-challenge@5.0.1: + resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} + engines: {node: '>=16.20.0'} + postcss@8.5.23: resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} engines: {node: ^10 || ^12 || >=14} @@ -790,6 +826,14 @@ packages: '@types/node': optional: true + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -865,6 +909,11 @@ packages: yaml: optional: true + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + zod@4.5.4: resolution: {integrity: sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==} @@ -982,6 +1031,16 @@ snapshots: '@img/sharp-win32-x64@0.35.4': optional: true + '@modelcontextprotocol/client@2.0.0': + dependencies: + '@modelcontextprotocol/core': 2.0.0 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.1.1 + jose: 6.2.10 + pkce-challenge: 5.0.1 + zod: 4.5.4 + '@modelcontextprotocol/core@2.0.0': dependencies: zod: 4.5.4 @@ -1092,10 +1151,22 @@ snapshots: commander@11.1.0: {} + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + csstype@3.2.3: {} detect-libc@2.1.2: {} + eventsource-parser@3.1.1: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.1.1 + fdir@6.5.0(picomatch@4.0.7): optionalDependencies: picomatch: 4.0.7 @@ -1103,6 +1174,10 @@ snapshots: fsevents@2.3.3: optional: true + isexe@2.0.0: {} + + jose@6.2.10: {} + lightningcss-android-arm64@1.33.0: optional: true @@ -1187,10 +1262,14 @@ snapshots: - '@types/node' - babel-plugin-macros + path-key@3.1.1: {} + picocolors@1.1.1: {} picomatch@4.0.7: {} + pkce-challenge@5.0.1: {} + postcss@8.5.23: dependencies: nanoid: 3.3.18 @@ -1270,6 +1349,12 @@ snapshots: '@types/node': 20.19.43 optional: true + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + source-map-js@1.2.1: {} styled-jsx@5.1.6(react@19.2.8): @@ -1299,4 +1384,8 @@ snapshots: '@types/node': 20.19.43 fsevents: 2.3.3 + which@2.0.2: + dependencies: + isexe: 2.0.0 + zod@4.5.4: {}