diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.mjs
index a4021b6..176cb98 100644
--- a/apps/eval/src/report.mjs
+++ b/apps/eval/src/report.mjs
@@ -11,7 +11,7 @@ function avg(arr) {
return arr.reduce((a, b) => a + b, 0) / arr.length;
}
-export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
+export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {
const scenarios = ["control", "blind", "mentor"];
const byScenario = {};
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
@@ -38,6 +38,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
accuracy: g.total ? round((g.correct / g.total) * 100, 1) : 0,
duckUsed: g.duckUsed,
toolCalls: g.toolCalls,
+ avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0,
avgPromptTokens: round(avg(g.promptTokens)),
avgGenTokens: round(avg(g.genTokens)),
avgDuckTokens: round(avg(g.duckTokens)),
@@ -49,6 +50,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {} }) {
model,
mcpUrl,
scenarios: summaries,
+ prompts,
perTask: rows.map((r) => ({
id: r.id,
scenario: r.scenario,
diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.mjs
index bea1714..bd11ae6 100644
--- a/apps/eval/src/run.mjs
+++ b/apps/eval/src/run.mjs
@@ -3,7 +3,7 @@ import path from "node:path";
import { Ollama } from "../lib/ollama.mjs";
import { McpClient } from "../lib/mcpClient.mjs";
import { runScenario } from "./runner.mjs";
-import { isCorrect } from "./runner.mjs";
+import { isCorrect, PROMPTS } from "./runner.mjs";
import { buildReport } from "./report.mjs";
function parseArgs(argv) {
@@ -94,7 +94,7 @@ async function main() {
}
}
- const report = buildReport({ model: args.model, mcpUrl, tasks, rows });
+ const report = buildReport({ model: args.model, mcpUrl, tasks, rows, prompts: PROMPTS });
const out = args.out || path.join("out", "report.json");
await mkdir(path.dirname(out), { recursive: true });
await writeFile(out, JSON.stringify(report, null, 2), "utf8");
diff --git a/apps/eval/src/runner.mjs b/apps/eval/src/runner.mjs
index 01e6558..143a87f 100644
--- a/apps/eval/src/runner.mjs
+++ b/apps/eval/src/runner.mjs
@@ -27,15 +27,19 @@ const CONTROL_SYSTEM =
const BLIND_SYSTEM =
"You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " +
"First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " +
- "Then use the 'quack' tool so the colleague can give you a brief reaction, and wait for that reaction. " +
- "Then, taking the colleague's reaction into account, complete your solution and give the final answer to the user. " +
- "Note: your colleague is a rubber duck; it replies only with a short quack. Even so, saying your reasoning aloud helps you spot mistakes.";
+ "Wait for the colleague's reply. Then, taking that reply into account, complete your solution and give the final answer to the user.";
const MENTOR_SYSTEM =
"You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " +
"your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " +
"Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user.";
+export const PROMPTS = {
+ control: CONTROL_SYSTEM,
+ blind: BLIND_SYSTEM,
+ mentor: MENTOR_SYSTEM,
+};
+
const MAX_DUCK_TURNS = 6;
function normalize(s) {