import type { ScenarioName, ScenarioSummary, PerTaskRow, ModelReport, Aggregate, AggregateScenario, AggregateModel } from "@duck/types"; import { SCENARIO_NAMES } from "@duck/types"; function round(x: number, d = 2): number { if (!Number.isFinite(x)) return x; const f = 10 ** d; return Math.round(x * f) / f; } function avg(arr: number[]): number { if (!arr.length) return 0; return arr.reduce((a, b) => a + b, 0) / arr.length; } export interface BuildReportOpts { model: string; mcpUrl: string; rows: PerTaskRow[]; meta?: Record; prompts?: Partial>; } export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: BuildReportOpts): ModelReport { const scenarios = SCENARIO_NAMES; const byScenario: Record = {}; for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; for (const row of rows) { const s = row.scenario; if (!byScenario[s]) continue; byScenario[s].total += 1; if (row.correct !== null && row.correct !== undefined) { byScenario[s].reviewed += 1; if (row.correct) byScenario[s].correct += 1; } if (row.duckUsed) byScenario[s].duckUsed += 1; byScenario[s].toolCalls += row.toolCalls ?? 0; byScenario[s].promptTokens.push(row.promptTokens ?? 0); byScenario[s].genTokens.push(row.genTokens ?? 0); byScenario[s].duckTokens.push(row.duckTokens ?? 0); byScenario[s].rows.push(row); } const summaries: Record = {} as Record; for (const s of scenarios) { const g = byScenario[s]; summaries[s] = { tasks: g.total, reviewed: g.reviewed, correct: g.correct, accuracy: g.reviewed ? round((g.correct / g.reviewed) * 100, 1) : 0, pending: g.total - g.reviewed, duckUsed: g.duckUsed, toolCalls: g.toolCalls, avgCalls: g.total ? round(g.toolCalls / g.total, 2) : 0, avgPromptTokens: round(avg(g.promptTokens)), avgGenTokens: round(avg(g.genTokens)), avgDuckTokens: round(avg(g.duckTokens)), }; } return { generatedAt: new Date().toISOString(), model, mcpUrl, scenarios: summaries, prompts: prompts as Record, perTask: rows.map((r) => ({ id: r.id, scenario: r.scenario, correct: r.correct, duckUsed: r.duckUsed, toolCalls: r.toolCalls, promptTokens: r.promptTokens, genTokens: r.genTokens, duckTokens: r.duckTokens, response: r.response, expected: r.expected, question: r.question, })), meta, }; } export function buildAggregate(modelsReport: Record): Aggregate { const perScenario: Record = {} as Record; for (const s of SCENARIO_NAMES) { let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0; const byModel: Record = {}; for (const [name, rep] of Object.entries(modelsReport)) { const g = rep.scenarios[s]; if (!g) continue; total += g.tasks; reviewed += g.reviewed; correct += g.correct; duckUsed += g.duckUsed; toolCalls += g.toolCalls; byModel[name] = { tasks: g.tasks, reviewed: g.reviewed, correct: g.correct, accuracy: g.reviewed ? g.accuracy : 0, duckUsed: g.duckUsed, }; } perScenario[s] = { tasks: total, reviewed, correct, accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0, pending: total - reviewed, duckUsed, toolCalls, avgCalls: total ? round(toolCalls / total, 2) : 0, byModel, }; } const perModel: Record = {}; for (const [name, rep] of Object.entries(modelsReport)) { let duckUsed = 0, toolCalls = 0; const gen: number[] = []; let reviewed = 0, correct = 0; for (const s of SCENARIO_NAMES) { const g = rep.scenarios[s]; duckUsed += g.duckUsed; toolCalls += g.toolCalls; reviewed += g.reviewed; correct += g.correct; for (const row of rep.perTask.filter((r) => r.scenario === s)) { gen.push(row.genTokens ?? 0); } } const taskCount = rep.scenarios.control?.tasks ?? 0; const totalRows = SCENARIO_NAMES.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0); perModel[name] = { tasks: taskCount, totalRows, reviewed, correct, accuracy: reviewed ? round((correct / reviewed) * 100, 1) : 0, duckUsed, toolCalls, avgGenTokens: round(avg(gen)), }; } return { perScenario, perModel }; }