feat: move runner to typescript, refactor

This commit is contained in:
2026-09-04 16:29:42 +05:00
parent d63cb77887
commit b9f98ea4b6
12 changed files with 649 additions and 263 deletions
@@ -1,28 +1,48 @@
const ACCEPT = "application/json, text/event-stream"; const ACCEPT = "application/json, text/event-stream";
function parseMcpResponse(text) { interface JsonRpcResponse {
result?: unknown;
error?: { code: number; message: string };
}
interface McpTool {
name: string;
description?: string;
inputSchema?: unknown;
}
type McpContentBlock = { type: string; text: string } | { type: string; [key: string]: unknown };
interface McpToolResult {
content?: McpContentBlock[];
}
function parseMcpResponse(text: string): JsonRpcResponse {
const trimmed = text.trim(); const trimmed = text.trim();
if (!trimmed) throw new Error("Empty MCP response"); if (!trimmed) throw new Error("Empty MCP response");
if (trimmed.startsWith("{")) { if (trimmed.startsWith("{")) {
return JSON.parse(trimmed); return JSON.parse(trimmed) as JsonRpcResponse;
} }
if (trimmed.includes("event:") || trimmed.includes("data:")) { if (trimmed.includes("event:") || trimmed.includes("data:")) {
let payload = ""; let payload = "";
for (const line of trimmed.split(/\r?\n/)) { for (const line of trimmed.split(/\r?\n/)) {
if (line.startsWith("data:")) payload += line.slice(5).trim(); if (line.startsWith("data:")) payload += line.slice(5).trim();
} }
if (payload.startsWith("{")) return JSON.parse(payload); if (payload.startsWith("{")) return JSON.parse(payload) as JsonRpcResponse;
} }
throw new Error(`Unrecognized MCP response format: ${trimmed.slice(0, 200)}`); throw new Error(`Unrecognized MCP response format: ${trimmed.slice(0, 200)}`);
} }
export class McpClient { export class McpClient {
constructor(url) { private url: string;
private seq: number;
constructor(url: string) {
this.url = url; this.url = url;
this.seq = 0; this.seq = 0;
} }
async request(method, params = {}) { async request(method: string, params: Record<string, unknown> = {}): Promise<JsonRpcResponse> {
this.seq += 1; this.seq += 1;
const res = await fetch(this.url, { const res = await fetch(this.url, {
method: "POST", method: "POST",
@@ -37,18 +57,18 @@ export class McpClient {
return rpc; return rpc;
} }
async listTools() { async listTools(): Promise<McpTool[]> {
const rpc = await this.request("tools/list"); const rpc = await this.request("tools/list");
return rpc.result?.tools ?? []; return (rpc.result as { tools?: McpTool[] })?.tools ?? [];
} }
async callTool(name, args = {}) { async callTool(name: string, args: Record<string, unknown> = {}): Promise<McpToolResult> {
const rpc = await this.request("tools/call", { name, arguments: args }); const rpc = await this.request("tools/call", { name, arguments: args });
return rpc.result; return rpc.result as McpToolResult;
} }
} }
export function textContentFrom(result) { export function textContentFrom(result: McpToolResult | null | undefined): string {
if (!result?.content) return ""; if (!result?.content) return "";
return result.content return result.content
.map((b) => (b.type === "text" ? b.text : JSON.stringify(b))) .map((b) => (b.type === "text" ? b.text : JSON.stringify(b)))
@@ -1,40 +1,113 @@
interface OllamaConstructorOpts {
url?: string;
timeoutMs?: number;
idleTimeoutMs?: number | null;
maxTotalMs?: number | null;
}
interface PingResult {
ok: boolean;
models: string[];
}
export type OllamaToolCall = {
function: { name: string; arguments: string };
};
export type OllamaToolDefinition = {
type: "function";
function: {
name: string;
description: string;
parameters: {
type: "object";
properties?: Record<string, unknown>;
[key: string]: unknown;
};
};
};
export type OllamaMessage = {
role: string;
content: string;
tool_calls?: OllamaToolCall[];
};
export type OllamaChunk = {
token: string | null;
content: string;
type: "content" | "tool_calls";
toolCalls?: OllamaToolCall[];
};
export interface ChatStreamOpts {
model: string;
messages: OllamaMessage[];
tools?: unknown[];
temperature?: number;
numCtx?: number;
timeoutMs?: number;
maxTotalMs?: number;
onChunk?: ((chunk: OllamaChunk) => void) | null;
}
export interface OllamaChatResult {
role: "assistant";
content: string;
toolCalls: OllamaToolCall[];
promptEvalCount: number;
evalCount: number;
raw: { stream: boolean };
}
export interface OllamaError extends Error {
partialContent?: string;
partialToolCalls?: OllamaToolCall[];
partialPrompt?: number;
partialGen?: number;
}
const DEFAULT_URL = "http://localhost:11434"; const DEFAULT_URL = "http://localhost:11434";
export class Ollama { export class Ollama {
constructor({ url = DEFAULT_URL, timeoutMs = 600000, idleTimeoutMs = null, maxTotalMs = null } = {}) { private url: string;
private timeoutMs: number;
private idleTimeoutMs: number;
private maxTotalMs: number;
constructor({ url = DEFAULT_URL, timeoutMs = 600000, idleTimeoutMs = null, maxTotalMs = null }: OllamaConstructorOpts = {}) {
this.url = url.replace(/\/$/, ""); this.url = url.replace(/\/$/, "");
this.timeoutMs = timeoutMs; this.timeoutMs = timeoutMs;
// Dynamic timeout: if set, the timer resets on every new token; the request
// only fails if this much time passes with NO progress (no new token).
this.idleTimeoutMs = idleTimeoutMs ?? timeoutMs; this.idleTimeoutMs = idleTimeoutMs ?? timeoutMs;
// Absolute hard cap on one request's lifetime, even if tokens keep flowing
// (guards against a model looping/chatting forever). Defaults to 5 minutes.
this.maxTotalMs = maxTotalMs ?? 5 * 60_000; this.maxTotalMs = maxTotalMs ?? 5 * 60_000;
} }
async ping({ timeoutMs = 30000 } = {}) { async ping({ timeoutMs = 30000 } = {}): Promise<PingResult> {
if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, model: process.env.OLLAMA_MODEL }; if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, models: [process.env.OLLAMA_MODEL ?? ""] };
const res = await fetch(`${this.url}/api/tags`, { signal: AbortSignal.timeout(timeoutMs) }); const res = await fetch(`${this.url}/api/tags`, { signal: AbortSignal.timeout(timeoutMs) });
if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`); if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`);
const data = await res.json(); const data = await res.json() as { models?: Array<{ name: string }> };
const models = (data.models ?? []).map((m) => m.name); const models = (data.models ?? []).map((m) => m.name);
return { ok: true, models }; return { ok: true, models };
} }
/** async chatStream({
* Streaming chat with a dynamic (progress-based) timeout. model,
* The timeout only fires when `timeoutMs` passes with no new token/chunk. messages,
* Optionally calls onChunk({token, content, type, toolCalls}) as data arrives. tools,
* Returns the same shape as the old non-streaming chat(). temperature = 0,
*/ numCtx = 8192,
async chatStream({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs, maxTotalMs, onChunk }) { timeoutMs,
const body = { maxTotalMs,
onChunk,
}: ChatStreamOpts): Promise<OllamaChatResult> {
const body: Record<string, unknown> = {
model, model,
messages, messages,
stream: true, stream: true,
options: { temperature }, options: { temperature },
}; };
if (numCtx) body.options.num_ctx = numCtx; if (numCtx) (body.options as Record<string, unknown>).num_ctx = numCtx;
if (tools && tools.length) body.tools = tools; if (tools && tools.length) body.tools = tools;
const limitMs = timeoutMs ?? this.idleTimeoutMs; const limitMs = timeoutMs ?? this.idleTimeoutMs;
@@ -60,20 +133,20 @@ export class Ollama {
const decoder = new TextDecoder(); const decoder = new TextDecoder();
let buffer = ""; let buffer = "";
let content = ""; let content = "";
let toolCalls = []; let toolCalls: OllamaToolCall[] = [];
let promptEvalCount = 0; let promptEvalCount = 0;
let evalCount = 0; let evalCount = 0;
let lastActivity = Date.now(); let lastActivity = Date.now();
const guardInterval = setInterval(() => { const guardInterval = setInterval(() => {
if (Date.now() - startedAt > totalCapMs) { if (Date.now() - startedAt > totalCapMs) {
const err = new Error(`Request exceeded hard cap of ${totalCapMs}ms; aborting`); const err = new Error(`Request exceeded hard cap of ${totalCapMs}ms; aborting`) as OllamaError;
err.name = "TimeoutError"; err.name = "TimeoutError";
controller.abort(err); controller.abort(err);
return; return;
} }
if (Date.now() - lastActivity > limitMs) { if (Date.now() - lastActivity > limitMs) {
const err = new Error(`No progress from Ollama for ${limitMs}ms; aborting`); const err = new Error(`No progress from Ollama for ${limitMs}ms; aborting`) as OllamaError;
err.name = "TimeoutError"; err.name = "TimeoutError";
controller.abort(err); controller.abort(err);
} }
@@ -81,15 +154,13 @@ export class Ollama {
try { try {
for (;;) { for (;;) {
let chunk; let chunk: ReadableStreamReadResult<Uint8Array>;
try { try {
chunk = await reader.read(); chunk = await reader.read();
} catch (readErr) { } catch (readErr) {
// Abort / network drop mid-stream. Surface whatever we got so the
// caller can tell "slowly progressing" from "hung".
const e = new Error( const e = new Error(
`Aborted mid-stream after ${content.length} chars (toolCalls=${toolCalls.length}): ${readErr.message}` `Aborted mid-stream after ${content.length} chars (toolCalls=${toolCalls.length}): ${(readErr as Error).message}`
); ) as OllamaError;
e.name = "TimeoutError"; e.name = "TimeoutError";
e.partialContent = content; e.partialContent = content;
e.partialToolCalls = toolCalls; e.partialToolCalls = toolCalls;
@@ -101,27 +172,27 @@ export class Ollama {
if (done) break; if (done) break;
lastActivity = Date.now(); lastActivity = Date.now();
buffer += decoder.decode(value, { stream: true }); buffer += decoder.decode(value, { stream: true });
let idx; let idx: number;
while ((idx = buffer.indexOf("\n")) !== -1) { while ((idx = buffer.indexOf("\n")) !== -1) {
const line = buffer.slice(0, idx).trim(); const line = buffer.slice(0, idx).trim();
buffer = buffer.slice(idx + 1); buffer = buffer.slice(idx + 1);
if (!line) continue; if (!line) continue;
let obj; let obj: Record<string, unknown>;
try { try {
obj = JSON.parse(line); obj = JSON.parse(line);
} catch { } catch {
continue; continue;
} }
if (obj.prompt_eval_count != null) promptEvalCount = obj.prompt_eval_count; if (obj.prompt_eval_count != null) promptEvalCount = obj.prompt_eval_count as number;
if (obj.eval_count != null) evalCount = obj.eval_count; if (obj.eval_count != null) evalCount = obj.eval_count as number;
const msg = obj.message ?? {}; const msg = (obj.message ?? {}) as Record<string, unknown>;
if (msg.content) { if (msg.content) {
content += msg.content; content += msg.content as string;
onChunk?.({ token: msg.content, content, type: "content" }); onChunk?.({ token: msg.content as string, content, type: "content" });
} }
if (msg.tool_calls && msg.tool_calls.length) { if (msg.tool_calls && (msg.tool_calls as unknown[]).length) {
toolCalls = msg.tool_calls; toolCalls = msg.tool_calls as OllamaToolCall[];
onChunk?.({ token: null, content, type: "tool_calls", toolCalls: msg.tool_calls }); onChunk?.({ token: null, content, type: "tool_calls", toolCalls });
} }
} }
} }
@@ -139,7 +210,7 @@ export class Ollama {
}; };
} }
async chat({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs }) { async chat({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs }: ChatStreamOpts): Promise<OllamaChatResult> {
return this.chatStream({ return this.chatStream({
model, model,
messages, messages,
@@ -151,7 +222,7 @@ export class Ollama {
}); });
} }
async warmup({ model, text = "say OK", timeoutMs = 60000 }) { async warmup({ model, text = "say OK", timeoutMs = 60000 }: { model: string; text?: string; timeoutMs?: number }): Promise<Partial<OllamaChatResult> & { error?: string }> {
try { try {
return await this.chatStream({ return await this.chatStream({
model, model,
@@ -159,15 +230,15 @@ export class Ollama {
timeoutMs, timeoutMs,
}); });
} catch (err) { } catch (err) {
return { error: err.message }; return { error: (err as Error).message };
} }
} }
} }
export async function listModels() { export async function listModels(): Promise<string[]> {
const o = new Ollama(); const o = new Ollama();
const res = await fetch(`${o.url}/api/tags`); const res = await fetch(`${o["url"]}/api/tags`);
if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status})`); if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status})`);
const data = await res.json(); const data = await res.json() as { models?: Array<{ name: string }> };
return (data.models ?? []).map((m) => m.name); return (data.models ?? []).map((m) => m.name);
} }
+10 -3
View File
@@ -3,12 +3,19 @@
"version": "1.0.0", "version": "1.0.0",
"description": "MCP rubber-duck evaluation harness", "description": "MCP rubber-duck evaluation harness",
"scripts": { "scripts": {
"run": "node src/run.mjs", "run": "tsx src/run.ts",
"report-html": "node src/report-html.mjs" "review": "tsx src/review.ts",
"report-html": "tsx src/report-html.ts",
"typecheck": "tsc --noEmit"
}, },
"license": "ISC", "license": "ISC",
"type": "module", "type": "module",
"dependencies": { "dependencies": {
"@modelcontextprotocol/client": "^2.0.0" "@duck/types": "workspace:*"
},
"devDependencies": {
"@types/node": "^20",
"tsx": "^4",
"typescript": "^5.9.3"
} }
} }
@@ -1,51 +1,55 @@
import { readFile, writeFile, mkdir } from "node:fs/promises"; import { readFile, writeFile, mkdir } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import type { ReportRoot, ScenarioName, ScenarioSummary, AggregateScenario, PerTaskRow } from "@duck/types";
import { SCENARIO_NAMES } from "@duck/types";
const SCENARIOS = ["control", "thinking", "blind", "mentor"]; const SCENARIO_LABELS: Record<ScenarioName, string> = {
const SCENARIO_LABELS = {
control: "Без утки (контроль)", control: "Без утки (контроль)",
thinking: "Думать вслух", thinking: "Думать вслух",
blind: "Слепая утка", blind: "Слепая утка",
mentor: "Утка-помощник", mentor: "Утка-помощник",
}; };
const SCENARIO_DESCS = { const SCENARIO_DESCS: Record<ScenarioName, string> = {
control: "Модель решает задачу напрямую, без инструментов.", control: "Модель решает задачу напрямую, без инструментов.",
thinking: "Модель выписывает рассуждения вслух, без дука.", thinking: "Модель выписывает рассуждения вслух, без дука.",
blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.", blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.",
mentor: "Модель работает в паре, зная, что ответ будет только «quack».", mentor: "Модель работает в паре, зная, что ответ будет только «quack».",
}; };
function esc(s) { function esc(s: unknown): string {
return String(s ?? "") return String(s ?? "")
.replace(/&/g, "&amp;") .replace(/&/g, "&amp;")
.replace(/</g, "&lt;") .replace(/</g, "&lt;")
.replace(/>/g, "&gt;"); .replace(/>/g, "&gt;");
} }
function nl2br(s) { function nl2br(s: string): string {
return esc(s).replace(/\n/g, "<br>"); return esc(s).replace(/\n/g, "<br>");
} }
function pctColor(p) { function pctColor(p: number): string {
return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444"; return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444";
} }
function modelShort(name) { function modelShort(name: string): string {
const base = String(name).split(/:/)[0]; const base = String(name).split(/:/)[0];
const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : ""; const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : "";
return esc(base + ver); return esc(base + ver);
} }
function summaryTable(report) { interface RowHtmlItem extends PerTaskRow {
model: string;
}
function summaryTable(report: ReportRoot): string {
const perModel = report.aggregate.perModel; const perModel = report.aggregate.perModel;
const models = Object.keys(perModel); const models = Object.keys(perModel);
const rows = models const rows = models
.map((m) => { .map((m) => {
const g = perModel[m]; const g = perModel[m];
const scen = report.models[m].scenarios; const scen = report.models[m].scenarios;
const acc = (s) => scen[s].accuracy; const acc = (s: ScenarioName) => scen[s].accuracy;
const row = ` const row = `
<td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td> <td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td>
<td class="pm-num" data-sc="thinking" style="color:${pctColor(acc("thinking"))}">${acc("thinking")}%</td> <td class="pm-num" data-sc="thinking" style="color:${pctColor(acc("thinking"))}">${acc("thinking")}%</td>
@@ -86,7 +90,7 @@ function summaryTable(report) {
</div>`; </div>`;
} }
function scenarioCard(key, agg) { function scenarioCard(key: ScenarioName, agg: AggregateScenario): string {
const pct = agg.accuracy; const pct = agg.accuracy;
const color = pctColor(pct); const color = pctColor(pct);
const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0; const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0;
@@ -122,7 +126,7 @@ function scenarioCard(key, agg) {
</div>`; </div>`;
} }
function rowHtml(r) { function rowHtml(r: RowHtmlItem): string {
const pending = r.correct === null || r.correct === undefined; const pending = r.correct === null || r.correct === undefined;
const cls = pending ? "pend" : r.correct ? "ok" : "no"; const cls = pending ? "pend" : r.correct ? "ok" : "no";
const result = pending ? "?" : r.correct ? "✓" : "✗"; const result = pending ? "?" : r.correct ? "✓" : "✗";
@@ -141,10 +145,10 @@ function rowHtml(r) {
</tr>`; </tr>`;
} }
function promptBlock(report) { function promptBlock(report: ReportRoot): string {
const p = report.prompts || {}; const p = report.prompts;
const items = SCENARIOS const items = (SCENARIO_NAMES as readonly ScenarioName[])
.filter((k) => p[k]) .filter((k): k is ScenarioName => !!p[k])
.map( .map(
(k) => ` (k) => `
<div class="prompt-item"> <div class="prompt-item">
@@ -161,15 +165,15 @@ function promptBlock(report) {
</details>`; </details>`;
} }
export function buildHtml(report) { export function buildHtml(report: ReportRoot): string {
const perTask = Object.entries(report.models).flatMap(([model, rep]) => const perTask: RowHtmlItem[] = Object.entries(report.models).flatMap(([model, rep]) =>
rep.perTask.map((r) => ({ ...r, model })) rep.perTask.map((r) => ({ ...r, model }))
); );
const rows = perTask.map(rowHtml).join(""); const rows = perTask.map(rowHtml).join("");
const models = Object.keys(report.aggregate.perModel); const models = Object.keys(report.aggregate.perModel);
let totalPending = 0, totalReviewed = 0; let totalPending = 0, totalReviewed = 0;
for (const rep of Object.values(report.models)) { for (const rep of Object.values(report.models)) {
for (const s of SCENARIOS) { for (const s of SCENARIO_NAMES) {
if (rep.scenarios[s]) { if (rep.scenarios[s]) {
totalReviewed += rep.scenarios[s].reviewed || 0; totalReviewed += rep.scenarios[s].reviewed || 0;
totalPending += rep.scenarios[s].pending || 0; totalPending += rep.scenarios[s].pending || 0;
@@ -182,8 +186,8 @@ export function buildHtml(report) {
const modelOptions = models const modelOptions = models
.map((m) => `<option value="${esc(m)}">${modelShort(m)}</option>`) .map((m) => `<option value="${esc(m)}">${modelShort(m)}</option>`)
.join(""); .join("");
const aggCards = SCENARIOS const aggCards = (SCENARIO_NAMES as readonly ScenarioName[])
.filter((k) => report.aggregate.perScenario[k]) .filter((k): k is ScenarioName => !!report.aggregate.perScenario[k])
.map((k) => scenarioCard(k, report.aggregate.perScenario[k])) .map((k) => scenarioCard(k, report.aggregate.perScenario[k]))
.join(""); .join("");
@@ -353,9 +357,9 @@ export function buildHtml(report) {
</html>`; </html>`;
} }
async function main() { async function main(): Promise<void> {
const args = process.argv.slice(2); const args = process.argv.slice(2);
let inFile = null, outFile = "out/report.html"; let inFile: string | null = null, outFile = "out/report.html";
for (let i = 0; i < args.length; i += 1) { for (let i = 0; i < args.length; i += 1) {
const a = args[i]; const a = args[i];
const next = () => args[i + 1]; const next = () => args[i + 1];
@@ -371,7 +375,7 @@ async function main() {
inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base)); inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base));
} }
const report = JSON.parse(await readFile(inFile, "utf8")); const report: ReportRoot = JSON.parse(await readFile(inFile, "utf8"));
const html = buildHtml(report); const html = buildHtml(report);
await mkdir(path.dirname(outFile), { recursive: true }); await mkdir(path.dirname(outFile), { recursive: true });
await writeFile(outFile, html, "utf8"); await writeFile(outFile, html, "utf8");
@@ -1,16 +1,28 @@
function round(x, d = 2) { import type { ScenarioName, ScenarioSummary, PerTaskRow, ModelReport, Aggregate, AggregateScenario, AggregateModel } from "@duck/types";
import { SCENARIO_NAMES } from "@duck/types";
function round(x: number, d = 2): number {
if (!Number.isFinite(x)) return x; if (!Number.isFinite(x)) return x;
const f = 10 ** d; const f = 10 ** d;
return Math.round(x * f) / f; return Math.round(x * f) / f;
} }
function avg(arr) { function avg(arr: number[]): number {
if (!arr.length) return 0; if (!arr.length) return 0;
return arr.reduce((a, b) => a + b, 0) / arr.length; return arr.reduce((a, b) => a + b, 0) / arr.length;
} }
export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "thinking", "blind", "mentor"]; export interface BuildReportOpts {
const byScenario = {}; model: string;
mcpUrl: string;
rows: PerTaskRow[];
meta?: Record<string, unknown>;
prompts?: Partial<Record<ScenarioName, string>>;
}
export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: BuildReportOpts): ModelReport {
const scenarios = SCENARIO_NAMES;
const byScenario: Record<string, { total: number; correct: number; reviewed: number; duckUsed: number; toolCalls: number; promptTokens: number[]; genTokens: number[]; duckTokens: number[]; rows: PerTaskRow[] }> = {};
for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
for (const row of rows) { for (const row of rows) {
@@ -29,10 +41,9 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
byScenario[s].rows.push(row); byScenario[s].rows.push(row);
} }
const summaries = {}; const summaries: Record<ScenarioName, ScenarioSummary> = {} as Record<ScenarioName, ScenarioSummary>;
for (const s of scenarios) { for (const s of scenarios) {
const g = byScenario[s]; const g = byScenario[s];
const denom = g.reviewed || 1;
summaries[s] = { summaries[s] = {
tasks: g.total, tasks: g.total,
reviewed: g.reviewed, reviewed: g.reviewed,
@@ -53,7 +64,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
model, model,
mcpUrl, mcpUrl,
scenarios: summaries, scenarios: summaries,
prompts, prompts: prompts as Record<ScenarioName, string>,
perTask: rows.map((r) => ({ perTask: rows.map((r) => ({
id: r.id, id: r.id,
scenario: r.scenario, scenario: r.scenario,
@@ -71,13 +82,11 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
}; };
} }
const SCENARIOS = ["control", "thinking", "blind", "mentor"]; export function buildAggregate(modelsReport: Record<string, ModelReport>): Aggregate {
const perScenario: Record<ScenarioName, AggregateScenario> = {} as Record<ScenarioName, AggregateScenario>;
export function buildAggregate(modelsReport) { for (const s of SCENARIO_NAMES) {
const perScenario = {};
for (const s of SCENARIOS) {
let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0; let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0;
const byModel = {}; const byModel: Record<string, AggregateScenario["byModel"][string]> = {};
for (const [name, rep] of Object.entries(modelsReport)) { for (const [name, rep] of Object.entries(modelsReport)) {
const g = rep.scenarios[s]; const g = rep.scenarios[s];
if (!g) continue; if (!g) continue;
@@ -107,12 +116,12 @@ export function buildAggregate(modelsReport) {
}; };
} }
const perModel = {}; const perModel: Record<string, AggregateModel> = {};
for (const [name, rep] of Object.entries(modelsReport)) { for (const [name, rep] of Object.entries(modelsReport)) {
let duckUsed = 0, toolCalls = 0; let duckUsed = 0, toolCalls = 0;
const gen = []; const gen: number[] = [];
let reviewed = 0, correct = 0; let reviewed = 0, correct = 0;
for (const s of SCENARIOS) { for (const s of SCENARIO_NAMES) {
const g = rep.scenarios[s]; const g = rep.scenarios[s];
duckUsed += g.duckUsed; duckUsed += g.duckUsed;
toolCalls += g.toolCalls; toolCalls += g.toolCalls;
@@ -123,7 +132,7 @@ export function buildAggregate(modelsReport) {
} }
} }
const taskCount = rep.scenarios.control?.tasks ?? 0; const taskCount = rep.scenarios.control?.tasks ?? 0;
const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0); const totalRows = SCENARIO_NAMES.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0);
perModel[name] = { perModel[name] = {
tasks: taskCount, tasks: taskCount,
totalRows, totalRows,
@@ -1,24 +1,36 @@
import { readFile, writeFile, mkdir } from "node:fs/promises"; import { readFile, writeFile, mkdir } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import readline from "node:readline"; import readline from "node:readline";
import { buildReport, buildAggregate } from "./report.mjs"; import type { ReportRoot, ScenarioName, PerTaskRow } from "@duck/types";
import { SCENARIO_NAMES } from "@duck/types";
import { buildReport, buildAggregate } from "./report.js";
const SCENARIO_LABELS = { const SCENARIO_LABELS: Record<ScenarioName, string> = {
control: "Без утки (контроль)", control: "Без утки (контроль)",
thinking: "Думать вслух", thinking: "Думать вслух",
blind: "Слепая утка", blind: "Слепая утка",
mentor: "Утка-помощник", mentor: "Утка-помощник",
}; };
function esc(s) { interface ReviewEntry {
return String(s ?? "") model: string;
.replace(/&/g, "&amp;") taskId: string;
.replace(/</g, "&lt;") scenario: ScenarioName;
.replace(/>/g, "&gt;"); row: PerTaskRow;
} }
function parseArgs(argv) { interface ReviewArgs {
const args = { report: string | null;
out: string | null;
reviewFile: string | null;
model: string | null;
scenario: string | null;
limit: number | null;
reAsk?: boolean;
}
function parseArgs(argv: string[]): ReviewArgs {
const args: ReviewArgs = {
report: null, report: null,
out: null, out: null,
reviewFile: null, reviewFile: null,
@@ -40,22 +52,22 @@ function parseArgs(argv) {
return args; return args;
} }
async function main() { async function main(): Promise<void> {
const args = parseArgs(process.argv.slice(2)); const args = parseArgs(process.argv.slice(2));
const reportFile = args.report || path.join("out", "report.json"); const reportFile = args.report || path.join("out", "report.json");
const outFile = args.out || path.join("out", "report.reviewed.json"); const outFile = args.out || path.join("out", "report.reviewed.json");
const reviewFile = args.reviewFile || path.join("out", "review.json"); const reviewFile = args.reviewFile || path.join("out", "review.json");
const report = JSON.parse(await readFile(reportFile, "utf8")); const report: ReportRoot = JSON.parse(await readFile(reportFile, "utf8"));
let decisions = {}; let decisions: Record<string, boolean> = {};
try { try {
decisions = JSON.parse(await readFile(reviewFile, "utf8")); decisions = JSON.parse(await readFile(reviewFile, "utf8"));
} catch { } catch {
/* no prior review */ /* no prior review */
} }
const entries = []; const entries: ReviewEntry[] = [];
for (const [model, rep] of Object.entries(report.models)) { for (const [model, rep] of Object.entries(report.models)) {
for (const row of rep.perTask) { for (const row of rep.perTask) {
if (args.model && model !== args.model) continue; if (args.model && model !== args.model) continue;
@@ -65,13 +77,13 @@ async function main() {
} }
if (args.limit && entries.length > args.limit) entries.length = args.limit; if (args.limit && entries.length > args.limit) entries.length = args.limit;
const key = (e) => `${e.model}|${e.taskId}|${e.scenario}`; const key = (e: ReviewEntry) => `${e.model}|${e.taskId}|${e.scenario}`;
const isTTY = !!process.stdin.isTTY; const isTTY = !!process.stdin.isTTY;
let batchLines = []; let batchLines: string[] = [];
if (!isTTY) { if (!isTTY) {
const input = await new Promise((res, rej) => { const input = await new Promise<string>((res, rej) => {
let data = ""; let data = "";
process.stdin.setEncoding("utf8"); process.stdin.setEncoding("utf8");
process.stdin.on("data", (c) => (data += c)); process.stdin.on("data", (c) => (data += c));
@@ -83,17 +95,21 @@ async function main() {
} }
let batchIdx = 0; let batchIdx = 0;
const nextInput = async () => { const nextInput = async (): Promise<string> => {
if (isTTY) { if (isTTY) {
const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
const ans = await new Promise((res) => rl.question("", res)); const ans = await new Promise<string>((res) => rl.question("", res));
rl.close(); rl.close();
return ans.trim().toLowerCase(); return ans.trim().toLowerCase();
} }
return batchLines[batchIdx++] ?? ""; return batchLines[batchIdx++] ?? "";
}; };
async function askOne(e, nextInput, decisions) { let done = 0;
let skipped = 0;
let changed = 0;
async function askOne(e: ReviewEntry): Promise<number> {
const k = key(e); const k = key(e);
const prev = decisions[k]; const prev = decisions[k];
if (prev !== undefined) { if (prev !== undefined) {
@@ -115,7 +131,7 @@ async function main() {
process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `); process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `);
const a = (await nextInput()) || ""; const a = (await nextInput()) || "";
let decision = null; let decision: boolean | null = null;
if (a === "y" || a === "д") decision = true; if (a === "y" || a === "д") decision = true;
else if (a === "n" || a === "н") decision = false; else if (a === "n" || a === "н") decision = false;
else if (a === "" && cur) decision = e.row.correct; else if (a === "" && cur) decision = e.row.correct;
@@ -131,10 +147,6 @@ async function main() {
return 1; return 1;
} }
let done = 0;
let skipped = 0;
let changed = 0;
const reAsk = !!args.reAsk; const reAsk = !!args.reAsk;
for (const e of entries) { for (const e of entries) {
@@ -142,7 +154,7 @@ async function main() {
const prev = decisions[k]; const prev = decisions[k];
if (prev === undefined || reAsk) { if (prev === undefined || reAsk) {
done += await askOne(e, nextInput, decisions); done += await askOne(e);
continue; continue;
} }
@@ -161,7 +173,6 @@ async function main() {
scenarios: buildReport({ scenarios: buildReport({
model, model,
mcpUrl: report.mcpUrl, mcpUrl: report.mcpUrl,
tasks: [],
rows: rep.perTask.map((r) => ({ rows: rep.perTask.map((r) => ({
id: r.id, id: r.id,
scenario: r.scenario, scenario: r.scenario,
@@ -192,7 +203,7 @@ async function main() {
const scen = rep.scenarios; const scen = rep.scenarios;
console.log( console.log(
`${model} | ` + `${model} | ` +
["control", "thinking", "blind", "mentor"] SCENARIO_NAMES
.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`) .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`)
.join(" ") + .join(" ") +
` всего rev=${g.reviewed}` ` всего rev=${g.reviewed}`
+43 -33
View File
@@ -1,13 +1,15 @@
import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises"; import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
import { Ollama } from "../lib/ollama.mjs"; import type { ScenarioName, ReportRoot, PerTaskRow, Task } from "@duck/types";
import { McpClient } from "../lib/mcpClient.mjs"; import { SCENARIO_NAMES } from "@duck/types";
import { runScenario } from "./runner.mjs"; import { Ollama } from "../lib/ollama.js";
import { PROMPTS } from "./runner.mjs"; import type { OllamaError } from "../lib/ollama.js";
import { buildReport, buildAggregate } from "./report.mjs"; import { McpClient } from "../lib/mcpClient.js";
import { runScenario, PROMPTS } from "./runner.js";
import { buildReport, buildAggregate } from "./report.js";
let logTarget = null; let logTarget: string | null = null;
async function log(msg) { async function log(msg: string): Promise<void> {
const line = `[${new Date().toISOString()}] ${msg}`; const line = `[${new Date().toISOString()}] ${msg}`;
console.log(line); console.log(line);
if (logTarget) { if (logTarget) {
@@ -19,8 +21,6 @@ async function log(msg) {
} }
} }
const SCENARIOS = ["control", "thinking", "blind", "mentor"];
const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:1.7b", "qwen3:4b", "granite4.1:3b", "phi4-mini:3.8b"]; const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:1.7b", "qwen3:4b", "granite4.1:3b", "phi4-mini:3.8b"];
const DEFAULT_MCP_URL = "https://rubber-duck-mcp.vercel.app/api/mcp"; const DEFAULT_MCP_URL = "https://rubber-duck-mcp.vercel.app/api/mcp";
const REQUEST_TIMEOUT_MS = 60_000; const REQUEST_TIMEOUT_MS = 60_000;
@@ -28,9 +28,9 @@ const MAX_ATTEMPTS = 3;
const PAUSE_BETWEEN_TASKS_MS = 2_000; const PAUSE_BETWEEN_TASKS_MS = 2_000;
const PAUSE_BETWEEN_RETRIES_MS = 10_000; const PAUSE_BETWEEN_RETRIES_MS = 10_000;
function classifyError(err) { function classifyError(err: unknown): string {
const name = err?.name ?? ""; const name = (err as OllamaError)?.name ?? "";
const msg = String(err?.message ?? ""); const msg = String((err as OllamaError)?.message ?? "");
const low = `${name} ${msg}`.toLowerCase(); const low = `${name} ${msg}`.toLowerCase();
if (name === "TimeoutError" || low.includes("timeout") || low.includes("aborted")) return "timeout"; if (name === "TimeoutError" || low.includes("timeout") || low.includes("aborted")) return "timeout";
if (low.includes("fetch failed") || low.includes("connect") || low.includes("etimedout")) return "network"; if (low.includes("fetch failed") || low.includes("connect") || low.includes("etimedout")) return "network";
@@ -39,10 +39,20 @@ function classifyError(err) {
return "other"; return "other";
} }
function parseArgs(argv) { interface Args {
const args = { models: string[];
scenarios: ScenarioName[];
limit: number;
model: string | null;
mcpUrl: string | null;
tasksFile: string | null;
out: string | null;
}
function parseArgs(argv: string[]): Args {
const args: Args = {
models: [], models: [],
scenarios: SCENARIOS, scenarios: [...SCENARIO_NAMES],
limit: Infinity, limit: Infinity,
model: process.env.OLLAMA_MODEL || null, model: process.env.OLLAMA_MODEL || null,
mcpUrl: null, mcpUrl: null,
@@ -55,7 +65,7 @@ function parseArgs(argv) {
if (a === "--models") args.models = next().split(",").map((s) => s.trim()).filter(Boolean), i += 1; if (a === "--models") args.models = next().split(",").map((s) => s.trim()).filter(Boolean), i += 1;
else if (a === "--model") args.model = next(), i += 1; else if (a === "--model") args.model = next(), i += 1;
else if (a === "--mcp-url") args.mcpUrl = next(), i += 1; else if (a === "--mcp-url") args.mcpUrl = next(), i += 1;
else if (a === "--scenarios") args.scenarios = next().split(","), i += 1; else if (a === "--scenarios") args.scenarios = next().split(",") as ScenarioName[], i += 1;
else if (a === "--limit") args.limit = Number(next()), i += 1; else if (a === "--limit") args.limit = Number(next()), i += 1;
else if (a === "--tasks") args.tasksFile = next(), i += 1; else if (a === "--tasks") args.tasksFile = next(), i += 1;
else if (a === "--out") args.out = next(), i += 1; else if (a === "--out") args.out = next(), i += 1;
@@ -63,26 +73,26 @@ function parseArgs(argv) {
return args; return args;
} }
async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { async function runOneModel({ model, ollama, mcp, tasks, scenarios }: { model: string; ollama: Ollama; mcp: McpClient; tasks: Task[]; scenarios: ScenarioName[] }): Promise<PerTaskRow[]> {
const rows = []; const rows: PerTaskRow[] = [];
for (const task of tasks) { for (const task of tasks) {
for (const scenario of scenarios) { for (const scenario of scenarios) {
let r = null; let r = null;
let lastErr = null; let lastErr: OllamaError | null = null;
const t0 = Date.now(); const t0 = Date.now();
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
try { try {
r = await runScenario({ scenario, ollama, model, task: task.question, mcp }); r = await runScenario({ scenario, ollama, model, task: task.question, mcp });
break; break;
} catch (err) { } catch (err) {
lastErr = err; lastErr = err as OllamaError;
const ms = Date.now() - t0; const ms = Date.now() - t0;
await log(` [${task.id}/${scenario}] attempt ${attempt} FAILED (${ms}ms): ${err.message}`); await log(` [${task.id}/${scenario}] attempt ${attempt} FAILED (${ms}ms): ${(err as Error).message}`);
if (err.partialContent != null || err.partialToolCalls != null) { if (lastErr.partialContent != null || lastErr.partialToolCalls != null) {
await log( await log(
` partial before abort: chars=${err.partialContent?.length ?? 0} ` + ` partial before abort: chars=${lastErr.partialContent?.length ?? 0} ` +
`toolCalls=${err.partialToolCalls?.length ?? 0} ` + `toolCalls=${lastErr.partialToolCalls?.length ?? 0} ` +
`gen=${err.partialGen ?? 0}` `gen=${lastErr.partialGen ?? 0}`
); );
} }
if (attempt < MAX_ATTEMPTS) await new Promise((res) => setTimeout(res, PAUSE_BETWEEN_RETRIES_MS)); if (attempt < MAX_ATTEMPTS) await new Promise((res) => setTimeout(res, PAUSE_BETWEEN_RETRIES_MS));
@@ -121,7 +131,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
return rows; return rows;
} }
async function main() { async function main(): Promise<void> {
const args = parseArgs(process.argv.slice(2)); const args = parseArgs(process.argv.slice(2));
let models = args.models; let models = args.models;
@@ -131,11 +141,11 @@ async function main() {
const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL; const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL;
const tasksFile = args.tasksFile || path.join("data", "tasks.json"); const tasksFile = args.tasksFile || path.join("data", "tasks.json");
const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8")); const tasksRaw: Task[] = JSON.parse(await readFile(tasksFile, "utf8"));
const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit); const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit);
const out = args.out || path.join("out", "report.json"); const out = args.out || path.join("out", "report.json");
const logFile = args.logTarget || path.join(path.dirname(out), "run.log"); const logFile = path.join(path.dirname(out), "run.log");
logTarget = logFile; logTarget = logFile;
await mkdir(path.dirname(logFile), { recursive: true }); await mkdir(path.dirname(logFile), { recursive: true });
await appendFile(logFile, "", "utf8").catch(() => {}); await appendFile(logFile, "", "utf8").catch(() => {});
@@ -156,7 +166,7 @@ async function main() {
const mcpTools = await mcp.listTools(); const mcpTools = await mcp.listTools();
await log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`); await log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`);
const modelsReport = {}; const modelsReport: ReportRoot["models"] = {};
for (const model of models) { for (const model of models) {
const tStart = Date.now(); const tStart = Date.now();
await log(`==== Running model: ${model} ====`); await log(`==== Running model: ${model} ====`);
@@ -168,18 +178,18 @@ async function main() {
await log(` warmup OK (${warmMs}ms, gen=${warm?.evalCount ?? 0})`); await log(` warmup OK (${warmMs}ms, gen=${warm?.evalCount ?? 0})`);
} }
const rows = await runOneModel({ model, ollama, mcp, tasks, scenarios: args.scenarios }); const rows = await runOneModel({ model, ollama, mcp, tasks, scenarios: args.scenarios });
const rep = buildReport({ model, mcpUrl, tasks, rows }); const rep = buildReport({ model, mcpUrl, rows });
modelsReport[model] = { modelsReport[model] = {
model, model,
scenarios: rep.scenarios, scenarios: rep.scenarios,
perTask: rep.perTask, perTask: rep.perTask,
}; } as ReportRoot["models"][string];
await log(`==== Done ${model} in ${Math.round((Date.now() - tStart) / 1000)}s ====`); await log(`==== Done ${model} in ${Math.round((Date.now() - tStart) / 1000)}s ====`);
} }
const aggregate = buildAggregate(modelsReport); const aggregate = buildAggregate(modelsReport);
const report = { const report: ReportRoot = {
generatedAt: new Date().toISOString(), generatedAt: new Date().toISOString(),
mcpUrl, mcpUrl,
prompts: PROMPTS, prompts: PROMPTS,
@@ -196,7 +206,7 @@ async function main() {
const scen = modelsReport[modelName].scenarios; const scen = modelsReport[modelName].scenarios;
await log( await log(
`${modelName} | ` + `${modelName} | ` +
SCENARIOS.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}%`).join(" ") + SCENARIO_NAMES.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}%`).join(" ") +
` duck=${g.duckUsed} calls=${g.toolCalls} avgGen=${g.avgGenTokens}` ` duck=${g.duckUsed} calls=${g.toolCalls} avgGen=${g.avgGenTokens}`
); );
} }
@@ -1,7 +1,9 @@
import { Ollama } from "../lib/ollama.mjs"; import type { ScenarioName } from "@duck/types";
import { McpClient, textContentFrom } from "../lib/mcpClient.mjs"; import { Ollama } from "../lib/ollama.js";
import type { OllamaMessage, OllamaToolDefinition } from "../lib/ollama.js";
import { McpClient, textContentFrom } from "../lib/mcpClient.js";
const DUCK_TOOL = { const DUCK_TOOL: OllamaToolDefinition = {
type: "function", type: "function",
function: { function: {
name: "quack", name: "quack",
@@ -48,7 +50,7 @@ const MENTOR_SYSTEM =
"Treat it as a way to voice your thoughts out loud, not as a source of answers. " + "Treat it as a way to voice your thoughts out loud, not as a source of answers. " +
"After the tool's reply, give the user a clear, concrete final answer to the question."; "After the tool's reply, give the user a clear, concrete final answer to the question.";
export const PROMPTS = { export const PROMPTS: Record<ScenarioName, string> = {
control: CONTROL_SYSTEM, control: CONTROL_SYSTEM,
thinking: THINKING_SYSTEM, thinking: THINKING_SYSTEM,
blind: BLIND_SYSTEM, blind: BLIND_SYSTEM,
@@ -57,11 +59,22 @@ export const PROMPTS = {
const MAX_DUCK_TURNS = 6; const MAX_DUCK_TURNS = 6;
function toolDefinition() { interface RunnerResult {
response: string;
toolCalls: number;
duckUsed: boolean;
duckTokens: number;
promptTokens: number;
genTokens: number;
turns?: number;
messages: OllamaMessage[];
}
function toolDefinition(): OllamaToolDefinition {
return DUCK_TOOL; return DUCK_TOOL;
} }
async function runNoTool(ollama, model, task, system) { async function runNoTool(ollama: Ollama, model: string, task: string, system: string): Promise<RunnerResult> {
const messages = [ const messages = [
{ role: "system", content: system }, { role: "system", content: system },
{ role: "user", content: task }, { role: "user", content: task },
@@ -78,16 +91,16 @@ async function runNoTool(ollama, model, task, system) {
}; };
} }
function runControl(ollama, model, task) { function runControl(ollama: Ollama, model: string, task: string): Promise<RunnerResult> {
return runNoTool(ollama, model, task, CONTROL_SYSTEM); return runNoTool(ollama, model, task, CONTROL_SYSTEM);
} }
function runThinking(ollama, model, task) { function runThinking(ollama: Ollama, model: string, task: string): Promise<RunnerResult> {
return runNoTool(ollama, model, task, THINKING_SYSTEM); return runNoTool(ollama, model, task, THINKING_SYSTEM);
} }
async function runDuck(ollama, model, task, mcp, system, scenario) { async function runDuck(ollama: Ollama, model: string, task: string, mcp: McpClient, system: string, scenario: ScenarioName): Promise<RunnerResult> {
const messages = [ const messages: OllamaMessage[] = [
{ role: "system", content: system }, { role: "system", content: system },
{ role: "user", content: task }, { role: "user", content: task },
]; ];
@@ -107,18 +120,21 @@ async function runDuck(ollama, model, task, mcp, system, scenario) {
if (r.toolCalls && r.toolCalls.length) { if (r.toolCalls && r.toolCalls.length) {
toolCalls += r.toolCalls.length; toolCalls += r.toolCalls.length;
const assistantMsg = { role: "assistant", content: r.content || "" }; const assistantMsg: OllamaMessage = {
assistantMsg.tool_calls = r.toolCalls.map((tc) => ({ role: "assistant",
function: { name: tc.function.name, arguments: tc.function.arguments }, content: r.content || "",
})); tool_calls: r.toolCalls.map((tc) => ({
function: { name: tc.function.name, arguments: tc.function.arguments },
})),
};
messages.push(assistantMsg); messages.push(assistantMsg);
for (const tc of r.toolCalls) { for (const tc of r.toolCalls) {
if (tc.function.name === "quack") { if (tc.function.name === "quack") {
duckUsed = true; duckUsed = true;
let args = {}; let args: Record<string, unknown> = {};
try { try {
args = tc.function.arguments ? JSON.parse(tc.function.arguments) : {}; args = tc.function.arguments ? JSON.parse(tc.function.arguments) as Record<string, unknown> : {};
} catch { } catch {
args = {}; args = {};
} }
@@ -146,7 +162,6 @@ async function runDuck(ollama, model, task, mcp, system, scenario) {
}; };
} }
// loop bound hit without a final answer
const last = messages[messages.length - 1]; const last = messages[messages.length - 1];
return { return {
response: last?.content ?? "", response: last?.content ?? "",
@@ -160,7 +175,15 @@ async function runDuck(ollama, model, task, mcp, system, scenario) {
}; };
} }
export async function runScenario({ scenario, ollama, model, task, mcp }) { export interface RunScenarioOpts {
scenario: ScenarioName;
ollama: Ollama;
model: string;
task: string;
mcp: McpClient;
}
export async function runScenario({ scenario, ollama, model, task, mcp }: RunScenarioOpts): Promise<RunnerResult> {
if (scenario === "control") { if (scenario === "control") {
return runControl(ollama, model, task); return runControl(ollama, model, task);
} }
+16
View File
@@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ES2022",
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"isolatedModules": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"paths": {
"@duck/types": ["../../packages/types/src/index.ts"]
}
},
"include": ["src", "lib"]
}
+4 -3
View File
@@ -10,9 +10,10 @@
"typecheck-mcp": "pnpm --dir apps/mcp exec tsc --noEmit", "typecheck-mcp": "pnpm --dir apps/mcp exec tsc --noEmit",
"deploy-mcp": "vercel --cwd apps/mcp --prod --yes", "deploy-mcp": "vercel --cwd apps/mcp --prod --yes",
"deploy-frontend": "vercel --cwd apps/frontend --prod --yes", "deploy-frontend": "vercel --cwd apps/frontend --prod --yes",
"eval:run": "pnpm --dir apps/eval exec node src/run.mjs", "eval:run": "pnpm --dir apps/eval run run",
"eval:review": "pnpm --dir apps/eval exec node src/review.mjs", "eval:review": "pnpm --dir apps/eval run review",
"eval:html": "pnpm --dir apps/eval exec node src/report-html.mjs" "eval:html": "pnpm --dir apps/eval run report-html",
"eval:typecheck": "pnpm --dir apps/eval run typecheck"
}, },
"license": "ISC", "license": "ISC",
"devEngines": { "devEngines": {
+299 -87
View File
@@ -209,15 +209,25 @@ importers:
apps/eval: apps/eval:
dependencies: dependencies:
'@modelcontextprotocol/client': '@duck/types':
specifier: ^2.0.0 specifier: workspace:*
version: 2.0.0 version: link:../../packages/types
devDependencies:
'@types/node':
specifier: ^20
version: 20.19.43
tsx:
specifier: ^4
version: 4.23.13
typescript:
specifier: ^5.9.3
version: 5.9.3
apps/frontend: apps/frontend:
devDependencies: devDependencies:
vite: vite:
specifier: ^8.0.11 specifier: ^8.0.11
version: 8.2.2(@types/node@20.19.43) version: 8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(tsx@4.23.13)
apps/mcp: apps/mcp:
dependencies: dependencies:
@@ -253,13 +263,173 @@ importers:
specifier: ^5 specifier: ^5
version: 5.9.3 version: 5.9.3
packages/types: {} packages/types:
devDependencies:
typescript:
specifier: ^5.9.3
version: 5.9.3
packages: packages:
'@emnapi/runtime@1.11.3': '@emnapi/runtime@1.11.3':
resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==}
'@esbuild/aix-ppc64@0.28.2':
resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [aix]
'@esbuild/android-arm64@0.28.2':
resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==}
engines: {node: '>=18'}
cpu: [arm64]
os: [android]
'@esbuild/android-arm@0.28.2':
resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==}
engines: {node: '>=18'}
cpu: [arm]
os: [android]
'@esbuild/android-x64@0.28.2':
resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==}
engines: {node: '>=18'}
cpu: [x64]
os: [android]
'@esbuild/darwin-arm64@0.28.2':
resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [darwin]
'@esbuild/darwin-x64@0.28.2':
resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==}
engines: {node: '>=18'}
cpu: [x64]
os: [darwin]
'@esbuild/freebsd-arm64@0.28.2':
resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [freebsd]
'@esbuild/freebsd-x64@0.28.2':
resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==}
engines: {node: '>=18'}
cpu: [x64]
os: [freebsd]
'@esbuild/linux-arm64@0.28.2':
resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==}
engines: {node: '>=18'}
cpu: [arm64]
os: [linux]
'@esbuild/linux-arm@0.28.2':
resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==}
engines: {node: '>=18'}
cpu: [arm]
os: [linux]
'@esbuild/linux-ia32@0.28.2':
resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==}
engines: {node: '>=18'}
cpu: [ia32]
os: [linux]
'@esbuild/linux-loong64@0.28.2':
resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==}
engines: {node: '>=18'}
cpu: [loong64]
os: [linux]
'@esbuild/linux-mips64el@0.28.2':
resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==}
engines: {node: '>=18'}
cpu: [mips64el]
os: [linux]
'@esbuild/linux-ppc64@0.28.2':
resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [linux]
'@esbuild/linux-riscv64@0.28.2':
resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==}
engines: {node: '>=18'}
cpu: [riscv64]
os: [linux]
'@esbuild/linux-s390x@0.28.2':
resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==}
engines: {node: '>=18'}
cpu: [s390x]
os: [linux]
'@esbuild/linux-x64@0.28.2':
resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [linux]
'@esbuild/netbsd-arm64@0.28.2':
resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [netbsd]
'@esbuild/netbsd-x64@0.28.2':
resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==}
engines: {node: '>=18'}
cpu: [x64]
os: [netbsd]
'@esbuild/openbsd-arm64@0.28.2':
resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==}
engines: {node: '>=18'}
cpu: [arm64]
os: [openbsd]
'@esbuild/openbsd-x64@0.28.2':
resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==}
engines: {node: '>=18'}
cpu: [x64]
os: [openbsd]
'@esbuild/openharmony-arm64@0.28.2':
resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==}
engines: {node: '>=18'}
cpu: [arm64]
os: [openharmony]
'@esbuild/sunos-x64@0.28.2':
resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==}
engines: {node: '>=18'}
cpu: [x64]
os: [sunos]
'@esbuild/win32-arm64@0.28.2':
resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==}
engines: {node: '>=18'}
cpu: [arm64]
os: [win32]
'@esbuild/win32-ia32@0.28.2':
resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==}
engines: {node: '>=18'}
cpu: [ia32]
os: [win32]
'@esbuild/win32-x64@0.28.2':
resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==}
engines: {node: '>=18'}
cpu: [x64]
os: [win32]
'@img/colour@1.1.0': '@img/colour@1.1.0':
resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==}
engines: {node: '>=18'} engines: {node: '>=18'}
@@ -422,10 +592,6 @@ packages:
cpu: [x64] cpu: [x64]
os: [win32] os: [win32]
'@modelcontextprotocol/client@2.0.0':
resolution: {integrity: sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==}
engines: {node: '>=20'}
'@modelcontextprotocol/core@2.0.0': '@modelcontextprotocol/core@2.0.0':
resolution: {integrity: sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==} resolution: {integrity: sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==}
engines: {node: '>=20'} engines: {node: '>=20'}
@@ -624,10 +790,6 @@ packages:
resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==}
engines: {node: '>=16'} engines: {node: '>=16'}
cross-spawn@7.0.6:
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
engines: {node: '>= 8'}
csstype@3.2.3: csstype@3.2.3:
resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==}
@@ -635,13 +797,10 @@ packages:
resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==}
engines: {node: '>=8'} engines: {node: '>=8'}
eventsource-parser@3.1.1: esbuild@0.28.2:
resolution: {integrity: sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==} resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==}
engines: {node: '>=18.0.0'} engines: {node: '>=18'}
hasBin: true
eventsource@3.0.7:
resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==}
engines: {node: '>=18.0.0'}
fdir@6.5.0: fdir@6.5.0:
resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==}
@@ -657,12 +816,6 @@ packages:
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
os: [darwin] os: [darwin]
isexe@2.0.0:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
jose@6.2.10:
resolution: {integrity: sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==}
lightningcss-android-arm64@1.33.0: lightningcss-android-arm64@1.33.0:
resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==}
engines: {node: '>= 12.0.0'} engines: {node: '>= 12.0.0'}
@@ -774,10 +927,6 @@ packages:
sass: sass:
optional: true optional: true
path-key@3.1.1:
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
engines: {node: '>=8'}
picocolors@1.1.1: picocolors@1.1.1:
resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==}
@@ -785,10 +934,6 @@ packages:
resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==}
engines: {node: '>=12'} engines: {node: '>=12'}
pkce-challenge@5.0.1:
resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==}
engines: {node: '>=16.20.0'}
postcss@8.5.23: postcss@8.5.23:
resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==}
engines: {node: ^10 || ^12 || >=14} engines: {node: ^10 || ^12 || >=14}
@@ -828,14 +973,6 @@ packages:
'@types/node': '@types/node':
optional: true optional: true
shebang-command@2.0.0:
resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
engines: {node: '>=8'}
shebang-regex@3.0.0:
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
engines: {node: '>=8'}
source-map-js@1.2.1: source-map-js@1.2.1:
resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}
engines: {node: '>=0.10.0'} engines: {node: '>=0.10.0'}
@@ -860,6 +997,11 @@ packages:
tslib@2.8.1: tslib@2.8.1:
resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==}
tsx@4.23.13:
resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==}
engines: {node: '>=18.0.0'}
hasBin: true
typescript@5.9.3: typescript@5.9.3:
resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}
engines: {node: '>=14.17'} engines: {node: '>=14.17'}
@@ -911,11 +1053,6 @@ packages:
yaml: yaml:
optional: true optional: true
which@2.0.2:
resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
engines: {node: '>= 8'}
hasBin: true
zod@4.5.4: zod@4.5.4:
resolution: {integrity: sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==} resolution: {integrity: sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==}
@@ -926,6 +1063,84 @@ snapshots:
tslib: 2.8.1 tslib: 2.8.1
optional: true optional: true
'@esbuild/aix-ppc64@0.28.2':
optional: true
'@esbuild/android-arm64@0.28.2':
optional: true
'@esbuild/android-arm@0.28.2':
optional: true
'@esbuild/android-x64@0.28.2':
optional: true
'@esbuild/darwin-arm64@0.28.2':
optional: true
'@esbuild/darwin-x64@0.28.2':
optional: true
'@esbuild/freebsd-arm64@0.28.2':
optional: true
'@esbuild/freebsd-x64@0.28.2':
optional: true
'@esbuild/linux-arm64@0.28.2':
optional: true
'@esbuild/linux-arm@0.28.2':
optional: true
'@esbuild/linux-ia32@0.28.2':
optional: true
'@esbuild/linux-loong64@0.28.2':
optional: true
'@esbuild/linux-mips64el@0.28.2':
optional: true
'@esbuild/linux-ppc64@0.28.2':
optional: true
'@esbuild/linux-riscv64@0.28.2':
optional: true
'@esbuild/linux-s390x@0.28.2':
optional: true
'@esbuild/linux-x64@0.28.2':
optional: true
'@esbuild/netbsd-arm64@0.28.2':
optional: true
'@esbuild/netbsd-x64@0.28.2':
optional: true
'@esbuild/openbsd-arm64@0.28.2':
optional: true
'@esbuild/openbsd-x64@0.28.2':
optional: true
'@esbuild/openharmony-arm64@0.28.2':
optional: true
'@esbuild/sunos-x64@0.28.2':
optional: true
'@esbuild/win32-arm64@0.28.2':
optional: true
'@esbuild/win32-ia32@0.28.2':
optional: true
'@esbuild/win32-x64@0.28.2':
optional: true
'@img/colour@1.1.0': '@img/colour@1.1.0':
optional: true optional: true
@@ -1033,16 +1248,6 @@ snapshots:
'@img/sharp-win32-x64@0.35.4': '@img/sharp-win32-x64@0.35.4':
optional: true optional: true
'@modelcontextprotocol/client@2.0.0':
dependencies:
'@modelcontextprotocol/core': 2.0.0
cross-spawn: 7.0.6
eventsource: 3.0.7
eventsource-parser: 3.1.1
jose: 6.2.10
pkce-challenge: 5.0.1
zod: 4.5.4
'@modelcontextprotocol/core@2.0.0': '@modelcontextprotocol/core@2.0.0':
dependencies: dependencies:
zod: 4.5.4 zod: 4.5.4
@@ -1153,21 +1358,38 @@ snapshots:
commander@11.1.0: {} commander@11.1.0: {}
cross-spawn@7.0.6:
dependencies:
path-key: 3.1.1
shebang-command: 2.0.0
which: 2.0.2
csstype@3.2.3: {} csstype@3.2.3: {}
detect-libc@2.1.2: {} detect-libc@2.1.2: {}
eventsource-parser@3.1.1: {} esbuild@0.28.2:
optionalDependencies:
eventsource@3.0.7: '@esbuild/aix-ppc64': 0.28.2
dependencies: '@esbuild/android-arm': 0.28.2
eventsource-parser: 3.1.1 '@esbuild/android-arm64': 0.28.2
'@esbuild/android-x64': 0.28.2
'@esbuild/darwin-arm64': 0.28.2
'@esbuild/darwin-x64': 0.28.2
'@esbuild/freebsd-arm64': 0.28.2
'@esbuild/freebsd-x64': 0.28.2
'@esbuild/linux-arm': 0.28.2
'@esbuild/linux-arm64': 0.28.2
'@esbuild/linux-ia32': 0.28.2
'@esbuild/linux-loong64': 0.28.2
'@esbuild/linux-mips64el': 0.28.2
'@esbuild/linux-ppc64': 0.28.2
'@esbuild/linux-riscv64': 0.28.2
'@esbuild/linux-s390x': 0.28.2
'@esbuild/linux-x64': 0.28.2
'@esbuild/netbsd-arm64': 0.28.2
'@esbuild/netbsd-x64': 0.28.2
'@esbuild/openbsd-arm64': 0.28.2
'@esbuild/openbsd-x64': 0.28.2
'@esbuild/openharmony-arm64': 0.28.2
'@esbuild/sunos-x64': 0.28.2
'@esbuild/win32-arm64': 0.28.2
'@esbuild/win32-ia32': 0.28.2
'@esbuild/win32-x64': 0.28.2
fdir@6.5.0(picomatch@4.0.7): fdir@6.5.0(picomatch@4.0.7):
optionalDependencies: optionalDependencies:
@@ -1176,10 +1398,6 @@ snapshots:
fsevents@2.3.3: fsevents@2.3.3:
optional: true optional: true
isexe@2.0.0: {}
jose@6.2.10: {}
lightningcss-android-arm64@1.33.0: lightningcss-android-arm64@1.33.0:
optional: true optional: true
@@ -1264,14 +1482,10 @@ snapshots:
- '@types/node' - '@types/node'
- babel-plugin-macros - babel-plugin-macros
path-key@3.1.1: {}
picocolors@1.1.1: {} picocolors@1.1.1: {}
picomatch@4.0.7: {} picomatch@4.0.7: {}
pkce-challenge@5.0.1: {}
postcss@8.5.23: postcss@8.5.23:
dependencies: dependencies:
nanoid: 3.3.18 nanoid: 3.3.18
@@ -1351,12 +1565,6 @@ snapshots:
'@types/node': 20.19.43 '@types/node': 20.19.43
optional: true optional: true
shebang-command@2.0.0:
dependencies:
shebang-regex: 3.0.0
shebang-regex@3.0.0: {}
source-map-js@1.2.1: {} source-map-js@1.2.1: {}
styled-jsx@5.1.6(react@19.2.8): styled-jsx@5.1.6(react@19.2.8):
@@ -1371,11 +1579,17 @@ snapshots:
tslib@2.8.1: {} tslib@2.8.1: {}
tsx@4.23.13:
dependencies:
esbuild: 0.28.2
optionalDependencies:
fsevents: 2.3.3
typescript@5.9.3: {} typescript@5.9.3: {}
undici-types@6.21.0: {} undici-types@6.21.0: {}
vite@8.2.2(@types/node@20.19.43): vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(tsx@4.23.13):
dependencies: dependencies:
lightningcss: 1.33.0 lightningcss: 1.33.0
picomatch: 4.0.7 picomatch: 4.0.7
@@ -1384,10 +1598,8 @@ snapshots:
tinyglobby: 0.2.17 tinyglobby: 0.2.17
optionalDependencies: optionalDependencies:
'@types/node': 20.19.43 '@types/node': 20.19.43
esbuild: 0.28.2
fsevents: 2.3.3 fsevents: 2.3.3
tsx: 4.23.13
which@2.0.2:
dependencies:
isexe: 2.0.0
zod@4.5.4: {} zod@4.5.4: {}
+2
View File
@@ -1,3 +1,5 @@
packages: packages:
- apps/* - apps/*
- packages/* - packages/*
allowBuilds:
esbuild: true