From b9f98ea4b619b7e9815e0f3432ae7e79dea87d3c Mon Sep 17 00:00:00 2001 From: Ku6epXBOCTuK Date: Fri, 4 Sep 2026 16:29:42 +0500 Subject: [PATCH] feat: move runner to typescript, refactor --- apps/eval/lib/{mcpClient.mjs => mcpClient.ts} | 40 +- apps/eval/lib/{ollama.mjs => ollama.ts} | 153 +++++-- apps/eval/package.json | 13 +- .../src/{report-html.mjs => report-html.ts} | 52 +-- apps/eval/src/{report.mjs => report.ts} | 43 +- apps/eval/src/{review.mjs => review.ts} | 65 +-- apps/eval/src/{run.mjs => run.ts} | 76 ++-- apps/eval/src/{runner.mjs => runner.ts} | 59 ++- apps/eval/tsconfig.json | 16 + package.json | 7 +- pnpm-lock.yaml | 386 ++++++++++++++---- pnpm-workspace.yaml | 2 + 12 files changed, 649 insertions(+), 263 deletions(-) rename apps/eval/lib/{mcpClient.mjs => mcpClient.ts} (56%) rename apps/eval/lib/{ollama.mjs => ollama.ts} (56%) rename apps/eval/src/{report-html.mjs => report-html.ts} (92%) rename apps/eval/src/{report.mjs => report.ts} (70%) rename apps/eval/src/{review.mjs => review.ts} (82%) rename apps/eval/src/{run.mjs => run.ts} (76%) rename apps/eval/src/{runner.mjs => runner.ts} (76%) create mode 100644 apps/eval/tsconfig.json diff --git a/apps/eval/lib/mcpClient.mjs b/apps/eval/lib/mcpClient.ts similarity index 56% rename from apps/eval/lib/mcpClient.mjs rename to apps/eval/lib/mcpClient.ts index 29adbdf..800fc72 100644 --- a/apps/eval/lib/mcpClient.mjs +++ b/apps/eval/lib/mcpClient.ts @@ -1,28 +1,48 @@ const ACCEPT = "application/json, text/event-stream"; -function parseMcpResponse(text) { +interface JsonRpcResponse { + result?: unknown; + error?: { code: number; message: string }; +} + +interface McpTool { + name: string; + description?: string; + inputSchema?: unknown; +} + +type McpContentBlock = { type: string; text: string } | { type: string; [key: string]: unknown }; + +interface McpToolResult { + content?: McpContentBlock[]; +} + +function parseMcpResponse(text: string): JsonRpcResponse { const trimmed = text.trim(); if (!trimmed) throw new Error("Empty MCP response"); if (trimmed.startsWith("{")) { - return JSON.parse(trimmed); + return JSON.parse(trimmed) as JsonRpcResponse; } if (trimmed.includes("event:") || trimmed.includes("data:")) { let payload = ""; for (const line of trimmed.split(/\r?\n/)) { if (line.startsWith("data:")) payload += line.slice(5).trim(); } - if (payload.startsWith("{")) return JSON.parse(payload); + if (payload.startsWith("{")) return JSON.parse(payload) as JsonRpcResponse; } throw new Error(`Unrecognized MCP response format: ${trimmed.slice(0, 200)}`); } export class McpClient { - constructor(url) { + private url: string; + private seq: number; + + constructor(url: string) { this.url = url; this.seq = 0; } - async request(method, params = {}) { + async request(method: string, params: Record = {}): Promise { this.seq += 1; const res = await fetch(this.url, { method: "POST", @@ -37,18 +57,18 @@ export class McpClient { return rpc; } - async listTools() { + async listTools(): Promise { const rpc = await this.request("tools/list"); - return rpc.result?.tools ?? []; + return (rpc.result as { tools?: McpTool[] })?.tools ?? []; } - async callTool(name, args = {}) { + async callTool(name: string, args: Record = {}): Promise { const rpc = await this.request("tools/call", { name, arguments: args }); - return rpc.result; + return rpc.result as McpToolResult; } } -export function textContentFrom(result) { +export function textContentFrom(result: McpToolResult | null | undefined): string { if (!result?.content) return ""; return result.content .map((b) => (b.type === "text" ? b.text : JSON.stringify(b))) diff --git a/apps/eval/lib/ollama.mjs b/apps/eval/lib/ollama.ts similarity index 56% rename from apps/eval/lib/ollama.mjs rename to apps/eval/lib/ollama.ts index 6fe6025..529807a 100644 --- a/apps/eval/lib/ollama.mjs +++ b/apps/eval/lib/ollama.ts @@ -1,40 +1,113 @@ +interface OllamaConstructorOpts { + url?: string; + timeoutMs?: number; + idleTimeoutMs?: number | null; + maxTotalMs?: number | null; +} + +interface PingResult { + ok: boolean; + models: string[]; +} + +export type OllamaToolCall = { + function: { name: string; arguments: string }; +}; + +export type OllamaToolDefinition = { + type: "function"; + function: { + name: string; + description: string; + parameters: { + type: "object"; + properties?: Record; + [key: string]: unknown; + }; + }; +}; + +export type OllamaMessage = { + role: string; + content: string; + tool_calls?: OllamaToolCall[]; +}; + +export type OllamaChunk = { + token: string | null; + content: string; + type: "content" | "tool_calls"; + toolCalls?: OllamaToolCall[]; +}; + +export interface ChatStreamOpts { + model: string; + messages: OllamaMessage[]; + tools?: unknown[]; + temperature?: number; + numCtx?: number; + timeoutMs?: number; + maxTotalMs?: number; + onChunk?: ((chunk: OllamaChunk) => void) | null; +} + +export interface OllamaChatResult { + role: "assistant"; + content: string; + toolCalls: OllamaToolCall[]; + promptEvalCount: number; + evalCount: number; + raw: { stream: boolean }; +} + +export interface OllamaError extends Error { + partialContent?: string; + partialToolCalls?: OllamaToolCall[]; + partialPrompt?: number; + partialGen?: number; +} + const DEFAULT_URL = "http://localhost:11434"; export class Ollama { - constructor({ url = DEFAULT_URL, timeoutMs = 600000, idleTimeoutMs = null, maxTotalMs = null } = {}) { + private url: string; + private timeoutMs: number; + private idleTimeoutMs: number; + private maxTotalMs: number; + + constructor({ url = DEFAULT_URL, timeoutMs = 600000, idleTimeoutMs = null, maxTotalMs = null }: OllamaConstructorOpts = {}) { this.url = url.replace(/\/$/, ""); this.timeoutMs = timeoutMs; - // Dynamic timeout: if set, the timer resets on every new token; the request - // only fails if this much time passes with NO progress (no new token). this.idleTimeoutMs = idleTimeoutMs ?? timeoutMs; - // Absolute hard cap on one request's lifetime, even if tokens keep flowing - // (guards against a model looping/chatting forever). Defaults to 5 minutes. this.maxTotalMs = maxTotalMs ?? 5 * 60_000; } - async ping({ timeoutMs = 30000 } = {}) { - if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, model: process.env.OLLAMA_MODEL }; + async ping({ timeoutMs = 30000 } = {}): Promise { + if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, models: [process.env.OLLAMA_MODEL ?? ""] }; const res = await fetch(`${this.url}/api/tags`, { signal: AbortSignal.timeout(timeoutMs) }); if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`); - const data = await res.json(); + const data = await res.json() as { models?: Array<{ name: string }> }; const models = (data.models ?? []).map((m) => m.name); return { ok: true, models }; } - /** - * Streaming chat with a dynamic (progress-based) timeout. - * The timeout only fires when `timeoutMs` passes with no new token/chunk. - * Optionally calls onChunk({token, content, type, toolCalls}) as data arrives. - * Returns the same shape as the old non-streaming chat(). - */ - async chatStream({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs, maxTotalMs, onChunk }) { - const body = { + async chatStream({ + model, + messages, + tools, + temperature = 0, + numCtx = 8192, + timeoutMs, + maxTotalMs, + onChunk, + }: ChatStreamOpts): Promise { + const body: Record = { model, messages, stream: true, options: { temperature }, }; - if (numCtx) body.options.num_ctx = numCtx; + if (numCtx) (body.options as Record).num_ctx = numCtx; if (tools && tools.length) body.tools = tools; const limitMs = timeoutMs ?? this.idleTimeoutMs; @@ -60,20 +133,20 @@ export class Ollama { const decoder = new TextDecoder(); let buffer = ""; let content = ""; - let toolCalls = []; + let toolCalls: OllamaToolCall[] = []; let promptEvalCount = 0; let evalCount = 0; let lastActivity = Date.now(); const guardInterval = setInterval(() => { if (Date.now() - startedAt > totalCapMs) { - const err = new Error(`Request exceeded hard cap of ${totalCapMs}ms; aborting`); + const err = new Error(`Request exceeded hard cap of ${totalCapMs}ms; aborting`) as OllamaError; err.name = "TimeoutError"; controller.abort(err); return; } if (Date.now() - lastActivity > limitMs) { - const err = new Error(`No progress from Ollama for ${limitMs}ms; aborting`); + const err = new Error(`No progress from Ollama for ${limitMs}ms; aborting`) as OllamaError; err.name = "TimeoutError"; controller.abort(err); } @@ -81,15 +154,13 @@ export class Ollama { try { for (;;) { - let chunk; + let chunk: ReadableStreamReadResult; try { chunk = await reader.read(); } catch (readErr) { - // Abort / network drop mid-stream. Surface whatever we got so the - // caller can tell "slowly progressing" from "hung". const e = new Error( - `Aborted mid-stream after ${content.length} chars (toolCalls=${toolCalls.length}): ${readErr.message}` - ); + `Aborted mid-stream after ${content.length} chars (toolCalls=${toolCalls.length}): ${(readErr as Error).message}` + ) as OllamaError; e.name = "TimeoutError"; e.partialContent = content; e.partialToolCalls = toolCalls; @@ -101,27 +172,27 @@ export class Ollama { if (done) break; lastActivity = Date.now(); buffer += decoder.decode(value, { stream: true }); - let idx; + let idx: number; while ((idx = buffer.indexOf("\n")) !== -1) { const line = buffer.slice(0, idx).trim(); buffer = buffer.slice(idx + 1); if (!line) continue; - let obj; + let obj: Record; try { obj = JSON.parse(line); } catch { continue; } - if (obj.prompt_eval_count != null) promptEvalCount = obj.prompt_eval_count; - if (obj.eval_count != null) evalCount = obj.eval_count; - const msg = obj.message ?? {}; + if (obj.prompt_eval_count != null) promptEvalCount = obj.prompt_eval_count as number; + if (obj.eval_count != null) evalCount = obj.eval_count as number; + const msg = (obj.message ?? {}) as Record; if (msg.content) { - content += msg.content; - onChunk?.({ token: msg.content, content, type: "content" }); + content += msg.content as string; + onChunk?.({ token: msg.content as string, content, type: "content" }); } - if (msg.tool_calls && msg.tool_calls.length) { - toolCalls = msg.tool_calls; - onChunk?.({ token: null, content, type: "tool_calls", toolCalls: msg.tool_calls }); + if (msg.tool_calls && (msg.tool_calls as unknown[]).length) { + toolCalls = msg.tool_calls as OllamaToolCall[]; + onChunk?.({ token: null, content, type: "tool_calls", toolCalls }); } } } @@ -139,7 +210,7 @@ export class Ollama { }; } - async chat({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs }) { + async chat({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs }: ChatStreamOpts): Promise { return this.chatStream({ model, messages, @@ -151,7 +222,7 @@ export class Ollama { }); } - async warmup({ model, text = "say OK", timeoutMs = 60000 }) { + async warmup({ model, text = "say OK", timeoutMs = 60000 }: { model: string; text?: string; timeoutMs?: number }): Promise & { error?: string }> { try { return await this.chatStream({ model, @@ -159,15 +230,15 @@ export class Ollama { timeoutMs, }); } catch (err) { - return { error: err.message }; + return { error: (err as Error).message }; } } } -export async function listModels() { +export async function listModels(): Promise { const o = new Ollama(); - const res = await fetch(`${o.url}/api/tags`); + const res = await fetch(`${o["url"]}/api/tags`); if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status})`); - const data = await res.json(); + const data = await res.json() as { models?: Array<{ name: string }> }; return (data.models ?? []).map((m) => m.name); } diff --git a/apps/eval/package.json b/apps/eval/package.json index 61d3e63..f0d8532 100644 --- a/apps/eval/package.json +++ b/apps/eval/package.json @@ -3,12 +3,19 @@ "version": "1.0.0", "description": "MCP rubber-duck evaluation harness", "scripts": { - "run": "node src/run.mjs", - "report-html": "node src/report-html.mjs" + "run": "tsx src/run.ts", + "review": "tsx src/review.ts", + "report-html": "tsx src/report-html.ts", + "typecheck": "tsc --noEmit" }, "license": "ISC", "type": "module", "dependencies": { - "@modelcontextprotocol/client": "^2.0.0" + "@duck/types": "workspace:*" + }, + "devDependencies": { + "@types/node": "^20", + "tsx": "^4", + "typescript": "^5.9.3" } } diff --git a/apps/eval/src/report-html.mjs b/apps/eval/src/report-html.ts similarity index 92% rename from apps/eval/src/report-html.mjs rename to apps/eval/src/report-html.ts index 32dc97c..cedfe3d 100644 --- a/apps/eval/src/report-html.mjs +++ b/apps/eval/src/report-html.ts @@ -1,51 +1,55 @@ import { readFile, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; +import type { ReportRoot, ScenarioName, ScenarioSummary, AggregateScenario, PerTaskRow } from "@duck/types"; +import { SCENARIO_NAMES } from "@duck/types"; -const SCENARIOS = ["control", "thinking", "blind", "mentor"]; - -const SCENARIO_LABELS = { +const SCENARIO_LABELS: Record = { control: "Без утки (контроль)", thinking: "Думать вслух", blind: "Слепая утка", mentor: "Утка-помощник", }; -const SCENARIO_DESCS = { +const SCENARIO_DESCS: Record = { control: "Модель решает задачу напрямую, без инструментов.", thinking: "Модель выписывает рассуждения вслух, без дука.", blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.", mentor: "Модель работает в паре, зная, что ответ будет только «quack».", }; -function esc(s) { +function esc(s: unknown): string { return String(s ?? "") .replace(/&/g, "&") .replace(//g, ">"); } -function nl2br(s) { +function nl2br(s: string): string { return esc(s).replace(/\n/g, "
"); } -function pctColor(p) { +function pctColor(p: number): string { return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444"; } -function modelShort(name) { +function modelShort(name: string): string { const base = String(name).split(/:/)[0]; const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : ""; return esc(base + ver); } -function summaryTable(report) { +interface RowHtmlItem extends PerTaskRow { + model: string; +} + +function summaryTable(report: ReportRoot): string { const perModel = report.aggregate.perModel; const models = Object.keys(perModel); const rows = models .map((m) => { const g = perModel[m]; const scen = report.models[m].scenarios; - const acc = (s) => scen[s].accuracy; + const acc = (s: ScenarioName) => scen[s].accuracy; const row = ` ${acc("control")}% ${acc("thinking")}% @@ -86,7 +90,7 @@ function summaryTable(report) { `; } -function scenarioCard(key, agg) { +function scenarioCard(key: ScenarioName, agg: AggregateScenario): string { const pct = agg.accuracy; const color = pctColor(pct); const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0; @@ -122,7 +126,7 @@ function scenarioCard(key, agg) { `; } -function rowHtml(r) { +function rowHtml(r: RowHtmlItem): string { const pending = r.correct === null || r.correct === undefined; const cls = pending ? "pend" : r.correct ? "ok" : "no"; const result = pending ? "?" : r.correct ? "✓" : "✗"; @@ -141,10 +145,10 @@ function rowHtml(r) { `; } -function promptBlock(report) { - const p = report.prompts || {}; - const items = SCENARIOS - .filter((k) => p[k]) +function promptBlock(report: ReportRoot): string { + const p = report.prompts; + const items = (SCENARIO_NAMES as readonly ScenarioName[]) + .filter((k): k is ScenarioName => !!p[k]) .map( (k) => `
@@ -161,15 +165,15 @@ function promptBlock(report) { `; } -export function buildHtml(report) { - const perTask = Object.entries(report.models).flatMap(([model, rep]) => +export function buildHtml(report: ReportRoot): string { + const perTask: RowHtmlItem[] = Object.entries(report.models).flatMap(([model, rep]) => rep.perTask.map((r) => ({ ...r, model })) ); const rows = perTask.map(rowHtml).join(""); const models = Object.keys(report.aggregate.perModel); let totalPending = 0, totalReviewed = 0; for (const rep of Object.values(report.models)) { - for (const s of SCENARIOS) { + for (const s of SCENARIO_NAMES) { if (rep.scenarios[s]) { totalReviewed += rep.scenarios[s].reviewed || 0; totalPending += rep.scenarios[s].pending || 0; @@ -182,8 +186,8 @@ export function buildHtml(report) { const modelOptions = models .map((m) => ``) .join(""); - const aggCards = SCENARIOS - .filter((k) => report.aggregate.perScenario[k]) + const aggCards = (SCENARIO_NAMES as readonly ScenarioName[]) + .filter((k): k is ScenarioName => !!report.aggregate.perScenario[k]) .map((k) => scenarioCard(k, report.aggregate.perScenario[k])) .join(""); @@ -353,9 +357,9 @@ export function buildHtml(report) { `; } -async function main() { +async function main(): Promise { const args = process.argv.slice(2); - let inFile = null, outFile = "out/report.html"; + let inFile: string | null = null, outFile = "out/report.html"; for (let i = 0; i < args.length; i += 1) { const a = args[i]; const next = () => args[i + 1]; @@ -371,7 +375,7 @@ async function main() { inFile = (await readFile(reviewed, "utf8").then(() => reviewed).catch(() => base)); } - const report = JSON.parse(await readFile(inFile, "utf8")); + const report: ReportRoot = JSON.parse(await readFile(inFile, "utf8")); const html = buildHtml(report); await mkdir(path.dirname(outFile), { recursive: true }); await writeFile(outFile, html, "utf8"); diff --git a/apps/eval/src/report.mjs b/apps/eval/src/report.ts similarity index 70% rename from apps/eval/src/report.mjs rename to apps/eval/src/report.ts index 377b22e..5b0089c 100644 --- a/apps/eval/src/report.mjs +++ b/apps/eval/src/report.ts @@ -1,16 +1,28 @@ -function round(x, d = 2) { +import type { ScenarioName, ScenarioSummary, PerTaskRow, ModelReport, Aggregate, AggregateScenario, AggregateModel } from "@duck/types"; +import { SCENARIO_NAMES } from "@duck/types"; + +function round(x: number, d = 2): number { if (!Number.isFinite(x)) return x; const f = 10 ** d; return Math.round(x * f) / f; } -function avg(arr) { +function avg(arr: number[]): number { if (!arr.length) return 0; return arr.reduce((a, b) => a + b, 0) / arr.length; } -export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) { const scenarios = ["control", "thinking", "blind", "mentor"]; - const byScenario = {}; +export interface BuildReportOpts { + model: string; + mcpUrl: string; + rows: PerTaskRow[]; + meta?: Record; + prompts?: Partial>; +} + +export function buildReport({ model, mcpUrl, rows, meta = {}, prompts = {} }: BuildReportOpts): ModelReport { + const scenarios = SCENARIO_NAMES; + const byScenario: Record = {}; for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] }; for (const row of rows) { @@ -29,10 +41,9 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = { byScenario[s].rows.push(row); } - const summaries = {}; + const summaries: Record = {} as Record; for (const s of scenarios) { const g = byScenario[s]; - const denom = g.reviewed || 1; summaries[s] = { tasks: g.total, reviewed: g.reviewed, @@ -53,7 +64,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = { model, mcpUrl, scenarios: summaries, - prompts, + prompts: prompts as Record, perTask: rows.map((r) => ({ id: r.id, scenario: r.scenario, @@ -71,13 +82,11 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = { }; } -const SCENARIOS = ["control", "thinking", "blind", "mentor"]; - -export function buildAggregate(modelsReport) { - const perScenario = {}; - for (const s of SCENARIOS) { +export function buildAggregate(modelsReport: Record): Aggregate { + const perScenario: Record = {} as Record; + for (const s of SCENARIO_NAMES) { let total = 0, reviewed = 0, correct = 0, duckUsed = 0, toolCalls = 0; - const byModel = {}; + const byModel: Record = {}; for (const [name, rep] of Object.entries(modelsReport)) { const g = rep.scenarios[s]; if (!g) continue; @@ -107,12 +116,12 @@ export function buildAggregate(modelsReport) { }; } - const perModel = {}; + const perModel: Record = {}; for (const [name, rep] of Object.entries(modelsReport)) { let duckUsed = 0, toolCalls = 0; - const gen = []; + const gen: number[] = []; let reviewed = 0, correct = 0; - for (const s of SCENARIOS) { + for (const s of SCENARIO_NAMES) { const g = rep.scenarios[s]; duckUsed += g.duckUsed; toolCalls += g.toolCalls; @@ -123,7 +132,7 @@ export function buildAggregate(modelsReport) { } } const taskCount = rep.scenarios.control?.tasks ?? 0; - const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0); + const totalRows = SCENARIO_NAMES.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0); perModel[name] = { tasks: taskCount, totalRows, diff --git a/apps/eval/src/review.mjs b/apps/eval/src/review.ts similarity index 82% rename from apps/eval/src/review.mjs rename to apps/eval/src/review.ts index 9668d1d..be9aa3f 100644 --- a/apps/eval/src/review.mjs +++ b/apps/eval/src/review.ts @@ -1,24 +1,36 @@ import { readFile, writeFile, mkdir } from "node:fs/promises"; import path from "node:path"; import readline from "node:readline"; -import { buildReport, buildAggregate } from "./report.mjs"; +import type { ReportRoot, ScenarioName, PerTaskRow } from "@duck/types"; +import { SCENARIO_NAMES } from "@duck/types"; +import { buildReport, buildAggregate } from "./report.js"; -const SCENARIO_LABELS = { +const SCENARIO_LABELS: Record = { control: "Без утки (контроль)", thinking: "Думать вслух", blind: "Слепая утка", mentor: "Утка-помощник", }; -function esc(s) { - return String(s ?? "") - .replace(/&/g, "&") - .replace(//g, ">"); +interface ReviewEntry { + model: string; + taskId: string; + scenario: ScenarioName; + row: PerTaskRow; } -function parseArgs(argv) { - const args = { +interface ReviewArgs { + report: string | null; + out: string | null; + reviewFile: string | null; + model: string | null; + scenario: string | null; + limit: number | null; + reAsk?: boolean; +} + +function parseArgs(argv: string[]): ReviewArgs { + const args: ReviewArgs = { report: null, out: null, reviewFile: null, @@ -40,22 +52,22 @@ function parseArgs(argv) { return args; } -async function main() { +async function main(): Promise { const args = parseArgs(process.argv.slice(2)); const reportFile = args.report || path.join("out", "report.json"); const outFile = args.out || path.join("out", "report.reviewed.json"); const reviewFile = args.reviewFile || path.join("out", "review.json"); - const report = JSON.parse(await readFile(reportFile, "utf8")); + const report: ReportRoot = JSON.parse(await readFile(reportFile, "utf8")); - let decisions = {}; + let decisions: Record = {}; try { decisions = JSON.parse(await readFile(reviewFile, "utf8")); } catch { /* no prior review */ } - const entries = []; + const entries: ReviewEntry[] = []; for (const [model, rep] of Object.entries(report.models)) { for (const row of rep.perTask) { if (args.model && model !== args.model) continue; @@ -65,13 +77,13 @@ async function main() { } if (args.limit && entries.length > args.limit) entries.length = args.limit; - const key = (e) => `${e.model}|${e.taskId}|${e.scenario}`; + const key = (e: ReviewEntry) => `${e.model}|${e.taskId}|${e.scenario}`; const isTTY = !!process.stdin.isTTY; - let batchLines = []; + let batchLines: string[] = []; if (!isTTY) { - const input = await new Promise((res, rej) => { + const input = await new Promise((res, rej) => { let data = ""; process.stdin.setEncoding("utf8"); process.stdin.on("data", (c) => (data += c)); @@ -83,17 +95,21 @@ async function main() { } let batchIdx = 0; - const nextInput = async () => { + const nextInput = async (): Promise => { if (isTTY) { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - const ans = await new Promise((res) => rl.question("", res)); + const ans = await new Promise((res) => rl.question("", res)); rl.close(); return ans.trim().toLowerCase(); } return batchLines[batchIdx++] ?? ""; }; - async function askOne(e, nextInput, decisions) { + let done = 0; + let skipped = 0; + let changed = 0; + + async function askOne(e: ReviewEntry): Promise { const k = key(e); const prev = decisions[k]; if (prev !== undefined) { @@ -115,7 +131,7 @@ async function main() { process.stdout.write(` [y] верно [n] неверно [e] пропуск${hint}\n > `); const a = (await nextInput()) || ""; - let decision = null; + let decision: boolean | null = null; if (a === "y" || a === "д") decision = true; else if (a === "n" || a === "н") decision = false; else if (a === "" && cur) decision = e.row.correct; @@ -131,10 +147,6 @@ async function main() { return 1; } - let done = 0; - let skipped = 0; - let changed = 0; - const reAsk = !!args.reAsk; for (const e of entries) { @@ -142,7 +154,7 @@ async function main() { const prev = decisions[k]; if (prev === undefined || reAsk) { - done += await askOne(e, nextInput, decisions); + done += await askOne(e); continue; } @@ -161,7 +173,6 @@ async function main() { scenarios: buildReport({ model, mcpUrl: report.mcpUrl, - tasks: [], rows: rep.perTask.map((r) => ({ id: r.id, scenario: r.scenario, @@ -192,7 +203,7 @@ async function main() { const scen = rep.scenarios; console.log( `${model} | ` + - ["control", "thinking", "blind", "mentor"] + SCENARIO_NAMES .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`) .join(" ") + ` всего rev=${g.reviewed}` diff --git a/apps/eval/src/run.mjs b/apps/eval/src/run.ts similarity index 76% rename from apps/eval/src/run.mjs rename to apps/eval/src/run.ts index fe8c1c3..53d701b 100644 --- a/apps/eval/src/run.mjs +++ b/apps/eval/src/run.ts @@ -1,13 +1,15 @@ import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises"; import path from "node:path"; -import { Ollama } from "../lib/ollama.mjs"; -import { McpClient } from "../lib/mcpClient.mjs"; -import { runScenario } from "./runner.mjs"; -import { PROMPTS } from "./runner.mjs"; -import { buildReport, buildAggregate } from "./report.mjs"; +import type { ScenarioName, ReportRoot, PerTaskRow, Task } from "@duck/types"; +import { SCENARIO_NAMES } from "@duck/types"; +import { Ollama } from "../lib/ollama.js"; +import type { OllamaError } from "../lib/ollama.js"; +import { McpClient } from "../lib/mcpClient.js"; +import { runScenario, PROMPTS } from "./runner.js"; +import { buildReport, buildAggregate } from "./report.js"; -let logTarget = null; -async function log(msg) { +let logTarget: string | null = null; +async function log(msg: string): Promise { const line = `[${new Date().toISOString()}] ${msg}`; console.log(line); if (logTarget) { @@ -19,8 +21,6 @@ async function log(msg) { } } -const SCENARIOS = ["control", "thinking", "blind", "mentor"]; - const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:1.7b", "qwen3:4b", "granite4.1:3b", "phi4-mini:3.8b"]; const DEFAULT_MCP_URL = "https://rubber-duck-mcp.vercel.app/api/mcp"; const REQUEST_TIMEOUT_MS = 60_000; @@ -28,9 +28,9 @@ const MAX_ATTEMPTS = 3; const PAUSE_BETWEEN_TASKS_MS = 2_000; const PAUSE_BETWEEN_RETRIES_MS = 10_000; -function classifyError(err) { - const name = err?.name ?? ""; - const msg = String(err?.message ?? ""); +function classifyError(err: unknown): string { + const name = (err as OllamaError)?.name ?? ""; + const msg = String((err as OllamaError)?.message ?? ""); const low = `${name} ${msg}`.toLowerCase(); if (name === "TimeoutError" || low.includes("timeout") || low.includes("aborted")) return "timeout"; if (low.includes("fetch failed") || low.includes("connect") || low.includes("etimedout")) return "network"; @@ -39,10 +39,20 @@ function classifyError(err) { return "other"; } -function parseArgs(argv) { - const args = { +interface Args { + models: string[]; + scenarios: ScenarioName[]; + limit: number; + model: string | null; + mcpUrl: string | null; + tasksFile: string | null; + out: string | null; +} + +function parseArgs(argv: string[]): Args { + const args: Args = { models: [], - scenarios: SCENARIOS, + scenarios: [...SCENARIO_NAMES], limit: Infinity, model: process.env.OLLAMA_MODEL || null, mcpUrl: null, @@ -55,7 +65,7 @@ function parseArgs(argv) { if (a === "--models") args.models = next().split(",").map((s) => s.trim()).filter(Boolean), i += 1; else if (a === "--model") args.model = next(), i += 1; else if (a === "--mcp-url") args.mcpUrl = next(), i += 1; - else if (a === "--scenarios") args.scenarios = next().split(","), i += 1; + else if (a === "--scenarios") args.scenarios = next().split(",") as ScenarioName[], i += 1; else if (a === "--limit") args.limit = Number(next()), i += 1; else if (a === "--tasks") args.tasksFile = next(), i += 1; else if (a === "--out") args.out = next(), i += 1; @@ -63,26 +73,26 @@ function parseArgs(argv) { return args; } -async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { - const rows = []; +async function runOneModel({ model, ollama, mcp, tasks, scenarios }: { model: string; ollama: Ollama; mcp: McpClient; tasks: Task[]; scenarios: ScenarioName[] }): Promise { + const rows: PerTaskRow[] = []; for (const task of tasks) { for (const scenario of scenarios) { let r = null; - let lastErr = null; + let lastErr: OllamaError | null = null; const t0 = Date.now(); for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { try { r = await runScenario({ scenario, ollama, model, task: task.question, mcp }); break; } catch (err) { - lastErr = err; + lastErr = err as OllamaError; const ms = Date.now() - t0; - await log(` [${task.id}/${scenario}] attempt ${attempt} FAILED (${ms}ms): ${err.message}`); - if (err.partialContent != null || err.partialToolCalls != null) { + await log(` [${task.id}/${scenario}] attempt ${attempt} FAILED (${ms}ms): ${(err as Error).message}`); + if (lastErr.partialContent != null || lastErr.partialToolCalls != null) { await log( - ` partial before abort: chars=${err.partialContent?.length ?? 0} ` + - `toolCalls=${err.partialToolCalls?.length ?? 0} ` + - `gen=${err.partialGen ?? 0}` + ` partial before abort: chars=${lastErr.partialContent?.length ?? 0} ` + + `toolCalls=${lastErr.partialToolCalls?.length ?? 0} ` + + `gen=${lastErr.partialGen ?? 0}` ); } if (attempt < MAX_ATTEMPTS) await new Promise((res) => setTimeout(res, PAUSE_BETWEEN_RETRIES_MS)); @@ -121,7 +131,7 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) { return rows; } -async function main() { +async function main(): Promise { const args = parseArgs(process.argv.slice(2)); let models = args.models; @@ -131,11 +141,11 @@ async function main() { const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL; const tasksFile = args.tasksFile || path.join("data", "tasks.json"); - const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8")); + const tasksRaw: Task[] = JSON.parse(await readFile(tasksFile, "utf8")); const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit); const out = args.out || path.join("out", "report.json"); - const logFile = args.logTarget || path.join(path.dirname(out), "run.log"); + const logFile = path.join(path.dirname(out), "run.log"); logTarget = logFile; await mkdir(path.dirname(logFile), { recursive: true }); await appendFile(logFile, "", "utf8").catch(() => {}); @@ -156,7 +166,7 @@ async function main() { const mcpTools = await mcp.listTools(); await log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`); - const modelsReport = {}; + const modelsReport: ReportRoot["models"] = {}; for (const model of models) { const tStart = Date.now(); await log(`==== Running model: ${model} ====`); @@ -168,18 +178,18 @@ async function main() { await log(` warmup OK (${warmMs}ms, gen=${warm?.evalCount ?? 0})`); } const rows = await runOneModel({ model, ollama, mcp, tasks, scenarios: args.scenarios }); - const rep = buildReport({ model, mcpUrl, tasks, rows }); + const rep = buildReport({ model, mcpUrl, rows }); modelsReport[model] = { model, scenarios: rep.scenarios, perTask: rep.perTask, - }; + } as ReportRoot["models"][string]; await log(`==== Done ${model} in ${Math.round((Date.now() - tStart) / 1000)}s ====`); } const aggregate = buildAggregate(modelsReport); - const report = { + const report: ReportRoot = { generatedAt: new Date().toISOString(), mcpUrl, prompts: PROMPTS, @@ -196,7 +206,7 @@ async function main() { const scen = modelsReport[modelName].scenarios; await log( `${modelName} | ` + - SCENARIOS.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}%`).join(" ") + + SCENARIO_NAMES.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}%`).join(" ") + ` duck=${g.duckUsed} calls=${g.toolCalls} avgGen=${g.avgGenTokens}` ); } diff --git a/apps/eval/src/runner.mjs b/apps/eval/src/runner.ts similarity index 76% rename from apps/eval/src/runner.mjs rename to apps/eval/src/runner.ts index e74b871..5eb8dbd 100644 --- a/apps/eval/src/runner.mjs +++ b/apps/eval/src/runner.ts @@ -1,7 +1,9 @@ -import { Ollama } from "../lib/ollama.mjs"; -import { McpClient, textContentFrom } from "../lib/mcpClient.mjs"; +import type { ScenarioName } from "@duck/types"; +import { Ollama } from "../lib/ollama.js"; +import type { OllamaMessage, OllamaToolDefinition } from "../lib/ollama.js"; +import { McpClient, textContentFrom } from "../lib/mcpClient.js"; -const DUCK_TOOL = { +const DUCK_TOOL: OllamaToolDefinition = { type: "function", function: { name: "quack", @@ -48,7 +50,7 @@ const MENTOR_SYSTEM = "Treat it as a way to voice your thoughts out loud, not as a source of answers. " + "After the tool's reply, give the user a clear, concrete final answer to the question."; -export const PROMPTS = { +export const PROMPTS: Record = { control: CONTROL_SYSTEM, thinking: THINKING_SYSTEM, blind: BLIND_SYSTEM, @@ -57,11 +59,22 @@ export const PROMPTS = { const MAX_DUCK_TURNS = 6; -function toolDefinition() { +interface RunnerResult { + response: string; + toolCalls: number; + duckUsed: boolean; + duckTokens: number; + promptTokens: number; + genTokens: number; + turns?: number; + messages: OllamaMessage[]; +} + +function toolDefinition(): OllamaToolDefinition { return DUCK_TOOL; } -async function runNoTool(ollama, model, task, system) { +async function runNoTool(ollama: Ollama, model: string, task: string, system: string): Promise { const messages = [ { role: "system", content: system }, { role: "user", content: task }, @@ -78,16 +91,16 @@ async function runNoTool(ollama, model, task, system) { }; } -function runControl(ollama, model, task) { +function runControl(ollama: Ollama, model: string, task: string): Promise { return runNoTool(ollama, model, task, CONTROL_SYSTEM); } -function runThinking(ollama, model, task) { +function runThinking(ollama: Ollama, model: string, task: string): Promise { return runNoTool(ollama, model, task, THINKING_SYSTEM); } -async function runDuck(ollama, model, task, mcp, system, scenario) { - const messages = [ +async function runDuck(ollama: Ollama, model: string, task: string, mcp: McpClient, system: string, scenario: ScenarioName): Promise { + const messages: OllamaMessage[] = [ { role: "system", content: system }, { role: "user", content: task }, ]; @@ -107,18 +120,21 @@ async function runDuck(ollama, model, task, mcp, system, scenario) { if (r.toolCalls && r.toolCalls.length) { toolCalls += r.toolCalls.length; - const assistantMsg = { role: "assistant", content: r.content || "" }; - assistantMsg.tool_calls = r.toolCalls.map((tc) => ({ - function: { name: tc.function.name, arguments: tc.function.arguments }, - })); + const assistantMsg: OllamaMessage = { + role: "assistant", + content: r.content || "", + tool_calls: r.toolCalls.map((tc) => ({ + function: { name: tc.function.name, arguments: tc.function.arguments }, + })), + }; messages.push(assistantMsg); for (const tc of r.toolCalls) { if (tc.function.name === "quack") { duckUsed = true; - let args = {}; + let args: Record = {}; try { - args = tc.function.arguments ? JSON.parse(tc.function.arguments) : {}; + args = tc.function.arguments ? JSON.parse(tc.function.arguments) as Record : {}; } catch { args = {}; } @@ -146,7 +162,6 @@ async function runDuck(ollama, model, task, mcp, system, scenario) { }; } - // loop bound hit without a final answer const last = messages[messages.length - 1]; return { response: last?.content ?? "", @@ -160,7 +175,15 @@ async function runDuck(ollama, model, task, mcp, system, scenario) { }; } -export async function runScenario({ scenario, ollama, model, task, mcp }) { +export interface RunScenarioOpts { + scenario: ScenarioName; + ollama: Ollama; + model: string; + task: string; + mcp: McpClient; +} + +export async function runScenario({ scenario, ollama, model, task, mcp }: RunScenarioOpts): Promise { if (scenario === "control") { return runControl(ollama, model, task); } diff --git a/apps/eval/tsconfig.json b/apps/eval/tsconfig.json new file mode 100644 index 0000000..a37d906 --- /dev/null +++ b/apps/eval/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "strict": true, + "noEmit": true, + "isolatedModules": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "paths": { + "@duck/types": ["../../packages/types/src/index.ts"] + } + }, + "include": ["src", "lib"] +} diff --git a/package.json b/package.json index c6f0f74..88cc840 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,10 @@ "typecheck-mcp": "pnpm --dir apps/mcp exec tsc --noEmit", "deploy-mcp": "vercel --cwd apps/mcp --prod --yes", "deploy-frontend": "vercel --cwd apps/frontend --prod --yes", - "eval:run": "pnpm --dir apps/eval exec node src/run.mjs", - "eval:review": "pnpm --dir apps/eval exec node src/review.mjs", - "eval:html": "pnpm --dir apps/eval exec node src/report-html.mjs" + "eval:run": "pnpm --dir apps/eval run run", + "eval:review": "pnpm --dir apps/eval run review", + "eval:html": "pnpm --dir apps/eval run report-html", + "eval:typecheck": "pnpm --dir apps/eval run typecheck" }, "license": "ISC", "devEngines": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce7e240..1ec26a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -209,15 +209,25 @@ importers: apps/eval: dependencies: - '@modelcontextprotocol/client': - specifier: ^2.0.0 - version: 2.0.0 + '@duck/types': + specifier: workspace:* + version: link:../../packages/types + devDependencies: + '@types/node': + specifier: ^20 + version: 20.19.43 + tsx: + specifier: ^4 + version: 4.23.13 + typescript: + specifier: ^5.9.3 + version: 5.9.3 apps/frontend: devDependencies: vite: specifier: ^8.0.11 - version: 8.2.2(@types/node@20.19.43) + version: 8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(tsx@4.23.13) apps/mcp: dependencies: @@ -253,13 +263,173 @@ importers: specifier: ^5 version: 5.9.3 - packages/types: {} + packages/types: + devDependencies: + typescript: + specifier: ^5.9.3 + version: 5.9.3 packages: '@emnapi/runtime@1.11.3': resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} + '@esbuild/aix-ppc64@0.28.2': + resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.28.2': + resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.28.2': + resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.28.2': + resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.28.2': + resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.2': + resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.28.2': + resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.2': + resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.28.2': + resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.28.2': + resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.28.2': + resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.28.2': + resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.28.2': + resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.28.2': + resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.2': + resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.28.2': + resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.28.2': + resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.2': + resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.2': + resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.2': + resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.2': + resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.2': + resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.28.2': + resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.28.2': + resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.28.2': + resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.28.2': + resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@img/colour@1.1.0': resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} engines: {node: '>=18'} @@ -422,10 +592,6 @@ packages: cpu: [x64] os: [win32] - '@modelcontextprotocol/client@2.0.0': - resolution: {integrity: sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==} - engines: {node: '>=20'} - '@modelcontextprotocol/core@2.0.0': resolution: {integrity: sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==} engines: {node: '>=20'} @@ -624,10 +790,6 @@ packages: resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} engines: {node: '>=16'} - cross-spawn@7.0.6: - resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} - engines: {node: '>= 8'} - csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} @@ -635,13 +797,10 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} - eventsource-parser@3.1.1: - resolution: {integrity: sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==} - engines: {node: '>=18.0.0'} - - eventsource@3.0.7: - resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} - engines: {node: '>=18.0.0'} + esbuild@0.28.2: + resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} + engines: {node: '>=18'} + hasBin: true fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} @@ -657,12 +816,6 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - isexe@2.0.0: - resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - - jose@6.2.10: - resolution: {integrity: sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==} - lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -774,10 +927,6 @@ packages: sass: optional: true - path-key@3.1.1: - resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} - engines: {node: '>=8'} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -785,10 +934,6 @@ packages: resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} engines: {node: '>=12'} - pkce-challenge@5.0.1: - resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} - engines: {node: '>=16.20.0'} - postcss@8.5.23: resolution: {integrity: sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==} engines: {node: ^10 || ^12 || >=14} @@ -828,14 +973,6 @@ packages: '@types/node': optional: true - shebang-command@2.0.0: - resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} - engines: {node: '>=8'} - - shebang-regex@3.0.0: - resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} - engines: {node: '>=8'} - source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -860,6 +997,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.23.13: + resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==} + engines: {node: '>=18.0.0'} + hasBin: true + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -911,11 +1053,6 @@ packages: yaml: optional: true - which@2.0.2: - resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} - engines: {node: '>= 8'} - hasBin: true - zod@4.5.4: resolution: {integrity: sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==} @@ -926,6 +1063,84 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild/aix-ppc64@0.28.2': + optional: true + + '@esbuild/android-arm64@0.28.2': + optional: true + + '@esbuild/android-arm@0.28.2': + optional: true + + '@esbuild/android-x64@0.28.2': + optional: true + + '@esbuild/darwin-arm64@0.28.2': + optional: true + + '@esbuild/darwin-x64@0.28.2': + optional: true + + '@esbuild/freebsd-arm64@0.28.2': + optional: true + + '@esbuild/freebsd-x64@0.28.2': + optional: true + + '@esbuild/linux-arm64@0.28.2': + optional: true + + '@esbuild/linux-arm@0.28.2': + optional: true + + '@esbuild/linux-ia32@0.28.2': + optional: true + + '@esbuild/linux-loong64@0.28.2': + optional: true + + '@esbuild/linux-mips64el@0.28.2': + optional: true + + '@esbuild/linux-ppc64@0.28.2': + optional: true + + '@esbuild/linux-riscv64@0.28.2': + optional: true + + '@esbuild/linux-s390x@0.28.2': + optional: true + + '@esbuild/linux-x64@0.28.2': + optional: true + + '@esbuild/netbsd-arm64@0.28.2': + optional: true + + '@esbuild/netbsd-x64@0.28.2': + optional: true + + '@esbuild/openbsd-arm64@0.28.2': + optional: true + + '@esbuild/openbsd-x64@0.28.2': + optional: true + + '@esbuild/openharmony-arm64@0.28.2': + optional: true + + '@esbuild/sunos-x64@0.28.2': + optional: true + + '@esbuild/win32-arm64@0.28.2': + optional: true + + '@esbuild/win32-ia32@0.28.2': + optional: true + + '@esbuild/win32-x64@0.28.2': + optional: true + '@img/colour@1.1.0': optional: true @@ -1033,16 +1248,6 @@ snapshots: '@img/sharp-win32-x64@0.35.4': optional: true - '@modelcontextprotocol/client@2.0.0': - dependencies: - '@modelcontextprotocol/core': 2.0.0 - cross-spawn: 7.0.6 - eventsource: 3.0.7 - eventsource-parser: 3.1.1 - jose: 6.2.10 - pkce-challenge: 5.0.1 - zod: 4.5.4 - '@modelcontextprotocol/core@2.0.0': dependencies: zod: 4.5.4 @@ -1153,21 +1358,38 @@ snapshots: commander@11.1.0: {} - cross-spawn@7.0.6: - dependencies: - path-key: 3.1.1 - shebang-command: 2.0.0 - which: 2.0.2 - csstype@3.2.3: {} detect-libc@2.1.2: {} - eventsource-parser@3.1.1: {} - - eventsource@3.0.7: - dependencies: - eventsource-parser: 3.1.1 + esbuild@0.28.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.2 + '@esbuild/android-arm': 0.28.2 + '@esbuild/android-arm64': 0.28.2 + '@esbuild/android-x64': 0.28.2 + '@esbuild/darwin-arm64': 0.28.2 + '@esbuild/darwin-x64': 0.28.2 + '@esbuild/freebsd-arm64': 0.28.2 + '@esbuild/freebsd-x64': 0.28.2 + '@esbuild/linux-arm': 0.28.2 + '@esbuild/linux-arm64': 0.28.2 + '@esbuild/linux-ia32': 0.28.2 + '@esbuild/linux-loong64': 0.28.2 + '@esbuild/linux-mips64el': 0.28.2 + '@esbuild/linux-ppc64': 0.28.2 + '@esbuild/linux-riscv64': 0.28.2 + '@esbuild/linux-s390x': 0.28.2 + '@esbuild/linux-x64': 0.28.2 + '@esbuild/netbsd-arm64': 0.28.2 + '@esbuild/netbsd-x64': 0.28.2 + '@esbuild/openbsd-arm64': 0.28.2 + '@esbuild/openbsd-x64': 0.28.2 + '@esbuild/openharmony-arm64': 0.28.2 + '@esbuild/sunos-x64': 0.28.2 + '@esbuild/win32-arm64': 0.28.2 + '@esbuild/win32-ia32': 0.28.2 + '@esbuild/win32-x64': 0.28.2 fdir@6.5.0(picomatch@4.0.7): optionalDependencies: @@ -1176,10 +1398,6 @@ snapshots: fsevents@2.3.3: optional: true - isexe@2.0.0: {} - - jose@6.2.10: {} - lightningcss-android-arm64@1.33.0: optional: true @@ -1264,14 +1482,10 @@ snapshots: - '@types/node' - babel-plugin-macros - path-key@3.1.1: {} - picocolors@1.1.1: {} picomatch@4.0.7: {} - pkce-challenge@5.0.1: {} - postcss@8.5.23: dependencies: nanoid: 3.3.18 @@ -1351,12 +1565,6 @@ snapshots: '@types/node': 20.19.43 optional: true - shebang-command@2.0.0: - dependencies: - shebang-regex: 3.0.0 - - shebang-regex@3.0.0: {} - source-map-js@1.2.1: {} styled-jsx@5.1.6(react@19.2.8): @@ -1371,11 +1579,17 @@ snapshots: tslib@2.8.1: {} + tsx@4.23.13: + dependencies: + esbuild: 0.28.2 + optionalDependencies: + fsevents: 2.3.3 + typescript@5.9.3: {} undici-types@6.21.0: {} - vite@8.2.2(@types/node@20.19.43): + vite@8.2.2(@types/node@20.19.43)(esbuild@0.28.2)(tsx@4.23.13): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -1384,10 +1598,8 @@ snapshots: tinyglobby: 0.2.17 optionalDependencies: '@types/node': 20.19.43 + esbuild: 0.28.2 fsevents: 2.3.3 - - which@2.0.2: - dependencies: - isexe: 2.0.0 + tsx: 4.23.13 zod@4.5.4: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 286cf7f..32c2173 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,5 @@ packages: - apps/* - packages/* +allowBuilds: + esbuild: true