feat: add run test flow

apps/eval/lib/mcpClient.mjs

@@ -0,0 +1,56 @@
+const ACCEPT = "application/json, text/event-stream";
+
+function parseMcpResponse(text) {
+  const trimmed = text.trim();
+  if (!trimmed) throw new Error("Empty MCP response");
+  if (trimmed.startsWith("{")) {
+    return JSON.parse(trimmed);
+  }
+  if (trimmed.includes("event:") || trimmed.includes("data:")) {
+    let payload = "";
+    for (const line of trimmed.split(/\r?\n/)) {
+      if (line.startsWith("data:")) payload += line.slice(5).trim();
+    }
+    if (payload.startsWith("{")) return JSON.parse(payload);
+  }
+  throw new Error(`Unrecognized MCP response format: ${trimmed.slice(0, 200)}`);
+}
+
+export class McpClient {
+  constructor(url) {
+    this.url = url;
+    this.seq = 0;
+  }
+
+  async request(method, params = {}) {
+    this.seq += 1;
+    const res = await fetch(this.url, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Accept: ACCEPT },
+      body: JSON.stringify({ jsonrpc: "2.0", id: this.seq, method, params }),
+    });
+    if (!res.ok) {
+      throw new Error(`MCP HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
+    }
+    const rpc = parseMcpResponse(await res.text());
+    if (rpc.error) throw new Error(`MCP RPC error ${rpc.error.code}: ${rpc.error.message}`);
+    return rpc;
+  }
+
+  async listTools() {
+    const rpc = await this.request("tools/list");
+    return rpc.result?.tools ?? [];
+  }
+
+  async callTool(name, args = {}) {
+    const rpc = await this.request("tools/call", { name, arguments: args });
+    return rpc.result;
+  }
+}
+
+export function textContentFrom(result) {
+  if (!result?.content) return "";
+  return result.content
+    .map((b) => (b.type === "text" ? b.text : JSON.stringify(b)))
+    .join(" ");
+}

apps/eval/lib/ollama.mjs

@@ -0,0 +1,55 @@
+const DEFAULT_URL = "http://localhost:11434";
+
+export class Ollama {
+  constructor({ url = DEFAULT_URL } = {}) {
+    this.url = url.replace(/\/$/, "");
+  }
+
+  async ping() {
+    if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, model: process.env.OLLAMA_MODEL };
+    const res = await fetch(`${this.url}/api/tags`);
+    if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`);
+    const data = await res.json();
+    const models = (data.models ?? []).map((m) => m.name);
+    return { ok: true, models };
+  }
+
+  async chat({ model, messages, tools, temperature = 0, numCtx = 8192 }) {
+    const body = {
+      model,
+      messages,
+      stream: false,
+      options: { temperature },
+    };
+    if (numCtx) body.options.num_ctx = numCtx;
+    if (tools && tools.length) body.tools = tools;
+
+    const res = await fetch(`${this.url}/api/chat`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(body),
+    });
+    if (!res.ok) {
+      const t = await res.text();
+      throw new Error(`Ollama chat HTTP ${res.status}: ${t.slice(0, 300)}`);
+    }
+    const data = await res.json();
+    const message = data.message ?? {};
+    return {
+      role: message.role ?? "assistant",
+      content: message.content ?? "",
+      toolCalls: message.tool_calls ?? [],
+      promptEvalCount: data.prompt_eval_count ?? 0,
+      evalCount: data.eval_count ?? 0,
+      raw: data,
+    };
+  }
+}
+
+export async function listModels() {
+  const o = new Ollama();
+  const res = await fetch(`${o.url}/api/tags`);
+  if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status})`);
+  const data = await res.json();
+  return (data.models ?? []).map((m) => m.name);
+}