feat: update report to use multiple models

apps/eval/lib/ollama.mjs

@@ -1,20 +1,21 @@
 const DEFAULT_URL = "http://localhost:11434";
 
 export class Ollama {
-  constructor({ url = DEFAULT_URL } = {}) {
+  constructor({ url = DEFAULT_URL, timeoutMs = 600000 } = {}) {
     this.url = url.replace(/\/$/, "");
+    this.timeoutMs = timeoutMs;
   }
 
-  async ping() {
+  async ping({ timeoutMs = 30000 } = {}) {
     if (process.env.OLLAMA_SKIP_PING === "1") return { ok: true, model: process.env.OLLAMA_MODEL };
-    const res = await fetch(`${this.url}/api/tags`);
+    const res = await fetch(`${this.url}/api/tags`, { signal: AbortSignal.timeout(timeoutMs) });
     if (!res.ok) throw new Error(`Ollama not reachable (HTTP ${res.status}) at ${this.url}`);
     const data = await res.json();
     const models = (data.models ?? []).map((m) => m.name);
     return { ok: true, models };
   }
 
-  async chat({ model, messages, tools, temperature = 0, numCtx = 8192 }) {
+  async chat({ model, messages, tools, temperature = 0, numCtx = 8192, timeoutMs }) {
     const body = {
       model,
       messages,
@@ -28,6 +29,7 @@ export class Ollama {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify(body),
+      signal: AbortSignal.timeout(timeoutMs ?? this.timeoutMs),
     });
     if (!res.ok) {
       const t = await res.text();

apps/eval/src/report-html.mjs

@@ -1,6 +1,8 @@
 import { readFile, writeFile, mkdir } from "node:fs/promises";
 import path from "node:path";
 
+const SCENARIOS = ["control", "blind", "mentor"];
+
 const SCENARIO_LABELS = {
   control: "Без утки (контроль)",
   blind: "Слепая утка",
@@ -13,8 +15,6 @@ const SCENARIO_DESCS = {
   mentor: "Модель обязана выписать мысли и возможные ошибки, затем проверить себя уткой.",
 };
 
-const MOOD_COLORS = { happy: "#f59e0b", confused: "#8b5cf6", excited: "#10b981", sleepy: "#64748b" };
-
 function esc(s) {
   return String(s ?? "")
     .replace(/&/g, "&")
@@ -26,10 +26,72 @@ function nl2br(s) {
   return esc(s).replace(/\n/g, "<br>");
 }
 
-function scenarioCard(key, g) {
-  const pct = g.accuracy;
-  const color = pct >= 70 ? "#10b981" : pct >= 40 ? "#f59e0b" : "#ef4444";
-  const duckPct = g.tasks ? Math.round((g.duckUsed / g.tasks) * 100) : 0;
+function pctColor(p) {
+  return p >= 70 ? "#10b981" : p >= 40 ? "#f59e0b" : "#ef4444";
+}
+
+function modelShort(name) {
+  const base = String(name).split(/:/)[0];
+  const ver = String(name).split(/:/)[1] ? ":" + String(name).split(/:/)[1] : "";
+  return esc(base + ver);
+}
+
+function summaryTable(report) {
+  const perModel = report.aggregate.perModel;
+  const models = Object.keys(perModel);
+  const rows = models
+    .map((m) => {
+      const g = perModel[m];
+      const scen = report.models[m].scenarios;
+      const acc = (s) => scen[s].accuracy;
+      return `
+      <tr>
+        <td class="pm-name" data-model="${esc(m)}">${modelShort(m)}</td>
+        <td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td>
+        <td class="pm-num" data-sc="blind" style="color:${pctColor(acc("blind"))}">${acc("blind")}%</td>
+        <td class="pm-num" data-sc="mentor" style="color:${pctColor(acc("mentor"))}">${acc("mentor")}%</td>
+        <td class="pm-num">${g.accuracy}%</td>
+        <td class="pm-num">${g.duckUsed}</td>
+        <td class="pm-num">${g.toolCalls}</td>
+        <td class="pm-num">${g.avgGenTokens}</td>
+      </tr>`;
+    })
+    .join("");
+  return `
+  <div class="summary-shell">
+    <table class="summary">
+      <thead>
+        <tr>
+          <th>Модель</th>
+          <th>Контроль</th>
+          <th>Слепая</th>
+          <th>Помощник</th>
+          <th>Средняя acc</th>
+          <th>Уток</th>
+          <th>Вызовов</th>
+          <th>ср. токены out</th>
+        </tr>
+      </thead>
+      <tbody>${rows}</tbody>
+    </table>
+  </div>`;
+}
+
+function scenarioCard(key, agg) {
+  const pct = agg.accuracy;
+  const color = pctColor(pct);
+  const duckPct = agg.tasks ? Math.round((agg.duckUsed / agg.tasks) * 100) : 0;
+  const modelBars = Object.entries(agg.byModel)
+    .map(([m, g]) => {
+      const c = pctColor(g.accuracy);
+      return `
+      <div class="agg-model">
+        <span class="agg-model-name">${modelShort(m)}</span>
+        <div class="bar"><div class="bar-fill" style="width:${g.accuracy}%;background:${c}"></div></div>
+        <span class="agg-model-val" style="color:${c}">${g.accuracy}%</span>
+      </div>`;
+    })
+    .join("");
   return `
   <div class="card scenario" data-scenario="${key}">
     <div class="scenario-head">
@@ -41,13 +103,11 @@ function scenarioCard(key, g) {
     </div>
     <div class="bar"><div class="bar-fill" style="width:${pct}%;background:${color}"></div></div>
     <div class="metrics">
-      <div class="metric"><span class="m-label">Верных</span><span class="m-val">${g.correct} <span class="m-mod">/ ${g.tasks}</span></span></div>
-      <div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${g.duckUsed} из ${g.tasks} задач</span></div>
-      <div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${g.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
-      <div class="metric"><span class="m-label">Промпт (in)</span><span class="m-val">${g.avgPromptTokens}</span><span class="m-sub">токенов</span></div>
-      <div class="metric"><span class="m-label">Ответ (out)</span><span class="m-val">${g.avgGenTokens}</span><span class="m-sub">токенов</span></div>
-      <div class="metric"><span class="m-label">Рассужд.</span><span class="m-val">${g.avgDuckTokens}</span><span class="m-sub">токенов</span></div>
+      <div class="metric"><span class="m-label">Верных</span><span class="m-val">${agg.correct} <span class="m-mod">/ ${agg.tasks}</span></span></div>
+      <div class="metric"><span class="m-label">Утка (MCP)</span><span class="m-val">${duckPct}%</span><span class="m-sub">в ${agg.duckUsed} из ${agg.tasks} задач</span></div>
+      <div class="metric"><span class="m-label">quack / задачу</span><span class="m-val">${agg.avgCalls}</span><span class="m-sub">вызовов в среднем</span></div>
     </div>
+    <div class="agg-models">${modelBars}</div>
   </div>`;
 }
 
@@ -55,7 +115,8 @@ function rowHtml(r) {
   const cls = r.correct ? "ok" : "no";
   const duck = r.duckUsed ? "🦆" : "—";
   return `
-  <tr class="scenario-row ${cls}" data-scenario="${r.scenario}">
+  <tr class="scenario-row ${cls}" data-scenario="${r.scenario}" data-model="${esc(r.model)}">
+    <td class="r-model">${modelShort(r.model)}</td>
     <td class="r-id">${esc(r.id)}</td>
     <td class="r-s">${esc(SCENARIO_LABELS[r.scenario])}</td>
     <td class="r-result">${r.correct ? "✓" : "✗"}</td>
@@ -69,8 +130,8 @@ function rowHtml(r) {
 
 function promptBlock(report) {
   const p = report.prompts || {};
-  const items = ["control", "blind", "mentor"]
-    .filter((k) => p[k] && report.scenarios[k])
+  const items = SCENARIOS
+    .filter((k) => p[k])
     .map(
       (k) => `
       <div class="prompt-item">
@@ -88,13 +149,18 @@ function promptBlock(report) {
 }
 
 export function buildHtml(report) {
-  const s = report.scenarios;
-  const cards = ["control", "blind", "mentor"]
-    .filter((k) => s[k])
-    .map((k) => scenarioCard(k, s[k]))
+  const perTask = Object.entries(report.models).flatMap(([model, rep]) =>
+    rep.perTask.map((r) => ({ ...r, model }))
+  );
+  const rows = perTask.map(rowHtml).join("");
+  const models = Object.keys(report.aggregate.perModel);
+  const modelOptions = models
+    .map((m) => `<option value="${esc(m)}">${modelShort(m)}</option>`)
+    .join("");
+  const aggCards = SCENARIOS
+    .filter((k) => report.aggregate.perScenario[k])
+    .map((k) => scenarioCard(k, report.aggregate.perScenario[k]))
     .join("");
-
-  const rows = report.perTask.map(rowHtml).join("");
 
   return `<!doctype html>
 <html lang="ru">
@@ -110,12 +176,20 @@ export function buildHtml(report) {
   * { box-sizing:border-box; }
   body { margin:0; font-family:-apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif;
     background:var(--bg); color:var(--text); line-height:1.5; }
-  .wrap { max-width:1200px; margin:0 auto; padding:24px 20px 60px; }
+  .wrap { max-width:1280px; margin:0 auto; padding:24px 20px 60px; }
   header h1 { margin:0 0 4px; font-size:22px; }
   .meta { color:var(--muted); font-size:13px; }
   .meta b { color:var(--text); font-weight:600; }
-  .cards { display:grid; grid-template-columns:repeat(3,1fr); gap:16px; margin:24px 0; }
-  @media (max-width:900px){ .cards { grid-template-columns:1fr; } }
+  h2 { font-size:17px; margin:26px 0 12px; }
+  .summary-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; }
+  table.summary { width:100%; border-collapse:collapse; font-size:13px; }
+  table.summary th { background:var(--panel2); text-align:left; padding:10px 14px;
+    font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; }
+  table.summary td { padding:10px 14px; border-top:1px solid var(--border); }
+  .pm-name { font-weight:700; white-space:nowrap; }
+  .pm-num { text-align:right; font-variant-numeric:tabular-nums; }
+  .cards { display:grid; grid-template-columns:repeat(3,1fr); gap:16px; margin:0 0 8px; }
+  @media (max-width:1000px){ .cards { grid-template-columns:1fr; } }
   .card { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; }
   .scenario-head { display:flex; justify-content:space-between; align-items:flex-start; gap:12px; }
   .scenario-title { font-size:16px; font-weight:700; }
@@ -129,11 +203,12 @@ export function buildHtml(report) {
   .m-val { display:block; font-size:20px; font-weight:800; margin-top:2px; }
   .m-mod { font-size:14px; font-weight:600; color:var(--muted); }
   .m-sub { display:block; color:var(--muted); font-size:11px; margin-top:2px; }
-  .filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
-  .filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
-    padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
-  .filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
-  .prompts { margin:0 0 20px; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
+  .agg-models { margin-top:14px; display:grid; gap:8px; }
+  .agg-model { display:grid; grid-template-columns:auto 1fr auto; align-items:center; gap:10px; font-size:12px; }
+  .agg-model .bar { margin:0; }
+  .agg-model-name { color:var(--muted); white-space:nowrap; }
+  .agg-model-val { font-weight:700; font-variant-numeric:tabular-nums; }
+  .prompts { margin:20px 0; background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
   .prompts summary { cursor:pointer; padding:12px 16px; font-weight:700; font-size:14px; list-style:none; }
   .prompts summary::-webkit-details-marker { display:none; }
   .prompts summary::before { content:"▸ "; color:var(--muted); }
@@ -142,21 +217,27 @@ export function buildHtml(report) {
   .prompt-item { background:var(--panel2); border-radius:8px; padding:10px 12px; }
   .prompt-name { font-size:12px; color:var(--muted); text-transform:uppercase; letter-spacing:.05em; margin-bottom:6px; }
   .prompt-text { white-space:pre-wrap; font-size:13px; line-height:1.5; }
-  .tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:hidden; }
+  .filters { margin:8px 0 16px; display:flex; gap:8px; flex-wrap:wrap; align-items:center; }
+  .filters button { background:var(--panel); color:var(--text); border:1px solid var(--border);
+    padding:7px 14px; border-radius:8px; cursor:pointer; font-size:13px; }
+  .filters button.active { background:#3b82f6; border-color:#3b82f6; color:#fff; }
+  .filters select { background:var(--panel); color:var(--text); border:1px solid var(--border);
+    padding:7px 10px; border-radius:8px; font-size:13px; }
+  .tbl-shell { background:var(--panel); border:1px solid var(--border); border-radius:12px; overflow:auto; }
   table { width:100%; border-collapse:collapse; font-size:13px; }
   thead th { position:sticky; top:0; background:var(--panel2); text-align:left; padding:10px 12px;
-    font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); }
+    font-size:11px; text-transform:uppercase; letter-spacing:.05em; color:var(--muted); white-space:nowrap; }
   tbody td { padding:10px 12px; border-top:1px solid var(--border); vertical-align:top; }
   .scenario-row.ok .r-result { color:var(--ok); font-weight:800; }
   .scenario-row.no .r-result { color:var(--no); font-weight:800; }
+  .r-model { font-weight:600; white-space:nowrap; }
   .r-id { font-weight:700; color:var(--muted); }
   .r-s { white-space:nowrap; }
   .r-duck { text-align:center; white-space:nowrap; }
-  .r-question { max-width:340px; color:var(--text); }
-  .r-answer { max-width:420px; }
+  .r-question { max-width:300px; color:var(--text); }
+  .r-answer { max-width:380px; }
   .muted { color:var(--muted); }
   .hidden { display:none !important; }
-  .toggle-row { background:transparent; border:none; color:#3b82f6; cursor:pointer; font-size:12px; padding:0; }
   footer { margin-top:20px; color:var(--muted); font-size:12px; }
 </style>
 </head>
@@ -165,19 +246,27 @@ export function buildHtml(report) {
   <header>
     <h1>🦆 Rubber Duck MCP — отчёт тестирования</h1>
     <div class="meta">
-      Модель: <b>${esc(report.model)}</b> &nbsp;·&nbsp;
       MCP: <b>${esc(report.mcpUrl)}</b> &nbsp;·&nbsp;
-      Задач: <b>${s.control?.tasks ?? report.perTask.length}</b> &nbsp;·&nbsp;
+      Моделей: <b>${models.length}</b> &nbsp;·&nbsp;
       Дата: <b>${esc(report.generatedAt)}</b>
     </div>
   </header>
 
-  <div class="cards">${cards}</div>
+  <h2>Сводка по моделям</h2>
+  ${summaryTable(report)}
+
+  <h2>Агрегат по сценариям (все модели)</h2>
+  <div class="cards">${aggCards}</div>
 
   ${promptBlock(report)}
 
   <div class="filters" id="filters">
-    <button data-filter="all" class="active">Все</button>
+    <label style="font-size:12px;color:var(--muted)">Модель:</label>
+    <select id="modelSel">
+      <option value="all">Все модели</option>
+      ${modelOptions}
+    </select>
+    <button data-filter="all" class="active">Все сценарии</button>
     <button data-filter="control">Без утки</button>
     <button data-filter="blind">Слепая утка</button>
     <button data-filter="mentor">Утка-помощник</button>
@@ -188,7 +277,7 @@ export function buildHtml(report) {
     <table>
       <thead>
         <tr>
-          <th>#</th><th>Сценарий</th><th>✓</th><th>Утка</th>
+          <th>Модель</th><th>#</th><th>Сценарий</th><th>✓</th><th>Утка</th>
           <th>Вопрос</th><th>Ожид.</th><th>in/out</th><th>Ответ модели</th>
         </tr>
       </thead>
@@ -196,17 +285,22 @@ export function buildHtml(report) {
     </table>
   </div>
 
-  <footer>Генерировано локальным харнессом apps/eval. temperature=0, llama3.2:3b в данном прогоне.</footer>
+  <footer>Генерировано локальным харнессом apps/eval. temperature=0. Прогон по нескольким локальным моделям через Ollama.</footer>
 </div>
 
 <script>
   const buttons = document.querySelectorAll('#filters button');
+  const modelSel = document.getElementById('modelSel');
   const rows2 = document.querySelectorAll('tbody .scenario-row');
   const count = document.getElementById('count');
-  function apply(f) {
+  let sc = 'all';
+  let mdl = 'all';
+  function apply() {
     let n = 0;
     rows2.forEach(r => {
-      const show = f === 'all' || r.dataset.scenario === f;
+      const okSc = sc === 'all' || r.dataset.scenario === sc;
+      const okM = mdl === 'all' || r.dataset.model === mdl;
+      const show = okSc && okM;
       r.classList.toggle('hidden', !show);
       if (show) n++;
     });
@@ -215,8 +309,10 @@ export function buildHtml(report) {
   buttons.forEach(b => b.addEventListener('click', () => {
     buttons.forEach(x => x.classList.remove('active'));
     b.classList.add('active');
-    apply(b.dataset.filter);
+    sc = b.dataset.filter;
+    apply();
   }));
+  modelSel.addEventListener('change', () => { mdl = modelSel.value; apply(); });
   apply('all');
 </script>
 </body>

apps/eval/src/report.mjs

@@ -11,8 +11,7 @@ function avg(arr) {
   return arr.reduce((a, b) => a + b, 0) / arr.length;
 }
 
-export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {
-  const scenarios = ["control", "blind", "mentor"];
+export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {  const scenarios = ["control", "blind", "mentor"];
   const byScenario = {};
   for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
 
@@ -67,3 +66,65 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
     meta,
   };
 }
+
+const SCENARIOS = ["control", "blind", "mentor"];
+
+export function buildAggregate(modelsReport) {
+  const perScenario = {};
+  for (const s of SCENARIOS) {
+    let total = 0, correct = 0, duckUsed = 0, toolCalls = 0;
+    const gen = [];
+    const byModel = {};
+    for (const [name, rep] of Object.entries(modelsReport)) {
+      const g = rep.scenarios[s];
+      if (!g) continue;
+      total += g.tasks;
+      correct += g.correct;
+      duckUsed += g.duckUsed;
+      toolCalls += g.toolCalls;
+      byModel[name] = {
+        tasks: g.tasks,
+        correct: g.correct,
+        accuracy: g.accuracy,
+        duckUsed: g.duckUsed,
+      };
+    }
+    perScenario[s] = {
+      tasks: total,
+      correct,
+      accuracy: total ? round((correct / total) * 100, 1) : 0,
+      duckUsed,
+      toolCalls,
+      avgCalls: total ? round(toolCalls / total, 2) : 0,
+      byModel,
+    };
+  }
+
+  const perModel = {};
+  for (const [name, rep] of Object.entries(modelsReport)) {
+    let duckUsed = 0, toolCalls = 0;
+    const gen = [];
+    for (const s of SCENARIOS) {
+      const g = rep.scenarios[s];
+      duckUsed += g.duckUsed;
+      toolCalls += g.toolCalls;
+      for (const row of rep.perTask.filter((r) => r.scenario === s)) {
+        gen.push(row.genTokens ?? 0);
+      }
+    }
+    const taskCount = rep.scenarios.control?.tasks ?? 0;
+    const correct = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].correct || 0), 0);
+    const totalRows = SCENARIOS.reduce((acc, s) => acc + (rep.scenarios[s].tasks || 0), 0);
+    perModel[name] = {
+      tasks: taskCount,
+      totalRows,
+      correct,
+      accuracy: totalRows ? round((correct / totalRows) * 100, 1) : 0,
+      duckUsed,
+      toolCalls,
+      avgGenTokens: round(avg(gen)),
+    };
+  }
+
+  return { perScenario, perModel };
+}

apps/eval/src/run.mjs

@@ -1,17 +1,44 @@
-import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { readFile, writeFile, mkdir, appendFile } from "node:fs/promises";
 import path from "node:path";
 import { Ollama } from "../lib/ollama.mjs";
 import { McpClient } from "../lib/mcpClient.mjs";
 import { runScenario } from "./runner.mjs";
 import { isCorrect, PROMPTS } from "./runner.mjs";
-import { buildReport } from "./report.mjs";
+import { buildReport, buildAggregate } from "./report.mjs";
+
+let logTarget = null;
+async function log(msg) {
+  const line = `[${new Date().toISOString()}] ${msg}`;
+  console.log(line);
+  if (logTarget) {
+    try {
+      await appendFile(logTarget, line + "\n", "utf8");
+    } catch {
+      /* keep going */
+    }
+  }
+}
+
+const SCENARIOS = ["control", "blind", "mentor"];
+
+const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:4b", "gemma3:4b", "granite4.1:3b"];
+const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp";
 
 function parseArgs(argv) {
-  const args = { scenarios: ["control", "blind", "mentor"], limit: Infinity, model: process.env.OLLAMA_MODEL || null };
+  const args = {
+    models: [],
+    scenarios: SCENARIOS,
+    limit: Infinity,
+    model: process.env.OLLAMA_MODEL || null,
+    mcpUrl: null,
+    tasksFile: null,
+    out: null,
+  };
   for (let i = 0; i < argv.length; i += 1) {
     const a = argv[i];
     const next = () => argv[i + 1];
-    if (a === "--model") args.model = next(), i += 1;
+    if (a === "--models") args.models = next().split(",").map((s) => s.trim()).filter(Boolean), i += 1;
+    else if (a === "--model") args.model = next(), i += 1;
     else if (a === "--mcp-url") args.mcpUrl = next(), i += 1;
     else if (a === "--scenarios") args.scenarios = next().split(","), i += 1;
     else if (a === "--limit") args.limit = Number(next()), i += 1;
@@ -21,44 +48,17 @@ function parseArgs(argv) {
   return args;
 }
 
-const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp";
-
-async function main() {
-  const args = parseArgs(process.argv.slice(2));
-  if (!args.model) {
-    console.error("Usage: node src/run.mjs --model <ollama-model> [--mcp-url URL] [--scenarios c,b,m] [--limit N] [--tasks file] [--out file]");
-    process.exit(1);
-  }
-  const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL;
-  const tasksFile = args.tasksFile || path.join("data", "tasks.json");
-
-  const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8"));
-  const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit);
-
-  console.log(`\nModel: ${args.model}`);
-  console.log(`MCP endpoint: ${mcpUrl}`);
-  console.log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`);
-
-  const ollama = new Ollama();
-  const ping = await ollama.ping();
-  console.log(`Ollama OK (${ping.models.length} models)`);
-  const modelPresent = ping.models.some((m) => m.startsWith(args.model));
-  if (!modelPresent) {
-    console.warn(`  (warning) model "${args.model}" not found among: ${ping.models.join(", ")}`);
-  }
-
-  const mcp = new McpClient(mcpUrl);
-  const mcpTools = await mcp.listTools();
-  console.log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`);
-
+async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
   const rows = [];
   for (const task of tasks) {
-    for (const scenario of args.scenarios) {
+    for (const scenario of scenarios) {
       let r;
+      const t0 = Date.now();
       try {
-        r = await runScenario({ scenario, ollama, model: args.model, task: task.question, mcp });
+        r = await runScenario({ scenario, ollama, model, task: task.question, mcp });
       } catch (err) {
-        console.error(`  [${task.id}/${scenario}] FAILED: ${err.message}`);
+        const ms = Date.now() - t0;
+        await log(`  [${task.id}/${scenario}] FAILED (${ms}ms): ${err.message}`);
         rows.push({
           id: task.id,
           scenario,
@@ -90,21 +90,83 @@ async function main() {
       });
       const mark = correct ? "OK " : "XX ";
       const duck = r.duckUsed ? "duck" : "n/a ";
-      console.log(`${scenario.padEnd(8)} ${mark} ${task.id} duck=${duck} prompt=${r.promptTokens} gen=${r.genTokens} | ${r.response.slice(0, 70).replace(/\n/g, " ")}`);
+      const ms = Date.now() - t0;
+      await log(`  [${task.id}/${scenario}] ${mark} duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`);
     }
   }
+  return rows;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+
+  let models = args.models;
+  if (args.model && !models.length) models = [args.model];
+  if (!models.length) models = DEFAULT_MODELS;
+
+  const mcpUrl = args.mcpUrl || DEFAULT_MCP_URL;
+  const tasksFile = args.tasksFile || path.join("data", "tasks.json");
+
+  const tasksRaw = JSON.parse(await readFile(tasksFile, "utf8"));
+  const tasks = tasksRaw.slice(0, args.limit === Infinity ? tasksRaw.length : args.limit);
+
+  const out = args.out || path.join("out", "report.json");
+  const logFile = args.logTarget || path.join(path.dirname(out), "run.log");
+  logTarget = logFile;
+  await mkdir(path.dirname(logFile), { recursive: true });
+  await appendFile(logFile, "", "utf8").catch(() => {});
+
+  await log(`MCP endpoint: ${mcpUrl}`);
+  await log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`);
+  await log(`Models: ${models.join(", ")}`);
+
+  const ollama = new Ollama();
+  const ping = await ollama.ping();
+  await log(`Ollama OK (${ping.models.length} models): ${ping.models.join(", ")}`);
+  for (const m of models) {
+    const present = ping.models.some((x) => x.startsWith(m));
+    await log(`  ${present ? "OK " : "MISSING "} ${m}`);
+  }
+
+  const mcp = new McpClient(mcpUrl);
+  const mcpTools = await mcp.listTools();
+  await log(`MCP tools: ${mcpTools.map((t) => t.name).join(", ")}`);
+
+  const modelsReport = {};
+  for (const model of models) {
+    const tStart = Date.now();
+    await log(`==== Running model: ${model} ====`);
+    const rows = await runOneModel({ model, ollama, mcp, tasks, scenarios: args.scenarios });
+    const rep = buildReport({ model, mcpUrl, tasks, rows });
+    modelsReport[model] = {
+      model,
+      scenarios: rep.scenarios,
+      perTask: rep.perTask,
+    };
+    await log(`==== Done ${model} in ${Math.round((Date.now() - tStart) / 1000)}s ====`);
+  }
+
+  const aggregate = buildAggregate(modelsReport);
+
+  const report = {
+    generatedAt: new Date().toISOString(),
+    mcpUrl,
+    prompts: PROMPTS,
+    models: modelsReport,
+    aggregate,
+  };
 
-  const report = buildReport({ model: args.model, mcpUrl, tasks, rows, prompts: PROMPTS });
-  const out = args.out || path.join("out", "report.json");
   await mkdir(path.dirname(out), { recursive: true });
   await writeFile(out, JSON.stringify(report, null, 2), "utf8");
-  console.log(`\nReport written: ${out}`);
-  console.log("\n--- Summary ---");
-  for (const [s, g] of Object.entries(report.scenarios)) {
-    console.log(
-      `${s.padEnd(8)} accuracy=${String(g.accuracy).padStart(5)}%  (${g.correct}/${g.tasks})  ` +
-        `duckUsed=${g.duckUsed}  toolCalls=${g.toolCalls}  ` +
-        `avgPrompt=${g.avgPromptTokens}  avgGen=${g.avgGenTokens}  avgDuck=${g.avgDuckTokens}`
+  await log(`Report written: ${out}`);
+
+  await log("==== Aggregate summary ====");
+  for (const [modelName, g] of Object.entries(aggregate.perModel)) {
+    const scen = modelsReport[modelName].scenarios;
+    await log(
+      `${modelName}  |  ` +
+        SCENARIOS.map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}%`).join("  ") +
+        `  duck=${g.duckUsed}  calls=${g.toolCalls}  avgGen=${g.avgGenTokens}`
     );
   }
 }