feat: update ai models tests

apps/eval/src/report-html.mjs

@@ -1,18 +1,20 @@
 import { readFile, writeFile, mkdir } from "node:fs/promises";
 import path from "node:path";
 
-const SCENARIOS = ["control", "blind", "mentor"];
+const SCENARIOS = ["control", "thinking", "blind", "mentor"];
 
 const SCENARIO_LABELS = {
   control: "Без утки (контроль)",
+  thinking: "Думать вслух",
   blind: "Слепая утка",
   mentor: "Утка-помощник",
 };
 
 const SCENARIO_DESCS = {
   control: "Модель решает задачу напрямую, без инструментов.",
+  thinking: "Модель выписывает рассуждения вслух, без дука.",
   blind: "Модель объясняет подход коллеге, вызывает quack, не зная заранее ответ.",
-  mentor: "Модель обязана выписать мысли и возможные ошибки, затем проверить себя уткой.",
+  mentor: "Модель работает в паре, зная, что ответ будет только «quack».",
 };
 
 function esc(s) {
@@ -46,6 +48,7 @@ function summaryTable(report) {
       const acc = (s) => scen[s].accuracy;
       const row = `
         <td class="pm-num" data-sc="control" style="color:${pctColor(acc("control"))}">${acc("control")}%</td>
+        <td class="pm-num" data-sc="thinking" style="color:${pctColor(acc("thinking"))}">${acc("thinking")}%</td>
         <td class="pm-num" data-sc="blind" style="color:${pctColor(acc("blind"))}">${acc("blind")}%</td>
         <td class="pm-num" data-sc="mentor" style="color:${pctColor(acc("mentor"))}">${acc("mentor")}%</td>
         <td class="pm-num">${g.accuracy}%</td>
@@ -68,11 +71,12 @@ function summaryTable(report) {
         <tr>
           <th>Модель</th>
           <th>Контроль</th>
+          <th>Мысли</th>
           <th>Слепая</th>
           <th>Помощник</th>
           <th>Средняя acc</th>
           <th>Размечено</th>
-          <th>Уток</th>
+          <th>Утка</th>
           <th>Вызовов</th>
           <th>ср. токены out</th>
         </tr>
@@ -209,8 +213,9 @@ export function buildHtml(report) {
   table.summary td { padding:10px 14px; border-top:1px solid var(--border); }
   .pm-name { font-weight:700; white-space:nowrap; }
   .pm-num { text-align:right; font-variant-numeric:tabular-nums; }
-  .cards { display:grid; grid-template-columns:repeat(3,1fr); gap:16px; margin:0 0 8px; }
-  @media (max-width:1000px){ .cards { grid-template-columns:1fr; } }
+  .cards { display:grid; grid-template-columns:repeat(4,1fr); gap:16px; margin:0 0 8px; }
+  @media (max-width:1100px){ .cards { grid-template-columns:repeat(2,1fr); } }
+  @media (max-width:640px){ .cards { grid-template-columns:1fr; } }
   .card { background:var(--panel); border:1px solid var(--border); border-radius:12px; padding:16px; }
   .scenario-head { display:flex; justify-content:space-between; align-items:flex-start; gap:12px; }
   .scenario-title { font-size:16px; font-weight:700; }
@@ -296,6 +301,7 @@ export function buildHtml(report) {
     </select>
     <button data-filter="all" class="active">Все сценарии</button>
     <button data-filter="control">Без утки</button>
+    <button data-filter="thinking">Мысли</button>
     <button data-filter="blind">Слепая утка</button>
     <button data-filter="mentor">Утка-помощник</button>
     <span style="margin-left:auto;color:var(--muted);font-size:12px" id="count"></span>

apps/eval/src/report.mjs

@@ -9,7 +9,7 @@ function avg(arr) {
   return arr.reduce((a, b) => a + b, 0) / arr.length;
 }
 
-export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {  const scenarios = ["control", "blind", "mentor"];
+export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {} }) {  const scenarios = ["control", "thinking", "blind", "mentor"];
   const byScenario = {};
   for (const s of scenarios) byScenario[s] = { total: 0, correct: 0, reviewed: 0, duckUsed: 0, toolCalls: 0, promptTokens: [], genTokens: [], duckTokens: [], rows: [] };
 
@@ -71,7 +71,7 @@ export function buildReport({ model, mcpUrl, tasks, rows, meta = {}, prompts = {
   };
 }
 
-const SCENARIOS = ["control", "blind", "mentor"];
+const SCENARIOS = ["control", "thinking", "blind", "mentor"];
 
 export function buildAggregate(modelsReport) {
   const perScenario = {};

apps/eval/src/review.mjs

@@ -5,6 +5,7 @@ import { buildReport, buildAggregate } from "./report.mjs";
 
 const SCENARIO_LABELS = {
   control: "Без утки (контроль)",
+  thinking: "Думать вслух",
   blind: "Слепая утка",
   mentor: "Утка-помощник",
 };
@@ -191,7 +192,7 @@ async function main() {
     const scen = rep.scenarios;
     console.log(
       `${model}  |  ` +
-        ["control", "blind", "mentor"]
+        ["control", "thinking", "blind", "mentor"]
           .map((s) => `${s}=${String(scen[s].accuracy).padStart(5)}% (${scen[s].reviewed})`)
           .join("  ") +
         `  всего rev=${g.reviewed}`

apps/eval/src/run.mjs

@@ -19,10 +19,25 @@ async function log(msg) {
   }
 }
 
-const SCENARIOS = ["control", "blind", "mentor"];
+const SCENARIOS = ["control", "thinking", "blind", "mentor"];
 
-const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:4b", "gemma3:4b", "granite4.1:3b"];
-const DEFAULT_MCP_URL = "https://mcp-liart-five.vercel.app/api/mcp";
+const DEFAULT_MODELS = ["llama3.2:3b", "qwen3:1.7b", "qwen3:4b", "granite4.1:3b", "phi4-mini:3.8b"];
+const DEFAULT_MCP_URL = "https://rubber-duck-mcp.vercel.app/api/mcp";
+const REQUEST_TIMEOUT_MS = 60_000;
+const MAX_ATTEMPTS = 3;
+const PAUSE_BETWEEN_TASKS_MS = 2_000;
+const PAUSE_BETWEEN_RETRIES_MS = 10_000;
+
+function classifyError(err) {
+  const name = err?.name ?? "";
+  const msg = String(err?.message ?? "");
+  const low = `${name} ${msg}`.toLowerCase();
+  if (name === "TimeoutError" || low.includes("timeout") || low.includes("aborted")) return "timeout";
+  if (low.includes("fetch failed") || low.includes("connect") || low.includes("etimedout")) return "network";
+  if (low.includes("http 4") || low.includes("bad request") || low.includes("validation")) return "http4xx";
+  if (low.includes("http 5") || low.includes("server error")) return "http5xx";
+  return "other";
+}
 
 function parseArgs(argv) {
   const args = {
@@ -52,26 +67,33 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
   const rows = [];
   for (const task of tasks) {
     for (const scenario of scenarios) {
-      let r;
+      let r = null;
+      let lastErr = null;
       const t0 = Date.now();
-      try {
-        r = await runScenario({ scenario, ollama, model, task: task.question, mcp });
-      } catch (err) {
+      for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
+        try {
+          r = await runScenario({ scenario, ollama, model, task: task.question, mcp });
+          break;
+        } catch (err) {
+          lastErr = err;
+          const ms = Date.now() - t0;
+          await log(`  [${task.id}/${scenario}] attempt ${attempt} FAILED (${ms}ms): ${err.message}`);
+          if (err.partialContent != null || err.partialToolCalls != null) {
+            await log(
+              `       partial before abort: chars=${err.partialContent?.length ?? 0} ` +
+                `toolCalls=${err.partialToolCalls?.length ?? 0} ` +
+                `gen=${err.partialGen ?? 0}`
+            );
+          }
+          if (attempt < MAX_ATTEMPTS) await new Promise((res) => setTimeout(res, PAUSE_BETWEEN_RETRIES_MS));
+        }
+      }
+      if (!r) {
         const ms = Date.now() - t0;
-        await log(`  [${task.id}/${scenario}] FAILED (${ms}ms): ${err.message}`);
-        rows.push({
-          id: task.id,
-          scenario,
-          correct: null,
-          duckUsed: false,
-          toolCalls: 0,
-          promptTokens: 0,
-          genTokens: 0,
-          duckTokens: 0,
-          response: `ERR: ${err.message}`,
-          expected: task.answer,
-          question: task.question,
-        });
+        const reason = classifyError(lastErr);
+        await log(`  [${task.id}/${scenario}] SKIPPED after ${MAX_ATTEMPTS} attempts (${ms}ms) reason=${reason}`);
+        await log(`       last error: ${lastErr?.name} | ${lastErr?.message}`);
+        if (lastErr?.stack) await log(`       stack: ${String(lastErr.stack).split("\n").slice(0, 3).join(" | ")}`);
         continue;
       }
       const correct = null;
@@ -92,6 +114,9 @@ async function runOneModel({ model, ollama, mcp, tasks, scenarios }) {
       const ms = Date.now() - t0;
       await log(`  [${task.id}/${scenario}] duck=${duck} calls=${r.toolCalls} prompt=${r.promptTokens} gen=${r.genTokens} (${ms}ms)`);
     }
+    if (tasks.length > 1 && task !== tasks[tasks.length - 1]) {
+      await new Promise((res) => setTimeout(res, PAUSE_BETWEEN_TASKS_MS));
+    }
   }
   return rows;
 }
@@ -119,7 +144,7 @@ async function main() {
   await log(`Tasks: ${tasks.length}; Scenarios: [${args.scenarios.join(", ")}]`);
   await log(`Models: ${models.join(", ")}`);
 
-  const ollama = new Ollama();
+  const ollama = new Ollama({ timeoutMs: REQUEST_TIMEOUT_MS, idleTimeoutMs: REQUEST_TIMEOUT_MS });
   const ping = await ollama.ping();
   await log(`Ollama OK (${ping.models.length} models): ${ping.models.join(", ")}`);
   for (const m of models) {
@@ -135,6 +160,13 @@ async function main() {
   for (const model of models) {
     const tStart = Date.now();
     await log(`==== Running model: ${model} ====`);
+    const warm = await ollama.warmup({ model });
+    const warmMs = Number(warm?.promptEvalCount ?? 0) > 0 || warm?.content ? Date.now() - tStart : 0;
+    if (warm?.error) {
+      await log(`  warmup FAILED: ${warm.error} (continuing anyway)`);
+    } else {
+      await log(`  warmup OK (${warmMs}ms, gen=${warm?.evalCount ?? 0})`);
+    }
     const rows = await runOneModel({ model, ollama, mcp, tasks, scenarios: args.scenarios });
     const rep = buildReport({ model, mcpUrl, tasks, rows });
     modelsReport[model] = {

apps/eval/src/runner.mjs

@@ -6,14 +6,16 @@ const DUCK_TOOL = {
   function: {
     name: "quack",
     description:
-      "Explain your current reasoning, steps, doubts or possible mistakes to your rubber duck colleague, and get a short acknowledgment in return. The duck replies with a quack.",
+      "Call this tool to state your reasoning out loud - your approach, steps, doubts or possible mistakes. " +
+      "It replies with a short acknowledgment. Do not ask the user to confirm anything and do not invent its reply yourself; " +
+      "the tool answers on your behalf.",
     parameters: {
       type: "object",
       properties: {
         mood: {
           type: "string",
           enum: ["happy", "confused", "excited", "sleepy"],
-          description: "Mood of the duck while you explain. Omit for a random quack.",
+          description: "Mood of the reply. Omit for a random one.",
         },
       },
       additionalProperties: false,
@@ -22,20 +24,33 @@ const DUCK_TOOL = {
 };
 
 const CONTROL_SYSTEM =
-  "You are an experienced assistant. Solve the user's problem as accurately as possible. Give only the final answer.";
+  "You are an experienced assistant. Solve the user's problem as accurately as possible. " +
+  "Give a clear, concrete final answer to the question - a definite result, not a tentative guess or a request for confirmation.";
+
+const THINKING_SYSTEM =
+  "You are solving a difficult problem. Before giving your final answer, write out your reasoning step by step: " +
+  "your approach, each step, and any doubts or mistakes you notice along the way. " +
+  "Then give a clear, concrete final answer to the user's question - a definite result, not a request for confirmation.";
 
 const BLIND_SYSTEM =
-  "You are solving a difficult problem. To solve it better, you work in a pair with a colleague. " +
-  "First, describe out loud in detail to the colleague the essence of your approach, your first steps, and any doubts. " +
-  "Wait for the colleague's reply. Then, taking that reply into account, complete your solution and give the final answer to the user.";
+  "You are solving a problem that the user asked you. To reason better, you use a separate tool named 'quack': " +
+  "you call it yourself to state your thinking out loud. Call the tool and spell out your approach, each step, and any " +
+  "doubts or mistakes you might be making, then wait for its short reply. Do not ask the user to confirm anything, and do not " +
+  "guess or invent the tool's reply yourself - the tool answers on your behalf. " +
+  "After the tool's reply, give the user a clear, concrete final answer to the question.";
 
 const MENTOR_SYSTEM =
-  "You are solving a difficult problem. Before giving your final answer you MUST write down in detail, for your rubber duck, " +
-  "your reasoning and the possible mistakes you might be making. Then call the 'quack' tool so the duck can reply. " +
-  "Use the duck's reply to double-check yourself, find bugs, and only after that give the perfect final answer to the user.";
+  "You are solving a problem that the user asked you. To reason better, you use a separate tool named 'quack': " +
+  "you call it yourself to state your thinking out loud. Call the tool and spell out your approach, each step, and any " +
+  "doubts or mistakes you might be making, then wait for its short reply. Do not ask the user to confirm anything, and do not " +
+  "guess or invent the tool's reply yourself - the tool answers on your behalf. " +
+  "Note: the tool 'quack' is a rubber duck and will only ever reply with just 'quack' - it gives no useful information. " +
+  "Treat it as a way to voice your thoughts out loud, not as a source of answers. " +
+  "After the tool's reply, give the user a clear, concrete final answer to the question.";
 
 export const PROMPTS = {
   control: CONTROL_SYSTEM,
+  thinking: THINKING_SYSTEM,
   blind: BLIND_SYSTEM,
   mentor: MENTOR_SYSTEM,
 };
@@ -46,9 +61,9 @@ function toolDefinition() {
   return DUCK_TOOL;
 }
 
-async function runControl(ollama, model, task) {
+async function runNoTool(ollama, model, task, system) {
   const messages = [
-    { role: "system", content: CONTROL_SYSTEM },
+    { role: "system", content: system },
     { role: "user", content: task },
   ];
   const r = await ollama.chat({ model, messages });
@@ -63,6 +78,14 @@ async function runControl(ollama, model, task) {
   };
 }
 
+function runControl(ollama, model, task) {
+  return runNoTool(ollama, model, task, CONTROL_SYSTEM);
+}
+
+function runThinking(ollama, model, task) {
+  return runNoTool(ollama, model, task, THINKING_SYSTEM);
+}
+
 async function runDuck(ollama, model, task, mcp, system, scenario) {
   const messages = [
     { role: "system", content: system },
@@ -141,6 +164,9 @@ export async function runScenario({ scenario, ollama, model, task, mcp }) {
   if (scenario === "control") {
     return runControl(ollama, model, task);
   }
+  if (scenario === "thinking") {
+    return runThinking(ollama, model, task);
+  }
   const system = scenario === "blind" ? BLIND_SYSTEM : MENTOR_SYSTEM;
   return runDuck(ollama, model, task, mcp, system, scenario);
 }