feat(dashboard): compare providers side by side

apps/cli/src/commands/results/manifest.ts

@@ -449,8 +449,20 @@ export interface LightweightResultRecord {
   readonly executionStatus?: string;
   readonly error?: string;
   readonly costUsd?: number;
+  readonly durationMs?: number;
+  readonly tokenUsage?: {
+    readonly input?: number;
+    readonly output?: number;
+    readonly reasoning?: number;
+  };
   readonly timestamp?: string;
   readonly runtimeSource?: RunRuntimeSourceMetadata;
+  readonly resultDir?: string;
+  readonly outputPath?: string;
+  readonly answerPath?: string;
+  readonly gradingPath?: string;
+  readonly transcriptPath?: string;
+  readonly metricsPath?: string;
 }
 
 /**
@@ -488,7 +500,15 @@ export function loadLightweightResults(sourceFile: string): LightweightResultRec
     executionStatus: record.execution_status,
     error: record.error,
     costUsd: record.cost_usd,
+    durationMs: record.duration_ms,
+    tokenUsage: record.token_usage,
     timestamp: record.timestamp,
     runtimeSource: record.runtime_source,
+    resultDir: record.result_dir,
+    outputPath: record.output_path,
+    answerPath: record.answer_path,
+    gradingPath: record.grading_path,
+    transcriptPath: record.transcript_path,
+    metricsPath: record.metrics_path,
   }));
 }

apps/cli/src/commands/results/serve.ts

@@ -2438,10 +2438,69 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont
 
   type CompareTestEntry = {
     test_id: string;
+    target: string;
     category?: string;
     score: number;
     passed: boolean;
     execution_status?: string;
+    answer?: string;
+    answer_path?: string;
+    output_path?: string;
+    grading_path?: string;
+    transcript_path?: string;
+    metrics_path?: string;
+    result_dir?: string;
+    duration_ms?: number;
+    token_usage?: {
+      input?: number;
+      output?: number;
+      reasoning?: number;
+    };
+  };
+
+  const readAnswerPreview = (
+    manifestPath: string,
+    relativePath: string | undefined,
+  ): string | undefined => {
+    if (!relativePath) return undefined;
+    const resolved = resolveReadableRunArtifactFile(
+      runWorkspaceDirFromManifestPath(manifestPath),
+      relativePath,
+    );
+    if (!resolved.absolutePath) return undefined;
+    try {
+      const text = readFileSync(resolved.absolutePath, 'utf8').trim();
+      const maxLength = 2000;
+      return text.length > maxLength ? `${text.slice(0, maxLength)}…` : text;
+    } catch {
+      return undefined;
+    }
+  };
+
+  const toCompareTestEntry = (
+    manifestPath: string,
+    record: Awaited<ReturnType<typeof loadLightweightResultsForMeta>>[number],
+    passed: boolean,
+  ): CompareTestEntry => {
+    const target = record.target ?? 'default';
+    const answer = readAnswerPreview(manifestPath, record.answerPath ?? record.outputPath);
+    return {
+      test_id: record.testId,
+      target,
+      ...(record.category && { category: normalizeCategoryPath(record.category) }),
+      score: record.score,
+      passed,
+      execution_status: record.executionStatus,
+      ...(answer && { answer }),
+      ...(record.answerPath && { answer_path: record.answerPath }),
+      ...(record.outputPath && { output_path: record.outputPath }),
+      ...(record.gradingPath && { grading_path: record.gradingPath }),
+      ...(record.transcriptPath && { transcript_path: record.transcriptPath }),
+      ...(record.metricsPath && { metrics_path: record.metricsPath }),
+      ...(record.resultDir && { result_dir: record.resultDir }),
+      ...(record.durationMs !== undefined && { duration_ms: record.durationMs }),
+      ...(record.tokenUsage && { token_usage: record.tokenUsage }),
+    };
   };
 
   // Collect per-test-case results keyed by experiment × target (aggregated view)
@@ -2466,6 +2525,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont
     started_at: string;
     experiment: string;
     target: string;
+    targets: string[];
     source: 'local' | 'remote';
     eval_count: number;
     quality_count: number;
@@ -2484,6 +2544,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont
     try {
       const records = await loadLightweightResultsForMeta(searchDir, m, projectId);
       const runTestMap = new Map<string, CompareTestEntry>();
+      const runTargetsSet = new Set<string>();
       let runEvalCount = 0;
       let runQualityCount = 0;
       let runPassedCount = 0;
@@ -2501,6 +2562,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont
         const target = r.target ?? 'default';
         experimentsSet.add(experiment);
         targetsSet.add(target);
+        runTargetsSet.add(target);
         runExperiment = experiment;
         runTarget = target;
         if (r.timestamp && r.timestamp < runStartedAt) runStartedAt = r.timestamp;
@@ -2526,23 +2588,12 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont
           if (passed) entry.passedCount++;
           entry.scoreSum += r.score;
         }
-        entry.tests.push({
-          test_id: r.testId,
-          ...(r.category && { category: normalizeCategoryPath(r.category) }),
-          score: r.score,
-          passed,
-          execution_status: r.executionStatus,
-        });
+        entry.tests.push(toCompareTestEntry(m.path, r, passed));
         cellMap.set(key, entry);
 
-        // Per-run accumulation. Dedupe tests within the run by last-wins.
-        runTestMap.set(r.testId, {
-          test_id: r.testId,
-          ...(r.category && { category: normalizeCategoryPath(r.category) }),
-          score: r.score,
-          passed,
-          execution_status: r.executionStatus,
-        });
+        // Per-run accumulation. Dedupe each provider/model result within the
+        // run by last-wins while preserving the provider axis for comparison.
+        runTestMap.set(`${r.testId}::${target}`, toCompareTestEntry(m.path, r, passed));
         runEvalCount++;
         if (isExecutionError) {
           runExecutionErrorCount++;
@@ -2556,11 +2607,13 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont
       if (runEvalCount === 0) continue;
 
       const runTests = [...runTestMap.values()].slice(-MAX_TESTS_PER_CELL);
+      const runTargets = [...runTargetsSet].sort();
       runEntries.push({
         run_id: m.filename,
         started_at: runStartedAt,
         experiment: runExperiment,
-        target: runTarget,
+        target: runTargets.length > 1 ? `${runTargets.length} providers` : runTarget,
+        targets: runTargets,
         source: m.source,
         eval_count: runEvalCount,
         quality_count: runQualityCount,

apps/cli/test/commands/results/serve.test.ts

@@ -4018,13 +4018,21 @@ describe('serve app', () => {
         experiment: string;
         target: string;
         eval_count: number;
-        tests?: Array<{ test_id: string; category?: string }>;
+        tests?: Array<{ test_id: string; target?: string; category?: string; answer?: string }>;
       }>;
       runs?: Array<{
         run_id: string;
         experiment: string;
         target: string;
-        tests?: Array<{ test_id: string; category?: string }>;
+        targets?: string[];
+        tests?: Array<{
+          test_id: string;
+          target?: string;
+          category?: string;
+          answer?: string;
+          answer_path?: string;
+          result_dir?: string;
+        }>;
       }>;
     };
 
@@ -4055,6 +4063,53 @@ describe('serve app', () => {
       expect(cell?.tests?.[0]?.category).toBe('prompting');
       expect(run?.tests?.[0]?.category).toBe('prompting');
     });
+
+    it('preserves provider/model results and answer evidence inside a multi-provider run', async () => {
+      const runsDir = localResultsExperimentDir(tempDir);
+      const runDir = path.join(runsDir, '2026-04-05T10-00-00-000Z');
+      mkdirSync(path.join(runDir, '.internal'), { recursive: true });
+      mkdirSync(path.join(runDir, 'case-gpt-4o', 'sample-1', 'outputs'), { recursive: true });
+      mkdirSync(path.join(runDir, 'case-claude', 'sample-1', 'outputs'), { recursive: true });
+      writeFileSync(path.join(runDir, 'case-gpt-4o', 'sample-1', 'outputs', 'answer.md'), 'A');
+      writeFileSync(path.join(runDir, 'case-claude', 'sample-1', 'outputs', 'answer.md'), 'B');
+      writeFileSync(
+        path.join(runDir, '.internal', 'index.jsonl'),
+        toJsonl(
+          {
+            ...RESULT_A,
+            test_id: 'same-case',
+            experiment: 'model-compare',
+            target: 'gpt-4o',
+            result_dir: 'case-gpt-4o',
+            answer_path: 'case-gpt-4o/sample-1/outputs/answer.md',
+          },
+          {
+            ...RESULT_A,
+            test_id: 'same-case',
+            experiment: 'model-compare',
+            target: 'claude',
+            result_dir: 'case-claude',
+            answer_path: 'case-claude/sample-1/outputs/answer.md',
+            score: 0.5,
+          },
+        ),
+      );
+      const app = createApp([], tempDir, tempDir, undefined, { studioDir });
+
+      const res = await app.request('/api/compare');
+      expect(res.status).toBe(200);
+      const data = (await res.json()) as CompareJson;
+      const run = data.runs?.find((entry) => entry.run_id === '2026-04-05T10-00-00-000Z');
+
+      expect(run?.target).toBe('2 providers');
+      expect(run?.targets).toEqual(['claude', 'gpt-4o']);
+      expect(run?.tests?.map((test) => [test.test_id, test.target, test.answer])).toEqual([
+        ['same-case', 'gpt-4o', 'A'],
+        ['same-case', 'claude', 'B'],
+      ]);
+      expect(run?.tests?.[0]?.answer_path).toBe('case-gpt-4o/sample-1/outputs/answer.md');
+      expect(run?.tests?.[0]?.result_dir).toBe('case-gpt-4o');
+    });
   });
 
   // ── SPA fallback ──────────────────────────────────────────────────────

apps/dashboard/src/components/AnalyticsTab.test.tsx

@@ -0,0 +1,115 @@
+import { describe, expect, it } from 'bun:test';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { renderToStaticMarkup } from 'react-dom/server';
+
+import type { CompareResponse } from '~/lib/types';
+
+import { AnalyticsTab } from './AnalyticsTab';
+
+function compareResponse(): CompareResponse {
+  return {
+    experiments: ['model-compare'],
+    targets: ['gpt-5.4', 'gpt-5.4-mini'],
+    cells: [
+      {
+        experiment: 'model-compare',
+        target: 'gpt-5.4',
+        eval_count: 1,
+        quality_count: 1,
+        passed_count: 1,
+        pass_rate: 1,
+        avg_score: 1,
+        tests: [
+          {
+            test_id: 'exact-token',
+            target: 'gpt-5.4',
+            score: 1,
+            passed: true,
+            execution_status: 'ok',
+            answer: 'AV94NI_OK',
+          },
+        ],
+      },
+      {
+        experiment: 'model-compare',
+        target: 'gpt-5.4-mini',
+        eval_count: 1,
+        quality_count: 1,
+        passed_count: 1,
+        pass_rate: 1,
+        avg_score: 1,
+        tests: [
+          {
+            test_id: 'exact-token',
+            target: 'gpt-5.4-mini',
+            score: 1,
+            passed: true,
+            execution_status: 'ok',
+            answer: 'AV94NI_OK',
+          },
+        ],
+      },
+    ],
+    runs: [
+      {
+        run_id: '2026-07-07T14-06-40-813Z',
+        started_at: '2026-07-07T14:06:40.813Z',
+        experiment: 'model-compare',
+        target: '2 providers',
+        targets: ['gpt-5.4', 'gpt-5.4-mini'],
+        source: 'local',
+        eval_count: 2,
+        quality_count: 2,
+        passed_count: 2,
+        pass_rate: 1,
+        avg_score: 1,
+        tests: [
+          {
+            test_id: 'exact-token',
+            target: 'gpt-5.4',
+            score: 1,
+            passed: true,
+            execution_status: 'ok',
+            answer: 'AV94NI_OK from gpt-5.4',
+            answer_path: 'exact-token-gpt54/sample-1/outputs/answer.md',
+            grading_path: 'exact-token-gpt54/sample-1/grading.json',
+            result_dir: 'exact-token-gpt54',
+            duration_ms: 1234,
+          },
+          {
+            test_id: 'exact-token',
+            target: 'gpt-5.4-mini',
+            score: 1,
+            passed: true,
+            execution_status: 'ok',
+            answer: 'AV94NI_OK from gpt-5.4-mini',
+            answer_path: 'exact-token-mini/sample-1/outputs/answer.md',
+            result_dir: 'exact-token-mini',
+            duration_ms: 987,
+          },
+        ],
+      },
+    ],
+  };
+}
+
+describe('AnalyticsTab provider/model comparison', () => {
+  it('renders same-test provider/model outputs side by side', () => {
+    const queryClient = new QueryClient();
+    const html = renderToStaticMarkup(
+      <QueryClientProvider client={queryClient}>
+        <AnalyticsTab data={compareResponse()} isLoading={false} isError={false} />
+      </QueryClientProvider>,
+    );
+
+    expect(html).toContain('Provider/model comparison');
+    expect(html).toContain('Provider/Model');
+    expect(html).toContain('exact-token');
+    expect(html).toContain('gpt-5.4');
+    expect(html).toContain('gpt-5.4-mini');
+    expect(html).toContain('AV94NI_OK from gpt-5.4');
+    expect(html).toContain('AV94NI_OK from gpt-5.4-mini');
+    expect(html).toContain('Answer');
+    expect(html).toContain('result_dir=exact-token-gpt54');
+  });
+});