diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts index 0b4435036..c4b0eee50 100644 --- a/apps/cli/src/commands/results/manifest.ts +++ b/apps/cli/src/commands/results/manifest.ts @@ -449,8 +449,20 @@ export interface LightweightResultRecord { readonly executionStatus?: string; readonly error?: string; readonly costUsd?: number; + readonly durationMs?: number; + readonly tokenUsage?: { + readonly input?: number; + readonly output?: number; + readonly reasoning?: number; + }; readonly timestamp?: string; readonly runtimeSource?: RunRuntimeSourceMetadata; + readonly resultDir?: string; + readonly outputPath?: string; + readonly answerPath?: string; + readonly gradingPath?: string; + readonly transcriptPath?: string; + readonly metricsPath?: string; } /** @@ -488,7 +500,15 @@ export function loadLightweightResults(sourceFile: string): LightweightResultRec executionStatus: record.execution_status, error: record.error, costUsd: record.cost_usd, + durationMs: record.duration_ms, + tokenUsage: record.token_usage, timestamp: record.timestamp, runtimeSource: record.runtime_source, + resultDir: record.result_dir, + outputPath: record.output_path, + answerPath: record.answer_path, + gradingPath: record.grading_path, + transcriptPath: record.transcript_path, + metricsPath: record.metrics_path, })); } diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index b24391b57..e7527ec2b 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -2438,10 +2438,69 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont type CompareTestEntry = { test_id: string; + target: string; category?: string; score: number; passed: boolean; execution_status?: string; + answer?: string; + answer_path?: string; + output_path?: string; + grading_path?: string; + transcript_path?: string; + metrics_path?: string; + result_dir?: string; + duration_ms?: number; + token_usage?: { + input?: number; + output?: number; + reasoning?: number; + }; + }; + + const readAnswerPreview = ( + manifestPath: string, + relativePath: string | undefined, + ): string | undefined => { + if (!relativePath) return undefined; + const resolved = resolveReadableRunArtifactFile( + runWorkspaceDirFromManifestPath(manifestPath), + relativePath, + ); + if (!resolved.absolutePath) return undefined; + try { + const text = readFileSync(resolved.absolutePath, 'utf8').trim(); + const maxLength = 2000; + return text.length > maxLength ? `${text.slice(0, maxLength)}…` : text; + } catch { + return undefined; + } + }; + + const toCompareTestEntry = ( + manifestPath: string, + record: Awaited>[number], + passed: boolean, + ): CompareTestEntry => { + const target = record.target ?? 'default'; + const answer = readAnswerPreview(manifestPath, record.answerPath ?? record.outputPath); + return { + test_id: record.testId, + target, + ...(record.category && { category: normalizeCategoryPath(record.category) }), + score: record.score, + passed, + execution_status: record.executionStatus, + ...(answer && { answer }), + ...(record.answerPath && { answer_path: record.answerPath }), + ...(record.outputPath && { output_path: record.outputPath }), + ...(record.gradingPath && { grading_path: record.gradingPath }), + ...(record.transcriptPath && { transcript_path: record.transcriptPath }), + ...(record.metricsPath && { metrics_path: record.metricsPath }), + ...(record.resultDir && { result_dir: record.resultDir }), + ...(record.durationMs !== undefined && { duration_ms: record.durationMs }), + ...(record.tokenUsage && { token_usage: record.tokenUsage }), + }; }; // Collect per-test-case results keyed by experiment × target (aggregated view) @@ -2466,6 +2525,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont started_at: string; experiment: string; target: string; + targets: string[]; source: 'local' | 'remote'; eval_count: number; quality_count: number; @@ -2484,6 +2544,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont try { const records = await loadLightweightResultsForMeta(searchDir, m, projectId); const runTestMap = new Map(); + const runTargetsSet = new Set(); let runEvalCount = 0; let runQualityCount = 0; let runPassedCount = 0; @@ -2501,6 +2562,7 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont const target = r.target ?? 'default'; experimentsSet.add(experiment); targetsSet.add(target); + runTargetsSet.add(target); runExperiment = experiment; runTarget = target; if (r.timestamp && r.timestamp < runStartedAt) runStartedAt = r.timestamp; @@ -2526,23 +2588,12 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont if (passed) entry.passedCount++; entry.scoreSum += r.score; } - entry.tests.push({ - test_id: r.testId, - ...(r.category && { category: normalizeCategoryPath(r.category) }), - score: r.score, - passed, - execution_status: r.executionStatus, - }); + entry.tests.push(toCompareTestEntry(m.path, r, passed)); cellMap.set(key, entry); - // Per-run accumulation. Dedupe tests within the run by last-wins. - runTestMap.set(r.testId, { - test_id: r.testId, - ...(r.category && { category: normalizeCategoryPath(r.category) }), - score: r.score, - passed, - execution_status: r.executionStatus, - }); + // Per-run accumulation. Dedupe each provider/model result within the + // run by last-wins while preserving the provider axis for comparison. + runTestMap.set(`${r.testId}::${target}`, toCompareTestEntry(m.path, r, passed)); runEvalCount++; if (isExecutionError) { runExecutionErrorCount++; @@ -2556,11 +2607,13 @@ async function handleCompare(c: C, { searchDir, agentvDir, projectId }: DataCont if (runEvalCount === 0) continue; const runTests = [...runTestMap.values()].slice(-MAX_TESTS_PER_CELL); + const runTargets = [...runTargetsSet].sort(); runEntries.push({ run_id: m.filename, started_at: runStartedAt, experiment: runExperiment, - target: runTarget, + target: runTargets.length > 1 ? `${runTargets.length} providers` : runTarget, + targets: runTargets, source: m.source, eval_count: runEvalCount, quality_count: runQualityCount, diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 422bad10f..944c0683b 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -4018,13 +4018,21 @@ describe('serve app', () => { experiment: string; target: string; eval_count: number; - tests?: Array<{ test_id: string; category?: string }>; + tests?: Array<{ test_id: string; target?: string; category?: string; answer?: string }>; }>; runs?: Array<{ run_id: string; experiment: string; target: string; - tests?: Array<{ test_id: string; category?: string }>; + targets?: string[]; + tests?: Array<{ + test_id: string; + target?: string; + category?: string; + answer?: string; + answer_path?: string; + result_dir?: string; + }>; }>; }; @@ -4055,6 +4063,53 @@ describe('serve app', () => { expect(cell?.tests?.[0]?.category).toBe('prompting'); expect(run?.tests?.[0]?.category).toBe('prompting'); }); + + it('preserves provider/model results and answer evidence inside a multi-provider run', async () => { + const runsDir = localResultsExperimentDir(tempDir); + const runDir = path.join(runsDir, '2026-04-05T10-00-00-000Z'); + mkdirSync(path.join(runDir, '.internal'), { recursive: true }); + mkdirSync(path.join(runDir, 'case-gpt-4o', 'sample-1', 'outputs'), { recursive: true }); + mkdirSync(path.join(runDir, 'case-claude', 'sample-1', 'outputs'), { recursive: true }); + writeFileSync(path.join(runDir, 'case-gpt-4o', 'sample-1', 'outputs', 'answer.md'), 'A'); + writeFileSync(path.join(runDir, 'case-claude', 'sample-1', 'outputs', 'answer.md'), 'B'); + writeFileSync( + path.join(runDir, '.internal', 'index.jsonl'), + toJsonl( + { + ...RESULT_A, + test_id: 'same-case', + experiment: 'model-compare', + target: 'gpt-4o', + result_dir: 'case-gpt-4o', + answer_path: 'case-gpt-4o/sample-1/outputs/answer.md', + }, + { + ...RESULT_A, + test_id: 'same-case', + experiment: 'model-compare', + target: 'claude', + result_dir: 'case-claude', + answer_path: 'case-claude/sample-1/outputs/answer.md', + score: 0.5, + }, + ), + ); + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + + const res = await app.request('/api/compare'); + expect(res.status).toBe(200); + const data = (await res.json()) as CompareJson; + const run = data.runs?.find((entry) => entry.run_id === '2026-04-05T10-00-00-000Z'); + + expect(run?.target).toBe('2 providers'); + expect(run?.targets).toEqual(['claude', 'gpt-4o']); + expect(run?.tests?.map((test) => [test.test_id, test.target, test.answer])).toEqual([ + ['same-case', 'gpt-4o', 'A'], + ['same-case', 'claude', 'B'], + ]); + expect(run?.tests?.[0]?.answer_path).toBe('case-gpt-4o/sample-1/outputs/answer.md'); + expect(run?.tests?.[0]?.result_dir).toBe('case-gpt-4o'); + }); }); // ── SPA fallback ────────────────────────────────────────────────────── diff --git a/apps/dashboard/src/components/AnalyticsTab.test.tsx b/apps/dashboard/src/components/AnalyticsTab.test.tsx new file mode 100644 index 000000000..ca23ff039 --- /dev/null +++ b/apps/dashboard/src/components/AnalyticsTab.test.tsx @@ -0,0 +1,115 @@ +import { describe, expect, it } from 'bun:test'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { renderToStaticMarkup } from 'react-dom/server'; + +import type { CompareResponse } from '~/lib/types'; + +import { AnalyticsTab } from './AnalyticsTab'; + +function compareResponse(): CompareResponse { + return { + experiments: ['model-compare'], + targets: ['gpt-5.4', 'gpt-5.4-mini'], + cells: [ + { + experiment: 'model-compare', + target: 'gpt-5.4', + eval_count: 1, + quality_count: 1, + passed_count: 1, + pass_rate: 1, + avg_score: 1, + tests: [ + { + test_id: 'exact-token', + target: 'gpt-5.4', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'AV94NI_OK', + }, + ], + }, + { + experiment: 'model-compare', + target: 'gpt-5.4-mini', + eval_count: 1, + quality_count: 1, + passed_count: 1, + pass_rate: 1, + avg_score: 1, + tests: [ + { + test_id: 'exact-token', + target: 'gpt-5.4-mini', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'AV94NI_OK', + }, + ], + }, + ], + runs: [ + { + run_id: '2026-07-07T14-06-40-813Z', + started_at: '2026-07-07T14:06:40.813Z', + experiment: 'model-compare', + target: '2 providers', + targets: ['gpt-5.4', 'gpt-5.4-mini'], + source: 'local', + eval_count: 2, + quality_count: 2, + passed_count: 2, + pass_rate: 1, + avg_score: 1, + tests: [ + { + test_id: 'exact-token', + target: 'gpt-5.4', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'AV94NI_OK from gpt-5.4', + answer_path: 'exact-token-gpt54/sample-1/outputs/answer.md', + grading_path: 'exact-token-gpt54/sample-1/grading.json', + result_dir: 'exact-token-gpt54', + duration_ms: 1234, + }, + { + test_id: 'exact-token', + target: 'gpt-5.4-mini', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'AV94NI_OK from gpt-5.4-mini', + answer_path: 'exact-token-mini/sample-1/outputs/answer.md', + result_dir: 'exact-token-mini', + duration_ms: 987, + }, + ], + }, + ], + }; +} + +describe('AnalyticsTab provider/model comparison', () => { + it('renders same-test provider/model outputs side by side', () => { + const queryClient = new QueryClient(); + const html = renderToStaticMarkup( + + + , + ); + + expect(html).toContain('Provider/model comparison'); + expect(html).toContain('Provider/Model'); + expect(html).toContain('exact-token'); + expect(html).toContain('gpt-5.4'); + expect(html).toContain('gpt-5.4-mini'); + expect(html).toContain('AV94NI_OK from gpt-5.4'); + expect(html).toContain('AV94NI_OK from gpt-5.4-mini'); + expect(html).toContain('Answer'); + expect(html).toContain('result_dir=exact-token-gpt54'); + }); +}); diff --git a/apps/dashboard/src/components/AnalyticsTab.tsx b/apps/dashboard/src/components/AnalyticsTab.tsx index 31eceb49d..25721e3a0 100644 --- a/apps/dashboard/src/components/AnalyticsTab.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.tsx @@ -2,7 +2,7 @@ * Analytics tab — cross-model comparison view. * * Two modes: - * 1. Aggregated (default) — `(experiment, target)` matrix, one cell per pair. + * 1. Aggregated (default) — `(experiment, provider/model)` matrix, one cell per pair. * 2. Per run — individual runs are first-class; users select * 2+ runs to render a side-by-side comparison. * @@ -176,24 +176,32 @@ function AggregatedView({ data, projectId }: { data: CompareResponse; projectId? return map; }, [cells]); + const hasProviderModelRun = useMemo( + () => (data.runs ?? []).some((run) => providerLabelsForRun(run).length > 1), + [data.runs], + ); + if (experiments.length <= 1 && targets.length <= 1) { return ( ); } return (
+ {hasProviderModelRun && }
{experiments.map((exp) => ( - + @@ -450,7 +458,9 @@ function PerRunRow({ {subLabel &&
{subLabel}
} - +
- Target ↓ / Experiment → + + Provider/Model ↓ / Experiment → + @@ -361,7 +369,7 @@ function PerRunView({ data }: { data: CompareResponse }) { Timestamp ExperimentTargetProvider/Model Tests Pass Rate Avg {run.experiment}{run.target} + +
{qualityCount}
{errors > 0 &&
{errors} errors
} @@ -585,6 +595,232 @@ function PerRunCompareView({ ); } +function ProviderModelComparison({ + data, + projectId, +}: { + data: CompareResponse; + projectId?: string; +}) { + const run = useMemo( + () => (data.runs ?? []).find((entry) => providerLabelsForRun(entry).length > 1), + [data.runs], + ); + + const model = useMemo(() => (run ? buildProviderModelTable(run) : null), [run]); + + if (!run || !model) return null; + + return ( +
+
+
+

Provider/model comparison

+

+ {run.experiment} · {formatTimestamp(run.started_at)} +

+
+ +
+
+ + + + + {model.providerLabels.map((label) => ( + + ))} + + + + {model.rows.map((row) => ( + + + {model.providerLabels.map((label) => ( + + ))} + + ))} + +
+ Test case + + {label} +
+ {row.testId} + + +
+
+
+ ); +} + +function ProviderModelResultCell({ + run, + result, + projectId, +}: { + run: CompareRunEntry; + result: CompareTestResult | undefined; + projectId?: string; +}) { + if (!result) { + return
—
; + } + + const isError = result.execution_status === 'execution_error'; + const scoreLabel = isError ? 'error' : `${Math.round(result.score * 100)}%`; + + return ( +
+
+ + + {scoreLabel} + + {result.duration_ms !== undefined && ( + + {formatDurationMs(result.duration_ms)} + + )} +
+ {result.answer ? ( + 
+          {result.answer}
+
+ ) : ( +
+ No answer captured +
+ )} +
+ + Answer + + + Grading + + + Transcript + +
+
+ ); +} + +function ArtifactLink({ + run, + result, + path, + projectId, + children, +}: { + run: CompareRunEntry; + result: CompareTestResult; + path: string | undefined; + projectId?: string; + children: React.ReactNode; +}) { + if (!path) return null; + const href = artifactHref(run, result, path, projectId); + return ( + + {children} + + ); +} + +function ProviderModelLabel({ run }: { run: CompareRunEntry }) { + const labels = providerLabelsForRun(run); + if (labels.length <= 1) return <>{run.target}; + return ( +
+
{labels.length} providers
+
+ {labels.join(', ')} +
+
+ ); +} + +function providerLabelsForRun(run: CompareRunEntry): string[] { + const labels = new Set(); + for (const target of run.targets ?? []) { + if (target.trim()) labels.add(target); + } + for (const test of run.tests) { + const target = test.target?.trim(); + if (target) labels.add(target); + } + return [...labels].sort(); +} + +function buildProviderModelTable(run: CompareRunEntry): { + providerLabels: string[]; + rows: Array<{ testId: string; resultsByProvider: Map }>; +} | null { + const providerLabels = providerLabelsForRun(run); + if (providerLabels.length <= 1) return null; + + const rowsByTest = new Map>(); + for (const result of run.tests) { + const provider = result.target?.trim(); + if (!provider) continue; + const row = rowsByTest.get(result.test_id) ?? new Map(); + row.set(provider, result); + rowsByTest.set(result.test_id, row); + } + + return { + providerLabels, + rows: [...rowsByTest.entries()].map(([testId, resultsByProvider]) => ({ + testId, + resultsByProvider, + })), + }; +} + +function artifactHref( + run: CompareRunEntry, + result: CompareTestResult, + filePath: string, + projectId?: string, +): string { + const runBase = projectId ? `/api/projects/${encodeURIComponent(projectId)}/runs` : '/api/runs'; + const params = new URLSearchParams({ raw: '1' }); + if (result.result_dir) params.set('result_dir', result.result_dir); + return `${runBase}/${encodeURIComponent(run.run_id)}/evals/${encodeURIComponent( + result.test_id, + )}/files/${encodeURIComponent(filePath)}?${params.toString()}`; +} + +function formatDurationMs(value: number): string { + if (value >= 1000) return `${(value / 1000).toFixed(1)}s`; + return `${Math.round(value)}ms`; +} + function RunColumnHeader({ run }: { run: CompareRunEntry }) { return (
@@ -592,7 +828,7 @@ function RunColumnHeader({ run }: { run: CompareRunEntry }) { {formatTimestamp(run.started_at)}
- {run.experiment} · {run.target} + {run.experiment} · {providerLabelsForRun(run).join(', ') || run.target}
); @@ -632,7 +868,7 @@ function EmptyState() { return ( ); } diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index b8e7e3a00..0c400dcf9 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -449,11 +449,26 @@ export interface TagGroupsResponse { export interface CompareTestResult { test_id: string; + /** Provider/model label for this result row. Historical artifacts call this target. */ + target?: string; /** Optional per-test category from the source eval result, when available. */ category?: string; score: number; passed: boolean; execution_status?: string; + answer?: string; + answer_path?: string; + output_path?: string; + grading_path?: string; + transcript_path?: string; + metrics_path?: string; + result_dir?: string; + duration_ms?: number; + token_usage?: { + input?: number; + output?: number; + reasoning?: number; + }; } export interface CompareCell { @@ -485,6 +500,8 @@ export interface CompareRunEntry { started_at: string; experiment: string; target: string; + /** Provider/model labels represented inside this run. */ + targets?: string[]; source: 'local' | 'remote'; eval_count: number; quality_count?: number; diff --git a/apps/web/src/content/docs/docs/next/tools/dashboard.mdx b/apps/web/src/content/docs/docs/next/tools/dashboard.mdx index 7463fc7ca..5858e0501 100644 --- a/apps/web/src/content/docs/docs/next/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/next/tools/dashboard.mdx @@ -70,11 +70,11 @@ indexes into Phoenix. ## Features -- **Recent Runs** — table of all evaluation runs with source badge (`local` / `remote`), target, experiment, timestamp, test count, pass rate, and mean score +- **Recent Runs** — table of all evaluation runs with source badge (`local` / `remote`), provider/model label, experiment, timestamp, test count, pass rate, and mean score - **Experiments** — group and compare runs by experiment name -- **Targets** — group runs by target (model/agent) +- **Targets** — group runs by provider/model label - **Run Detail** — drill into a run to see per-test results, scores, and grader output -- **Analytics** — two modes: an aggregated experiment × target matrix, and a per-run view for selecting individual runs to compare side-by-side with optional retroactive tags. Includes a collapsible charts section with baseline comparison analytics +- **Analytics** — provider/model comparison for multi-provider eval runs, an aggregated experiment × provider/model matrix, and a per-run view for selecting individual runs to compare side-by-side with optional retroactive tags. Includes a collapsible charts section with baseline comparison analytics - **Remote Results** — sync and browse runs pushed from other machines or CI (see [Remote Results](#remote-results)) ## Pass threshold @@ -90,7 +90,7 @@ Legacy `studio.threshold`, `studio.pass_threshold`, and root-level `pass_thresho ## Run Detail -Click any run to see a breakdown by suite, per-test scores, target, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `index.jsonl`—including per-result test bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. +Click any run to see a breakdown by suite, per-test scores, provider/model label, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `index.jsonl`—including per-result test bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. In the per-test results table, click a test ID to open its checks, transcript, source, and files in a row detail panel while the table, filters, and scroll position stay in place. Use **Full page** from the panel when you want the standalone eval detail route. @@ -116,13 +116,13 @@ The Experiments tab groups runs by experiment name so you can compare the impact ## Analytics -The **Analytics** tab has two modes: **Aggregated** for the classic experiment × target matrix, and **Per run** for selecting individual runs and pitting them side-by-side. Toggle between them from the mode switch on the right of the masthead. +The **Analytics** tab has two modes: **Aggregated** for the experiment × provider/model matrix, and **Per run** for selecting individual runs and pitting them side-by-side. Toggle between them from the mode switch on the right of the masthead. AgentV Dashboard side-by-side comparison of two runs tagged improved-prompt and baseline, with per-test pass rates ### Aggregated matrix -The default view shows a cross-experiment, cross-target performance matrix. Numbers are colour-coded by pass rate — green (80%+), amber (50–80%), red (below 50%) — and each cell shows `passed/total` and the mean score. Click any cell to expand the per-test-case breakdown. +The default view shows a cross-experiment, cross-provider/model performance matrix. Numbers are colour-coded by pass rate — green (80%+), amber (50–80%), red (below 50%) — and each cell shows `passed/total` and the mean score. Click any cell to expand the per-test-case breakdown. AgentV Dashboard Analytics tab showing aggregated experiment × target matrix with pass rates for baseline, optimized-prompt, and with-rag across claude-sonnet, gemini-pro, and gpt-4o @@ -136,9 +136,25 @@ agentv eval my.EVAL.yaml --provider gemini --experiment with-caching agentv dashboard # Analytics tab shows 2x2 matrix ``` +When one run evaluates the same prompt/test matrix across multiple provider labels, Analytics also shows a **Provider/model comparison** table. Each row is a test case and each column is a provider/model label from the run, with pass/score, answer output, timing, and links to the raw answer, grading, and transcript artifacts. Use this for Promptfoo-style model comparison: + +```yaml +providers: + - id: openai + label: gpt-5.4 + config: + model: "{{ env.MODEL_A }}" + - id: openai + label: gpt-5.4-mini + config: + model: "{{ env.MODEL_B }}" +``` + +Keep model identity in `providers` and provider `label`s. Use `tags.experiment` for the condition under test, such as `baseline-prompt` or `with-tools`, not for model names. + ### Per-run comparison -Running the same `(experiment, target)` twice no longer collapses into a single cell. Switch to **Per run** mode to see every run as its own row, select two or more, and compare them head-to-head. +Running the same `(experiment, provider/model label)` twice no longer collapses into a single cell. Switch to **Per run** mode to see every run as its own row, select two or more, and compare them head-to-head. AgentV Dashboard per-run compare mode with a filter-by-tag chip row and individual runs listing timestamp, tags, experiment, target, and pass rate; experiment-prefixed runs surface the experiment name under the timestamp @@ -166,15 +182,15 @@ The same filter is available to API consumers via `GET /api/compare?tags=baselin ### Analytics charts -Below the aggregated matrix, a collapsible **Analytics** section provides visual charts for deeper comparison. Select a **baseline target** from the dropdown to compute deltas and normalized gain metrics against that target. +Below the aggregated matrix, a collapsible **Analytics** section provides visual charts for deeper comparison. Select a **baseline provider/model** from the dropdown to compute deltas and normalized gain metrics against that label. AgentV Dashboard analytics charts showing normalized gain bar chart with baseline selector and score distribution histogram The section includes the following visualizations: -- **Normalized Gain (g)** — horizontal bar chart showing how much of the remaining headroom each experiment × target captured relative to the baseline. Bars are colour-coded green (positive gain), red (regression), or grey (null / no headroom). See [Normalized Gain](/docs/tools/compare#normalized-gain-g) for the formula. -- **Tag × Target Heatmap** — pass-rate grid across tags and targets, colour-coded by performance (emerald for high, amber for medium, red for low). -- **Negative Delta Table** — filtered list of experiment × target pairs that scored worse than the baseline, sorted by largest regression. +- **Normalized Gain (g)** — horizontal bar chart showing how much of the remaining headroom each experiment × provider/model label captured relative to the baseline. Bars are colour-coded green (positive gain), red (regression), or grey (null / no headroom). See [Normalized Gain](/docs/tools/compare#normalized-gain-g) for the formula. +- **Tag × Provider/Model Heatmap** — pass-rate grid across tags and provider/model labels, colour-coded by performance (emerald for high, amber for medium, red for low). +- **Negative Delta Table** — filtered list of experiment × provider/model pairs that scored worse than the baseline, sorted by largest regression. - **Score Distribution** — histogram showing the variance of scores across all test cases, binned by 10% intervals. - **Score Trend Over Time** — line chart plotting mean score per target across runs over time, with a colour-coded legend for each target.