diff --git a/apps/dashboard/src/components/AnalyticsTab.test.tsx b/apps/dashboard/src/components/AnalyticsTab.test.tsx index ca23ff039..31810ef79 100644 --- a/apps/dashboard/src/components/AnalyticsTab.test.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.test.tsx @@ -93,6 +93,128 @@ function compareResponse(): CompareResponse { }; } +function differingCompareResponse(): CompareResponse { + return { + experiments: ['model-compare'], + targets: ['gpt-5.4', 'gpt-5.4-mini'], + cells: [ + { + experiment: 'model-compare', + target: 'gpt-5.4', + eval_count: 2, + quality_count: 2, + passed_count: 1, + pass_rate: 0.5, + avg_score: 0.5, + tests: [], + }, + { + experiment: 'model-compare', + target: 'gpt-5.4-mini', + eval_count: 2, + quality_count: 2, + passed_count: 2, + pass_rate: 1, + avg_score: 1, + tests: [], + }, + ], + runs: [ + { + run_id: '2026-07-07T14-06-40-813Z', + started_at: '2026-07-07T14:06:40.813Z', + experiment: 'model-compare', + target: '2 providers', + targets: ['gpt-5.4', 'gpt-5.4-mini'], + source: 'local', + eval_count: 4, + quality_count: 4, + passed_count: 3, + pass_rate: 0.75, + avg_score: 0.75, + tests: [ + { + test_id: 'matching-row', + target: 'gpt-5.4', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'same answer', + }, + { + test_id: 'matching-row', + target: 'gpt-5.4-mini', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'same answer', + }, + { + test_id: 'answer-differs', + target: 'gpt-5.4', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'alpha answer', + }, + { + test_id: 'answer-differs', + target: 'gpt-5.4-mini', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'beta answer', + }, + { + test_id: 'status-differs', + target: 'gpt-5.4', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'completed', + }, + { + test_id: 'status-differs', + target: 'gpt-5.4-mini', + score: 0, + passed: false, + execution_status: 'execution_error', + answer: 'completed', + }, + ], + }, + ], + }; +} + +function matchingCompareResponse(): CompareResponse { + const data = differingCompareResponse(); + return { + ...data, + runs: data.runs?.map((run) => ({ + ...run, + tests: [ + { + test_id: 'matching-row', + target: 'gpt-5.4', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'same answer', + }, + { + test_id: 'matching-row', + target: 'gpt-5.4-mini', + score: 1, + passed: true, + execution_status: 'ok', + answer: 'same answer', + }, + ], + })), + }; +} + describe('AnalyticsTab provider/model comparison', () => { it('renders same-test provider/model outputs side by side', () => { const queryClient = new QueryClient(); @@ -112,4 +234,45 @@ describe('AnalyticsTab provider/model comparison', () => { expect(html).toContain('Answer'); expect(html).toContain('result_dir=exact-token-gpt54'); }); + + it('filters provider/model comparison rows to differing outputs', () => { + const queryClient = new QueryClient(); + const html = renderToStaticMarkup( + + + , + ); + + expect(html).toContain('All'); + expect(html).toContain('Differing'); + expect(html).toContain('answer-differs'); + expect(html).toContain('alpha answer'); + expect(html).toContain('beta answer'); + expect(html).toContain('status-differs'); + expect(html).toContain('error'); + expect(html).not.toContain('matching-row'); + }); + + it('shows an empty state when all provider/model comparison rows match', () => { + const queryClient = new QueryClient(); + const html = renderToStaticMarkup( + + + , + ); + + expect(html).toContain('All compared rows match'); + expect(html).toContain('Switch back to All rows to inspect the full provider/model table.'); + expect(html).not.toContain('matching-row'); + }); }); diff --git a/apps/dashboard/src/components/AnalyticsTab.tsx b/apps/dashboard/src/components/AnalyticsTab.tsx index 25721e3a0..9c5a312a3 100644 --- a/apps/dashboard/src/components/AnalyticsTab.tsx +++ b/apps/dashboard/src/components/AnalyticsTab.tsx @@ -37,13 +37,23 @@ interface AnalyticsTabProps { projectId?: string; /** Read-only mode. Reserved for surfaces that disable mutating actions. */ readOnly?: boolean; + /** Initial row display for provider/model comparison. */ + defaultProviderModelDisplayMode?: ProviderModelDisplayMode; } type ViewMode = 'aggregated' | 'per-run'; +type ProviderModelDisplayMode = 'all' | 'differing'; // ── Top-level container ───────────────────────────────────────────────── -export function AnalyticsTab({ data, isLoading, isError, error, projectId }: AnalyticsTabProps) { +export function AnalyticsTab({ + data, + isLoading, + isError, + error, + projectId, + defaultProviderModelDisplayMode = 'all', +}: AnalyticsTabProps) { const [mode, setMode] = useState('aggregated'); const runsCount = data?.runs?.length ?? 0; @@ -60,7 +70,13 @@ export function AnalyticsTab({ data, isLoading, isError, error, projectId }: Ana {!isLoading && !isError && !underlyingHasData && } {!isLoading && !isError && underlyingHasData && data && ( <> - {mode === 'aggregated' && } + {mode === 'aggregated' && ( + + )} {mode === 'per-run' && } )} @@ -162,7 +178,15 @@ function ModeButton({ // ── Aggregated (matrix) view ──────────────────────────────────────────── -function AggregatedView({ data, projectId }: { data: CompareResponse; projectId?: string }) { +function AggregatedView({ + data, + projectId, + defaultProviderModelDisplayMode, +}: { + data: CompareResponse; + projectId?: string; + defaultProviderModelDisplayMode: ProviderModelDisplayMode; +}) { const { experiments, targets, cells } = data; // Hooks must run on every render regardless of the early-return below, @@ -192,7 +216,13 @@ function AggregatedView({ data, projectId }: { data: CompareResponse; projectId? return (
- {hasProviderModelRun && } + {hasProviderModelRun && ( + + )}
@@ -598,16 +628,24 @@ function PerRunCompareView({ function ProviderModelComparison({ data, projectId, + defaultDisplayMode, }: { data: CompareResponse; projectId?: string; + defaultDisplayMode: ProviderModelDisplayMode; }) { + const [displayMode, setDisplayMode] = useState(defaultDisplayMode); const run = useMemo( () => (data.runs ?? []).find((entry) => providerLabelsForRun(entry).length > 1), [data.runs], ); const model = useMemo(() => (run ? buildProviderModelTable(run) : null), [run]); + const rows = useMemo(() => { + if (!model) return []; + if (displayMode === 'all') return model.rows; + return model.rows.filter((row) => providerModelRowDiffers(row, model.providerLabels)); + }, [displayMode, model]); if (!run || !model) return null; @@ -620,46 +658,99 @@ function ProviderModelComparison({ {run.experiment} · {formatTimestamp(run.started_at)}

- +
+ + +
-
-
- - - - {model.providerLabels.map((label) => ( - + + {rows.map((row) => ( + + + {model.providerLabels.map((label) => ( + + ))} + + ))} + +
- Test case - - {label} + {rows.length > 0 ? ( +
+ + + + - ))} - - - - {model.rows.map((row) => ( - - {model.providerLabels.map((label) => ( - + ))} - ))} - -
+ Test case
- {row.testId} - - - + {label} +
-
+
+ {row.testId} + + +
+
+ ) : ( + + )} ); } +function ProviderModelDisplayToggle({ + mode, + onChange, +}: { + mode: ProviderModelDisplayMode; + onChange: (mode: ProviderModelDisplayMode) => void; +}) { + return ( +
+ Provider/model rows + onChange('all')}> + All + + onChange('differing')}> + Differing + +
+ ); +} + +function DisplayModeButton({ + active, + onClick, + children, +}: { + active: boolean; + onClick: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + function ProviderModelResultCell({ run, result, @@ -779,7 +870,7 @@ function providerLabelsForRun(run: CompareRunEntry): string[] { function buildProviderModelTable(run: CompareRunEntry): { providerLabels: string[]; - rows: Array<{ testId: string; resultsByProvider: Map }>; + rows: ProviderModelRow[]; } | null { const providerLabels = providerLabelsForRun(run); if (providerLabels.length <= 1) return null; @@ -802,6 +893,41 @@ function buildProviderModelTable(run: CompareRunEntry): { }; } +interface ProviderModelRow { + testId: string; + resultsByProvider: Map; +} + +function providerModelRowDiffers( + row: ProviderModelRow, + providerLabels: readonly string[], +): boolean { + const signatures = new Set(); + for (const label of providerLabels) { + signatures.add(providerModelResultSignature(row.resultsByProvider.get(label))); + if (signatures.size > 1) return true; + } + return false; +} + +function providerModelResultSignature(result: CompareTestResult | undefined): string { + if (!result) return 'missing'; + return JSON.stringify({ + answer: result.answer ?? '', + output: compareOptionalString(result, 'output'), + passed: result.passed, + score: result.score, + state: result.execution_status ?? '', + error: compareOptionalString(result, 'error'), + errorKind: compareOptionalString(result, 'target_error_kind'), + }); +} + +function compareOptionalString(result: CompareTestResult, key: string): string { + const value = (result as CompareTestResult & Record)[key]; + return typeof value === 'string' ? value : ''; +} + function artifactHref( run: CompareRunEntry, result: CompareTestResult,