- {hasProviderModelRun &&
}
+ {hasProviderModelRun && (
+
+ )}
@@ -598,16 +628,24 @@ function PerRunCompareView({
function ProviderModelComparison({
data,
projectId,
+ defaultDisplayMode,
}: {
data: CompareResponse;
projectId?: string;
+ defaultDisplayMode: ProviderModelDisplayMode;
}) {
+ const [displayMode, setDisplayMode] = useState(defaultDisplayMode);
const run = useMemo(
() => (data.runs ?? []).find((entry) => providerLabelsForRun(entry).length > 1),
[data.runs],
);
const model = useMemo(() => (run ? buildProviderModelTable(run) : null), [run]);
+ const rows = useMemo(() => {
+ if (!model) return [];
+ if (displayMode === 'all') return model.rows;
+ return model.rows.filter((row) => providerModelRowDiffers(row, model.providerLabels));
+ }, [displayMode, model]);
if (!run || !model) return null;
@@ -620,46 +658,99 @@ function ProviderModelComparison({
{run.experiment} · {formatTimestamp(run.started_at)}
-
+
-
-
-
-
- |
- Test case
- |
- {model.providerLabels.map((label) => (
-
- {label}
+ {rows.length > 0 ? (
+
+
+
+
+ |
+ Test case
|
- ))}
-
-
-
- {model.rows.map((row) => (
-
- |
- {row.testId}
- |
{model.providerLabels.map((label) => (
-
-
- |
+
+ {label}
+ |
))}
- ))}
-
-
-
+ |
+
+ {rows.map((row) => (
+
+ |
+ {row.testId}
+ |
+ {model.providerLabels.map((label) => (
+
+
+ |
+ ))}
+
+ ))}
+
+
+
+ ) : (
+
+ )}
);
}
+function ProviderModelDisplayToggle({
+ mode,
+ onChange,
+}: {
+ mode: ProviderModelDisplayMode;
+ onChange: (mode: ProviderModelDisplayMode) => void;
+}) {
+ return (
+
+ );
+}
+
+function DisplayModeButton({
+ active,
+ onClick,
+ children,
+}: {
+ active: boolean;
+ onClick: () => void;
+ children: React.ReactNode;
+}) {
+ return (
+
+ );
+}
+
function ProviderModelResultCell({
run,
result,
@@ -779,7 +870,7 @@ function providerLabelsForRun(run: CompareRunEntry): string[] {
function buildProviderModelTable(run: CompareRunEntry): {
providerLabels: string[];
- rows: Array<{ testId: string; resultsByProvider: Map }>;
+ rows: ProviderModelRow[];
} | null {
const providerLabels = providerLabelsForRun(run);
if (providerLabels.length <= 1) return null;
@@ -802,6 +893,41 @@ function buildProviderModelTable(run: CompareRunEntry): {
};
}
+interface ProviderModelRow {
+ testId: string;
+ resultsByProvider: Map;
+}
+
+function providerModelRowDiffers(
+ row: ProviderModelRow,
+ providerLabels: readonly string[],
+): boolean {
+ const signatures = new Set();
+ for (const label of providerLabels) {
+ signatures.add(providerModelResultSignature(row.resultsByProvider.get(label)));
+ if (signatures.size > 1) return true;
+ }
+ return false;
+}
+
+function providerModelResultSignature(result: CompareTestResult | undefined): string {
+ if (!result) return 'missing';
+ return JSON.stringify({
+ answer: result.answer ?? '',
+ output: compareOptionalString(result, 'output'),
+ passed: result.passed,
+ score: result.score,
+ state: result.execution_status ?? '',
+ error: compareOptionalString(result, 'error'),
+ errorKind: compareOptionalString(result, 'target_error_kind'),
+ });
+}
+
+function compareOptionalString(result: CompareTestResult, key: string): string {
+ const value = (result as CompareTestResult & Record)[key];
+ return typeof value === 'string' ? value : '';
+}
+
function artifactHref(
run: CompareRunEntry,
result: CompareTestResult,