refactor(cli): use provider labels in eval selection

apps/cli/src/commands/eval/artifact-writer.ts

@@ -40,7 +40,7 @@ import type { ProviderDefinition } from '@agentv/core';
 
 import {
   type MaterializedTaskBundlePaths,
-  type TaskBundleTargetSelection,
+  type TaskBundleProviderSelection,
   materializeTaskBundle,
 } from './task-bundle.js';
 
@@ -145,68 +145,71 @@ export function buildResultIndexArtifact(
   return buildCoreResultIndexArtifact(result, extraIndexFields);
 }
 
-function targetSelectionKey(evalFileAbsolutePath: string | undefined, targetName: string): string {
-  return `${evalFileAbsolutePath ? path.resolve(evalFileAbsolutePath) : ''}::${targetName}`;
+function providerSelectionKey(
+  evalFileAbsolutePath: string | undefined,
+  providerLabel: string,
+): string {
+  return `${evalFileAbsolutePath ? path.resolve(evalFileAbsolutePath) : ''}::${providerLabel}`;
 }
 
-function buildTargetSelectionMap(
-  selections: readonly TaskBundleTargetSelection[] | undefined,
-): Map<string, TaskBundleTargetSelection> {
-  const targets = new Map<string, TaskBundleTargetSelection>();
+function buildProviderSelectionMap(
+  selections: readonly TaskBundleProviderSelection[] | undefined,
+): Map<string, TaskBundleProviderSelection> {
+  const providers = new Map<string, TaskBundleProviderSelection>();
   for (const selection of selections ?? []) {
-    targets.set(
-      targetSelectionKey(selection.evalFileAbsolutePath, selection.targetName),
+    providers.set(
+      providerSelectionKey(selection.evalFileAbsolutePath, selection.providerLabel),
       selection,
     );
-    if (selection.resolvedTargetName) {
-      targets.set(
-        targetSelectionKey(selection.evalFileAbsolutePath, selection.resolvedTargetName),
+    if (selection.resolvedProviderName) {
+      providers.set(
+        providerSelectionKey(selection.evalFileAbsolutePath, selection.resolvedProviderName),
         selection,
       );
     }
     if (!selection.evalFileAbsolutePath) {
-      targets.set(targetSelectionKey(undefined, selection.targetName), selection);
-      if (selection.resolvedTargetName) {
-        targets.set(targetSelectionKey(undefined, selection.resolvedTargetName), selection);
+      providers.set(providerSelectionKey(undefined, selection.providerLabel), selection);
+      if (selection.resolvedProviderName) {
+        providers.set(providerSelectionKey(undefined, selection.resolvedProviderName), selection);
       }
     }
   }
-  return targets;
+  return providers;
 }
 
-function findTargetSelection(
+function findProviderSelection(
   result: EvaluationResult,
   test: EvalTest | undefined,
-  targets: ReadonlyMap<string, TaskBundleTargetSelection>,
-): TaskBundleTargetSelection | undefined {
-  const targetName = result.target ?? 'unknown';
+  providers: ReadonlyMap<string, TaskBundleProviderSelection>,
+): TaskBundleProviderSelection | undefined {
+  const providerLabel = result.target ?? 'unknown';
   const evalFileAbsolutePath = test?.source?.evalFileAbsolutePath;
   return (
-    targets.get(targetSelectionKey(evalFileAbsolutePath, targetName)) ??
-    targets.get(targetSelectionKey(undefined, targetName))
+    providers.get(providerSelectionKey(evalFileAbsolutePath, providerLabel)) ??
+    providers.get(providerSelectionKey(undefined, providerLabel))
   );
 }
 
 function createTaskBundleArtifactsWriter(options?: {
   cwd?: string;
   repoRoot?: string;
-  taskBundleTargets?: readonly TaskBundleTargetSelection[];
+  taskBundleTargets?: readonly TaskBundleProviderSelection[];
 }): AdditionalResultArtifactsWriter | undefined {
-  const targetSelections = buildTargetSelectionMap(options?.taskBundleTargets);
-  if (targetSelections.size === 0) {
+  const providerSelections = buildProviderSelectionMap(options?.taskBundleTargets);
+  if (providerSelections.size === 0) {
     return undefined;
   }
 
   return async ({ outputDir, result, sourceTest, testDir }) => {
-    const targetSelection = findTargetSelection(result, sourceTest, targetSelections);
-    if (!sourceTest || !targetSelection) {
+    const providerSelection = findProviderSelection(result, sourceTest, providerSelections);
+    if (!sourceTest || !providerSelection) {
       return undefined;
     }
 
     const taskBundle = await materializeTaskBundle({
       test: sourceTest,
-      targetName: targetSelection.targetName,
-      targetDefinitions: targetSelection.definitions as readonly ProviderDefinition[],
+      providerLabel: providerSelection.providerLabel,
+      providerDefinitions: providerSelection.definitions as readonly ProviderDefinition[],
       outputDir: testDir,
       cwd: options?.cwd,
       repoRoot: options?.repoRoot,
@@ -227,7 +230,7 @@ export async function writePerTestArtifacts(
     cwd?: string;
     repoRoot?: string;
     sourceTests?: readonly EvalTest[];
-    taskBundleTargets?: readonly TaskBundleTargetSelection[];
+    taskBundleTargets?: readonly TaskBundleProviderSelection[];
     additionalArtifacts?: AdditionalResultArtifactsWriter;
     runtimeSource?: RunRuntimeSourceMetadata;
     tags?: Record<string, string>;
@@ -259,7 +262,7 @@ export async function writeArtifactsFromResults(
     cwd?: string;
     repoRoot?: string;
     sourceTests?: readonly EvalTest[];
-    taskBundleTargets?: readonly TaskBundleTargetSelection[];
+    taskBundleTargets?: readonly TaskBundleProviderSelection[];
     additionalArtifacts?: AdditionalResultArtifactsWriter;
     runtimeSource?: RunRuntimeSourceMetadata;
     tags?: Record<string, string>;

apps/cli/src/commands/eval/commands/bundle.ts

@@ -11,8 +11,8 @@ import { array, command, multioption, option, optional, positional, string } fro
 import { discoverProvidersFile } from '../../../utils/providers.js';
 import { loadEnvFromHierarchy } from '../env.js';
 import { findRepoRoot, resolveEvalPaths } from '../shared.js';
-import { readTestSuiteTarget } from '../targets.js';
-import { type TaskBundleTargetSelection, materializeEvalBundle } from '../task-bundle.js';
+import { readTestSuiteProvider } from '../targets.js';
+import { type TaskBundleProviderSelection, materializeEvalBundle } from '../task-bundle.js';
 
 function unique(values: readonly string[]): readonly string[] {
   const result: string[] = [];
@@ -50,9 +50,9 @@ function targetReferenceNames(target: ProviderDefinition): readonly string[] {
 }
 
 function ensureTargetGraph(
-  targetName: string,
+  providerLabel: string,
   definitions: readonly ProviderDefinition[],
-  targetsFilePath: string,
+  providersFilePath: string,
 ): void {
   const byName = new Map(definitions.map((definition) => [definition.name, definition]));
   const seen = new Set<string>();
@@ -69,7 +69,7 @@ function ensureTargetGraph(
         .join(', ');
       const owner = requestedBy ? ` referenced by provider '${requestedBy}'` : '';
       throw new Error(
-        `Provider '${name}'${owner} not found in ${targetsFilePath}. Available providers: ${available}`,
+        `Provider '${name}'${owner} not found in ${providersFilePath}. Available providers: ${available}`,
       );
     }
     seen.add(name);
@@ -78,19 +78,19 @@ function ensureTargetGraph(
     }
   }
 
-  visit(targetName);
+  visit(providerLabel);
 }
 
-function definitionsWithEvalTargetRefs(
+function definitionsWithEvalProviderRefs(
   definitions: readonly ProviderDefinition[],
-  targetRefs: readonly EvalTargetRef[] | undefined,
+  providerRefs: readonly EvalTargetRef[] | undefined,
 ): readonly ProviderDefinition[] {
-  if (!targetRefs) {
+  if (!providerRefs) {
     return definitions;
   }
 
   const result = [...definitions];
-  for (const ref of targetRefs) {
+  for (const ref of providerRefs) {
     if (ref.definition && !result.some((definition) => definition.name === ref.name)) {
       result.push(ref.definition);
     } else if (ref.use_target && !result.some((definition) => definition.name === ref.name)) {
@@ -129,12 +129,12 @@ function definitionsWithEvalTargetSpec(
 }
 
 function buildBundleRuntime(options: {
-  readonly targetNames: readonly string[];
+  readonly providerLabels: readonly string[];
   readonly budgetUsd?: number;
   readonly threshold?: number;
 }): Record<string, unknown> {
   const runtime: Record<string, unknown> =
-    options.targetNames.length > 0 ? { providers: options.targetNames } : {};
+    options.providerLabels.length > 0 ? { providers: options.providerLabels } : {};
   if (options.budgetUsd !== undefined) {
     runtime.budget_usd = options.budgetUsd;
   }
@@ -211,50 +211,52 @@ export const evalBundleCommand = command({
     }
 
     let definitions: readonly ProviderDefinition[];
-    let targetNames: readonly string[];
+    let providerLabels: readonly string[];
     if (suite.inlineTarget) {
       definitions = [suite.inlineTarget];
-      targetNames = unique(args.provider.length > 0 ? args.provider : [suite.inlineTarget.name]);
+      providerLabels = unique(args.provider.length > 0 ? args.provider : [suite.inlineTarget.name]);
     } else {
-      const targetsFilePath = await discoverProvidersFile({
+      const providersFilePath = await discoverProvidersFile({
         explicitPath: args.providers,
         testFilePath: evalFilePath,
         repoRoot,
         cwd,
       });
       definitions = definitionsWithEvalTargetSpec(
-        definitionsWithEvalTargetRefs(
-          await readProviderDefinitions(targetsFilePath),
+        definitionsWithEvalProviderRefs(
+          await readProviderDefinitions(providersFilePath),
           suite.targetRefs,
         ),
         suite.targetSpec,
       );
-      const suiteTarget = await readTestSuiteTarget(evalFilePath);
-      targetNames = unique(
+      const suiteTarget = await readTestSuiteProvider(evalFilePath);
+      providerLabels = unique(
         args.provider.length > 0
           ? args.provider
           : (suite.targets ?? [suite.targetSpec?.name ?? suiteTarget ?? 'default']),
       );
-      for (const targetName of targetNames) {
-        ensureTargetGraph(targetName, definitions, targetsFilePath);
+      for (const providerLabel of providerLabels) {
+        ensureTargetGraph(providerLabel, definitions, providersFilePath);
       }
     }
 
-    const targetSelections: TaskBundleTargetSelection[] = targetNames.map((targetName) => ({
-      evalFileAbsolutePath: evalFilePath,
-      targetName,
-      definitions,
-    }));
+    const providerSelections: TaskBundleProviderSelection[] = providerLabels.map(
+      (providerLabel) => ({
+        evalFileAbsolutePath: evalFilePath,
+        providerLabel,
+        definitions,
+      }),
+    );
 
     const paths = await materializeEvalBundle({
       evalFilePath,
       tests: suite.tests,
-      targetSelections,
+      providerSelections,
       outputDir: args.out,
       cwd,
       repoRoot,
       runtime: buildBundleRuntime({
-        targetNames,
+        providerLabels,
         budgetUsd: suite.budgetUsd,
         threshold: suite.threshold,
       }),

apps/cli/src/commands/eval/commands/run.ts

@@ -281,8 +281,8 @@ export function buildEvalRunRawOptions(args: {
   }
 
   return {
-    target: args.provider,
-    targets: args.providers,
+    provider: args.provider,
+    providers: args.providers,
     filter: args.testId,
     workers: args.workers,
     out: args.out,

apps/cli/src/commands/eval/interactive.ts

@@ -331,7 +331,7 @@ async function executeConfig(
 ): Promise<void> {
   const cwd = process.cwd();
   const rawOptions: Record<string, unknown> = {
-    target: config.target,
+    provider: config.target,
     workers: config.workers,
     cache: config.cache,
     ...(opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {}),
@@ -382,7 +382,7 @@ async function promptRetryErrors(config: InteractiveConfig, outputPath: string):
   console.log(`\n${ANSI_DIM}Retrying execution errors...${ANSI_RESET}\n`);
 
   const rawOptions: Record<string, unknown> = {
-    target: config.target,
+    provider: config.target,
     workers: config.workers,
     cache: config.cache,
     retryErrors: outputPath,