From 19a6b95e6c5a57cda6701b7e0f1d83b3920f57be Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 23:43:40 +0200 Subject: [PATCH] refactor(cli): use provider labels in eval selection --- apps/cli/src/commands/eval/artifact-writer.ts | 65 ++-- apps/cli/src/commands/eval/commands/bundle.ts | 58 +-- apps/cli/src/commands/eval/commands/run.ts | 4 +- apps/cli/src/commands/eval/interactive.ts | 4 +- apps/cli/src/commands/eval/run-eval.ts | 341 ++++++++++-------- apps/cli/src/commands/eval/targets.ts | 174 ++++----- apps/cli/src/commands/eval/task-bundle.ts | 56 +-- apps/cli/src/commands/grade/index.ts | 10 +- apps/cli/src/commands/pipeline/input.ts | 12 +- apps/cli/src/commands/pipeline/run.ts | 16 +- apps/cli/src/commands/prepare/index.ts | 18 +- apps/cli/src/commands/runs/rerun.ts | 4 +- .../commands/eval/artifact-writer.test.ts | 8 +- .../commands/eval/run-command-options.test.ts | 6 +- apps/cli/test/commands/eval/targets.test.ts | 76 ++-- .../test/commands/eval/task-bundle.test.ts | 4 +- 16 files changed, 443 insertions(+), 413 deletions(-) diff --git a/apps/cli/src/commands/eval/artifact-writer.ts b/apps/cli/src/commands/eval/artifact-writer.ts index 55d02c1ce..81546ab71 100644 --- a/apps/cli/src/commands/eval/artifact-writer.ts +++ b/apps/cli/src/commands/eval/artifact-writer.ts @@ -40,7 +40,7 @@ import type { ProviderDefinition } from '@agentv/core'; import { type MaterializedTaskBundlePaths, - type TaskBundleTargetSelection, + type TaskBundleProviderSelection, materializeTaskBundle, } from './task-bundle.js'; @@ -145,68 +145,71 @@ export function buildResultIndexArtifact( return buildCoreResultIndexArtifact(result, extraIndexFields); } -function targetSelectionKey(evalFileAbsolutePath: string | undefined, targetName: string): string { - return `${evalFileAbsolutePath ? path.resolve(evalFileAbsolutePath) : ''}::${targetName}`; +function providerSelectionKey( + evalFileAbsolutePath: string | undefined, + providerLabel: string, +): string { + return `${evalFileAbsolutePath ? path.resolve(evalFileAbsolutePath) : ''}::${providerLabel}`; } -function buildTargetSelectionMap( - selections: readonly TaskBundleTargetSelection[] | undefined, -): Map { - const targets = new Map(); +function buildProviderSelectionMap( + selections: readonly TaskBundleProviderSelection[] | undefined, +): Map { + const providers = new Map(); for (const selection of selections ?? []) { - targets.set( - targetSelectionKey(selection.evalFileAbsolutePath, selection.targetName), + providers.set( + providerSelectionKey(selection.evalFileAbsolutePath, selection.providerLabel), selection, ); - if (selection.resolvedTargetName) { - targets.set( - targetSelectionKey(selection.evalFileAbsolutePath, selection.resolvedTargetName), + if (selection.resolvedProviderName) { + providers.set( + providerSelectionKey(selection.evalFileAbsolutePath, selection.resolvedProviderName), selection, ); } if (!selection.evalFileAbsolutePath) { - targets.set(targetSelectionKey(undefined, selection.targetName), selection); - if (selection.resolvedTargetName) { - targets.set(targetSelectionKey(undefined, selection.resolvedTargetName), selection); + providers.set(providerSelectionKey(undefined, selection.providerLabel), selection); + if (selection.resolvedProviderName) { + providers.set(providerSelectionKey(undefined, selection.resolvedProviderName), selection); } } } - return targets; + return providers; } -function findTargetSelection( +function findProviderSelection( result: EvaluationResult, test: EvalTest | undefined, - targets: ReadonlyMap, -): TaskBundleTargetSelection | undefined { - const targetName = result.target ?? 'unknown'; + providers: ReadonlyMap, +): TaskBundleProviderSelection | undefined { + const providerLabel = result.target ?? 'unknown'; const evalFileAbsolutePath = test?.source?.evalFileAbsolutePath; return ( - targets.get(targetSelectionKey(evalFileAbsolutePath, targetName)) ?? - targets.get(targetSelectionKey(undefined, targetName)) + providers.get(providerSelectionKey(evalFileAbsolutePath, providerLabel)) ?? + providers.get(providerSelectionKey(undefined, providerLabel)) ); } function createTaskBundleArtifactsWriter(options?: { cwd?: string; repoRoot?: string; - taskBundleTargets?: readonly TaskBundleTargetSelection[]; + taskBundleTargets?: readonly TaskBundleProviderSelection[]; }): AdditionalResultArtifactsWriter | undefined { - const targetSelections = buildTargetSelectionMap(options?.taskBundleTargets); - if (targetSelections.size === 0) { + const providerSelections = buildProviderSelectionMap(options?.taskBundleTargets); + if (providerSelections.size === 0) { return undefined; } return async ({ outputDir, result, sourceTest, testDir }) => { - const targetSelection = findTargetSelection(result, sourceTest, targetSelections); - if (!sourceTest || !targetSelection) { + const providerSelection = findProviderSelection(result, sourceTest, providerSelections); + if (!sourceTest || !providerSelection) { return undefined; } const taskBundle = await materializeTaskBundle({ test: sourceTest, - targetName: targetSelection.targetName, - targetDefinitions: targetSelection.definitions as readonly ProviderDefinition[], + providerLabel: providerSelection.providerLabel, + providerDefinitions: providerSelection.definitions as readonly ProviderDefinition[], outputDir: testDir, cwd: options?.cwd, repoRoot: options?.repoRoot, @@ -227,7 +230,7 @@ export async function writePerTestArtifacts( cwd?: string; repoRoot?: string; sourceTests?: readonly EvalTest[]; - taskBundleTargets?: readonly TaskBundleTargetSelection[]; + taskBundleTargets?: readonly TaskBundleProviderSelection[]; additionalArtifacts?: AdditionalResultArtifactsWriter; runtimeSource?: RunRuntimeSourceMetadata; tags?: Record; @@ -259,7 +262,7 @@ export async function writeArtifactsFromResults( cwd?: string; repoRoot?: string; sourceTests?: readonly EvalTest[]; - taskBundleTargets?: readonly TaskBundleTargetSelection[]; + taskBundleTargets?: readonly TaskBundleProviderSelection[]; additionalArtifacts?: AdditionalResultArtifactsWriter; runtimeSource?: RunRuntimeSourceMetadata; tags?: Record; diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index c2c8feaf4..e14de073b 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -11,8 +11,8 @@ import { array, command, multioption, option, optional, positional, string } fro import { discoverProvidersFile } from '../../../utils/providers.js'; import { loadEnvFromHierarchy } from '../env.js'; import { findRepoRoot, resolveEvalPaths } from '../shared.js'; -import { readTestSuiteTarget } from '../targets.js'; -import { type TaskBundleTargetSelection, materializeEvalBundle } from '../task-bundle.js'; +import { readTestSuiteProvider } from '../targets.js'; +import { type TaskBundleProviderSelection, materializeEvalBundle } from '../task-bundle.js'; function unique(values: readonly string[]): readonly string[] { const result: string[] = []; @@ -50,9 +50,9 @@ function targetReferenceNames(target: ProviderDefinition): readonly string[] { } function ensureTargetGraph( - targetName: string, + providerLabel: string, definitions: readonly ProviderDefinition[], - targetsFilePath: string, + providersFilePath: string, ): void { const byName = new Map(definitions.map((definition) => [definition.name, definition])); const seen = new Set(); @@ -69,7 +69,7 @@ function ensureTargetGraph( .join(', '); const owner = requestedBy ? ` referenced by provider '${requestedBy}'` : ''; throw new Error( - `Provider '${name}'${owner} not found in ${targetsFilePath}. Available providers: ${available}`, + `Provider '${name}'${owner} not found in ${providersFilePath}. Available providers: ${available}`, ); } seen.add(name); @@ -78,19 +78,19 @@ function ensureTargetGraph( } } - visit(targetName); + visit(providerLabel); } -function definitionsWithEvalTargetRefs( +function definitionsWithEvalProviderRefs( definitions: readonly ProviderDefinition[], - targetRefs: readonly EvalTargetRef[] | undefined, + providerRefs: readonly EvalTargetRef[] | undefined, ): readonly ProviderDefinition[] { - if (!targetRefs) { + if (!providerRefs) { return definitions; } const result = [...definitions]; - for (const ref of targetRefs) { + for (const ref of providerRefs) { if (ref.definition && !result.some((definition) => definition.name === ref.name)) { result.push(ref.definition); } else if (ref.use_target && !result.some((definition) => definition.name === ref.name)) { @@ -129,12 +129,12 @@ function definitionsWithEvalTargetSpec( } function buildBundleRuntime(options: { - readonly targetNames: readonly string[]; + readonly providerLabels: readonly string[]; readonly budgetUsd?: number; readonly threshold?: number; }): Record { const runtime: Record = - options.targetNames.length > 0 ? { providers: options.targetNames } : {}; + options.providerLabels.length > 0 ? { providers: options.providerLabels } : {}; if (options.budgetUsd !== undefined) { runtime.budget_usd = options.budgetUsd; } @@ -211,50 +211,52 @@ export const evalBundleCommand = command({ } let definitions: readonly ProviderDefinition[]; - let targetNames: readonly string[]; + let providerLabels: readonly string[]; if (suite.inlineTarget) { definitions = [suite.inlineTarget]; - targetNames = unique(args.provider.length > 0 ? args.provider : [suite.inlineTarget.name]); + providerLabels = unique(args.provider.length > 0 ? args.provider : [suite.inlineTarget.name]); } else { - const targetsFilePath = await discoverProvidersFile({ + const providersFilePath = await discoverProvidersFile({ explicitPath: args.providers, testFilePath: evalFilePath, repoRoot, cwd, }); definitions = definitionsWithEvalTargetSpec( - definitionsWithEvalTargetRefs( - await readProviderDefinitions(targetsFilePath), + definitionsWithEvalProviderRefs( + await readProviderDefinitions(providersFilePath), suite.targetRefs, ), suite.targetSpec, ); - const suiteTarget = await readTestSuiteTarget(evalFilePath); - targetNames = unique( + const suiteTarget = await readTestSuiteProvider(evalFilePath); + providerLabels = unique( args.provider.length > 0 ? args.provider : (suite.targets ?? [suite.targetSpec?.name ?? suiteTarget ?? 'default']), ); - for (const targetName of targetNames) { - ensureTargetGraph(targetName, definitions, targetsFilePath); + for (const providerLabel of providerLabels) { + ensureTargetGraph(providerLabel, definitions, providersFilePath); } } - const targetSelections: TaskBundleTargetSelection[] = targetNames.map((targetName) => ({ - evalFileAbsolutePath: evalFilePath, - targetName, - definitions, - })); + const providerSelections: TaskBundleProviderSelection[] = providerLabels.map( + (providerLabel) => ({ + evalFileAbsolutePath: evalFilePath, + providerLabel, + definitions, + }), + ); const paths = await materializeEvalBundle({ evalFilePath, tests: suite.tests, - targetSelections, + providerSelections, outputDir: args.out, cwd, repoRoot, runtime: buildBundleRuntime({ - targetNames, + providerLabels, budgetUsd: suite.budgetUsd, threshold: suite.threshold, }), diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 134140620..4c783aca8 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -281,8 +281,8 @@ export function buildEvalRunRawOptions(args: { } return { - target: args.provider, - targets: args.providers, + provider: args.provider, + providers: args.providers, filter: args.testId, workers: args.workers, out: args.out, diff --git a/apps/cli/src/commands/eval/interactive.ts b/apps/cli/src/commands/eval/interactive.ts index 2ca14c17c..958f9eddd 100644 --- a/apps/cli/src/commands/eval/interactive.ts +++ b/apps/cli/src/commands/eval/interactive.ts @@ -331,7 +331,7 @@ async function executeConfig( ): Promise { const cwd = process.cwd(); const rawOptions: Record = { - target: config.target, + provider: config.target, workers: config.workers, cache: config.cache, ...(opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {}), @@ -382,7 +382,7 @@ async function promptRetryErrors(config: InteractiveConfig, outputPath: string): console.log(`\n${ANSI_DIM}Retrying execution errors...${ANSI_RESET}\n`); const rawOptions: Record = { - target: config.target, + provider: config.target, workers: config.workers, cache: config.cache, retryErrors: outputPath, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index a3a1d9237..29137d4a6 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -90,8 +90,8 @@ import { formatEvaluationSummary, formatMatrixSummary, } from './statistics.js'; -import { type TargetSelection, selectMultipleTargets, selectTarget } from './targets.js'; -import type { TaskBundleTargetSelection } from './task-bundle.js'; +import { type ProviderSelection, selectMultipleProviders, selectProvider } from './targets.js'; +import type { TaskBundleProviderSelection } from './task-bundle.js'; import { WipCheckpointLoop } from './wip-checkpoint.js'; const DEFAULT_WORKERS = 3; @@ -270,9 +270,9 @@ interface RunEvalCommandInput { } interface NormalizedOptions { - readonly target?: string; - readonly cliTargets: readonly string[]; - readonly targetsPath?: string; + readonly provider?: string; + readonly cliProviderLabels: readonly string[]; + readonly providersPath?: string; /** Internal rerun-only carveout for generated test bundle targets.yaml artifacts. */ readonly allowLegacyTargetFiles?: boolean; /** Internal rerun-only provider catalog paths keyed by captured eval file. */ @@ -319,9 +319,9 @@ interface NormalizedOptions { readonly experiment?: string; readonly experimentConfig?: ExperimentConfig; readonly experimentMetadata?: ExperimentArtifactMetadata; - readonly experimentTargets?: readonly string[]; - readonly experimentTargetRefs?: readonly EvalTargetRef[]; - readonly targetModelOverride?: string; + readonly experimentProviderLabels?: readonly string[]; + readonly experimentProviderRefs?: readonly EvalTargetRef[]; + readonly providerModelOverride?: string; readonly experimentTrialsConfig?: TrialsConfig; readonly budgetUsd?: number; readonly cliBudgetUsd?: number; @@ -676,18 +676,19 @@ function normalizeOptions( const cliOutputDir = normalizeString(rawOptions.output); - // Normalize provider selection: public --provider lowers into the historical internal target key. - const rawTarget = rawOptions.target; - let cliTargets: string[] = []; - let singleTarget: string | undefined; - if (Array.isArray(rawTarget)) { - cliTargets = rawTarget.filter((v): v is string => typeof v === 'string' && v.trim().length > 0); - singleTarget = cliTargets.length === 1 ? cliTargets[0] : undefined; - } else if (typeof rawTarget === 'string') { - const trimmed = rawTarget.trim(); + const rawProvider = rawOptions.provider; + let cliProviderLabels: string[] = []; + let singleProvider: string | undefined; + if (Array.isArray(rawProvider)) { + cliProviderLabels = rawProvider.filter( + (v): v is string => typeof v === 'string' && v.trim().length > 0, + ); + singleProvider = cliProviderLabels.length === 1 ? cliProviderLabels[0] : undefined; + } else if (typeof rawProvider === 'string') { + const trimmed = rawProvider.trim(); if (trimmed.length > 0 && trimmed !== 'default') { - cliTargets = [trimmed]; - singleTarget = trimmed; + cliProviderLabels = [trimmed]; + singleProvider = trimmed; } } @@ -731,9 +732,9 @@ function normalizeOptions( const cliTags = splitCliTags(rawOptions.tag); return { - target: singleTarget, - cliTargets, - targetsPath: normalizeString(rawOptions.targets), + provider: singleProvider, + cliProviderLabels, + providersPath: normalizeString(rawOptions.providers), allowLegacyTargetFiles: normalizeBoolean(rawOptions.allowLegacyTargetFiles), providerCatalogPathByEvalFile: normalizeProviderCatalogPathByEvalFile( rawOptions.providerCatalogPathByEvalFile, @@ -831,8 +832,8 @@ function deriveEvalResultGroupName(evalFilePath: string | undefined): string { } const CLI_RUNTIME_SOURCE_OPTION_KEYS = [ - 'target', - 'targets', + 'provider', + 'providers', 'filter', 'tag', 'excludeTag', @@ -992,14 +993,14 @@ function applyExperimentOptions( return options; } - const experimentTargetRefs = buildExperimentTargetRefs(experiment); - const experimentTargetNames = experimentTargetRefs?.map((target) => target.name) ?? []; + const experimentProviderRefs = buildExperimentTargetRefs(experiment); + const experimentProviderLabels = experimentProviderRefs?.map((target) => target.name) ?? []; const experimentTarget = experiment.target && experiment.target.trim().length > 0 ? experiment.target : undefined; - const experimentTargets = - options.cliTargets.length === 0 - ? experimentTargetNames.length > 0 - ? experimentTargetNames + const experimentProviders = + options.cliProviderLabels.length === 0 + ? experimentProviderLabels.length > 0 + ? experimentProviderLabels : experimentTarget ? [experimentTarget] : undefined @@ -1007,16 +1008,16 @@ function applyExperimentOptions( return { ...options, - target: options.target, agentTimeoutSeconds: options.agentTimeoutSeconds ?? experiment.timeoutSeconds, workspacePath: options.workspacePath, budgetUsd: options.budgetUsd ?? experiment.budgetUsd, threshold: options.threshold ?? experiment.threshold, experimentConfig: experiment, experimentMetadata: buildExperimentArtifactMetadata(experiment), - experimentTargets, - experimentTargetRefs: options.cliTargets.length === 0 ? experimentTargetRefs : undefined, - targetModelOverride: options.targetModelOverride ?? experiment.model, + experimentProviderLabels: experimentProviders, + experimentProviderRefs: + options.cliProviderLabels.length === 0 ? experimentProviderRefs : undefined, + providerModelOverride: options.providerModelOverride ?? experiment.model, experimentTrialsConfig: buildExperimentTrialsConfig(experiment), }; } @@ -1223,29 +1224,32 @@ function buildTargetLabelSuffix(providerLabel: string, target: ResolvedProviderB * Override CLI provider verbose setting based on CLI --verbose flag. * CLI provider logs should only appear when --verbose is passed. */ -function applyVerboseOverride(selection: TargetSelection, cliVerbose: boolean): TargetSelection { - const { resolvedTarget } = selection; +function applyVerboseOverride( + selection: ProviderSelection, + cliVerbose: boolean, +): ProviderSelection { + const { resolvedProvider } = selection; // Only CLI providers have a verbose setting in their config - if (resolvedTarget.kind !== 'cli') { + if (resolvedProvider.kind !== 'cli') { return selection; } // Set verbose to match CLI --verbose flag return { ...selection, - resolvedTarget: { - ...resolvedTarget, + resolvedProvider: { + ...resolvedProvider, config: { - ...resolvedTarget.config, + ...resolvedProvider.config, verbose: cliVerbose, }, }, }; } -function targetVariantForSelection(selection: TargetSelection): string | undefined { - const target = selection.resolvedTarget; +function targetVariantForSelection(selection: ProviderSelection): string | undefined { + const target = selection.resolvedProvider; if (target.kind === 'replay') { return target.config.variant; } @@ -1346,7 +1350,7 @@ async function prepareFileMetadata(params: { readonly options: NormalizedOptions; readonly testIds: readonly string[]; readonly testCases: readonly EvalTest[]; - readonly selections: readonly { selection: TargetSelection; inlineTargetLabel: string }[]; + readonly selections: readonly { selection: ProviderSelection; inlineTargetLabel: string }[]; readonly trialsConfig?: TrialsConfig; readonly suiteTargets?: readonly string[]; readonly yamlCache?: boolean; @@ -1390,7 +1394,7 @@ async function prepareFileMetadata(params: { experimentOptions.workers === undefined && suite.workers !== undefined ? { ...experimentOptions, workers: suite.workers } : experimentOptions; - const effectiveProviderCatalogPath = effectiveOptions.targetsPath ?? providerCatalogPath; + const effectiveProviderCatalogPath = effectiveOptions.providersPath ?? providerCatalogPath; const testCases = suiteFilter && effectiveOptions.filter ? suite.tests.filter((testCase) => @@ -1429,21 +1433,21 @@ async function prepareFileMetadata(params: { }; } - let selections: { selection: TargetSelection; inlineTargetLabel: string }[]; + let selections: { selection: ProviderSelection; inlineTargetLabel: string }[]; if (fileOptions.transcript) { // --transcript mode: bypass target resolution entirely. - // Create a synthetic TargetSelection for the transcript provider. - const transcriptSelection: TargetSelection = { + // Create a synthetic ProviderSelection for the transcript provider. + const transcriptSelection: ProviderSelection = { definitions: [], - resolvedTarget: { + resolvedProvider: { kind: 'transcript', name: 'transcript', config: {} as Record, }, - targetName: 'transcript', - targetSource: 'cli', - targetsFilePath: fileOptions.transcript, + providerLabel: 'transcript', + providerSource: 'cli', + providersFilePath: fileOptions.transcript, }; selections = [ { @@ -1451,24 +1455,29 @@ async function prepareFileMetadata(params: { inlineTargetLabel: `transcript (${path.basename(fileOptions.transcript)})`, }, ]; - } else if (suite.inlineTarget && fileOptions.cliTargets.length === 0) { + } else if (suite.inlineTarget && fileOptions.cliProviderLabels.length === 0) { const targetDefinition = suite.inlineTarget; - const resolvedTarget = resolveProviderDefinition(targetDefinition, process.env, testFilePath, { - emitDeprecationWarnings: false, - }); + const resolvedProvider = resolveProviderDefinition( + targetDefinition, + process.env, + testFilePath, + { + emitDeprecationWarnings: false, + }, + ); selections = [ { selection: { definitions: [targetDefinition], - resolvedTarget, - targetName: targetDefinition.name, - targetSource: 'test-file', - targetsFilePath: testFilePath, + resolvedProvider, + providerLabel: targetDefinition.name, + providerSource: 'test-file', + providersFilePath: testFilePath, }, - inlineTargetLabel: resolveTargetLabel(targetDefinition.name, resolvedTarget.name), + inlineTargetLabel: resolveTargetLabel(targetDefinition.name, resolvedProvider.name), }, ]; - } else if (suite.providerFactory && fileOptions.cliTargets.length === 0) { + } else if (suite.providerFactory && fileOptions.cliProviderLabels.length === 0) { const taskTarget: ResolvedProviderBackend = { kind: 'mock', name: 'custom-task', @@ -1479,109 +1488,114 @@ async function prepareFileMetadata(params: { { selection: { definitions: [], - resolvedTarget: taskTarget, - targetName: 'custom-task', - targetSource: 'test-file', - targetsFilePath: testFilePath, + resolvedProvider: taskTarget, + providerLabel: 'custom-task', + providerSource: 'test-file', + providersFilePath: testFilePath, }, inlineTargetLabel: 'custom-task', }, ]; } else { // Determine provider labels: CLI --provider flags override YAML - const cliTargets = fileOptions.cliTargets; - const experimentTargets = fileOptions.experimentTargets ?? []; + const cliProviderLabels = fileOptions.cliProviderLabels; + const experimentProviderLabels = fileOptions.experimentProviderLabels ?? []; const suiteTargetSpec = suite.targetSpec; - const suiteTargets = suiteTargetSpec ? [suiteTargetSpec.name] : suite.targets; - const suiteTargetRefs = suite.targetRefs; - const experimentTargetRefs = fileOptions.experimentTargetRefs; - - // Resolve which target names to use (precedence: CLI/experiment > suite YAML targets > default) - let targetNames: readonly string[]; - let targetRefs: readonly EvalTargetRef[] | undefined; - let targetSource: 'cli' | 'test-file' = 'test-file'; - if (cliTargets.length > 0) { - targetNames = cliTargets; - targetRefs = experimentTargetRefs; - targetSource = 'cli'; - } else if (experimentTargets.length > 0) { - targetNames = experimentTargets; - targetRefs = experimentTargetRefs; - } else if (suiteTargets && suiteTargets.length > 0) { - targetNames = suiteTargets; - targetRefs = suiteTargetRefs; + const suiteProviderLabels = suiteTargetSpec ? [suiteTargetSpec.name] : suite.targets; + const suiteProviderRefs = suite.targetRefs; + const experimentProviderRefs = fileOptions.experimentProviderRefs; + + // Resolve which provider labels to use (precedence: CLI/experiment > suite YAML providers > default) + let providerLabels: readonly string[]; + let providerRefs: readonly EvalTargetRef[] | undefined; + let providerSource: 'cli' | 'test-file' = 'test-file'; + if (cliProviderLabels.length > 0) { + providerLabels = cliProviderLabels; + providerRefs = experimentProviderRefs; + providerSource = 'cli'; + } else if (experimentProviderLabels.length > 0) { + providerLabels = experimentProviderLabels; + providerRefs = experimentProviderRefs; + } else if (suiteProviderLabels && suiteProviderLabels.length > 0) { + providerLabels = suiteProviderLabels; + providerRefs = suiteProviderRefs; } else if (suiteDefaults?.provider) { - targetNames = [suiteDefaults.provider]; - targetRefs = undefined; + providerLabels = [suiteDefaults.provider]; + providerRefs = undefined; } else { - targetNames = []; - targetRefs = undefined; + providerLabels = []; + providerRefs = undefined; } - if (targetNames.length > 1 || (targetNames.length === 1 && targetRefs)) { - // Matrix mode: multiple targets - const multiSelections = await selectMultipleTargets({ + if (providerLabels.length > 1 || (providerLabels.length === 1 && providerRefs)) { + // Matrix mode: multiple providers + const multiSelections = await selectMultipleProviders({ testFilePath, repoRoot, cwd, - explicitTargetsPath: effectiveProviderCatalogPath, + explicitProvidersPath: effectiveProviderCatalogPath, providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog: true, allowLegacyTargetFiles: fileOptions.allowLegacyTargetFiles, env: process.env, - targetNames, - targetRefs, - targetSource, - modelOverride: fileOptions.targetModelOverride, + providerLabels, + providerRefs: providerRefs, + providerSource, + modelOverride: fileOptions.providerModelOverride, }); selections = multiSelections.map((sel) => ({ selection: sel, inlineTargetLabel: - sel.targetLabel ?? resolveTargetLabel(sel.targetName, sel.resolvedTarget.name), + sel.providerDisplayLabel ?? + resolveTargetLabel(sel.providerLabel, sel.resolvedProvider.name), })); } else { - // Single target mode (legacy path) - const selection = await selectTarget({ + // Single provider mode + const selection = await selectProvider({ testFilePath, repoRoot, cwd, - explicitTargetsPath: effectiveProviderCatalogPath, + explicitProvidersPath: effectiveProviderCatalogPath, providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog: true, allowLegacyTargetFiles: fileOptions.allowLegacyTargetFiles, - cliTargetName: - targetSource === 'cli' - ? targetNames.length === 1 - ? targetNames[0] - : fileOptions.target - : fileOptions.target, - fileTargetName: - targetSource === 'test-file' && targetNames.length === 1 ? targetNames[0] : undefined, + cliProviderLabel: + providerSource === 'cli' + ? providerLabels.length === 1 + ? providerLabels[0] + : fileOptions.provider + : fileOptions.provider, + fileProviderLabel: + providerSource === 'test-file' && providerLabels.length === 1 + ? providerLabels[0] + : undefined, fileTargetSpec: - targetSource === 'test-file' && targetNames.length === 1 ? suiteTargetSpec : undefined, - modelOverride: fileOptions.targetModelOverride, + providerSource === 'test-file' && providerLabels.length === 1 + ? suiteTargetSpec + : undefined, + modelOverride: fileOptions.providerModelOverride, env: process.env, }); - // Attach target hooks from eval file if available - const singleTargetRef = targetRefs?.find((ref) => ref.name === selection.targetName); - const augmentedSelection: TargetSelection = { + // Attach provider hooks from eval file if available + const singleProviderRef = providerRefs?.find((ref) => ref.name === selection.providerLabel); + const augmentedSelection: ProviderSelection = { ...selection, - ...(singleTargetRef?.label ? { targetLabel: singleTargetRef.label } : {}), - ...(singleTargetRef?.hooks ? { targetHooks: singleTargetRef.hooks } : {}), + ...(singleProviderRef?.label ? { providerDisplayLabel: singleProviderRef.label } : {}), + ...(singleProviderRef?.hooks ? { providerHooks: singleProviderRef.hooks } : {}), }; selections = [ { selection: augmentedSelection, inlineTargetLabel: - augmentedSelection.targetLabel ?? + augmentedSelection.providerDisplayLabel ?? resolveTargetLabel( - augmentedSelection.targetName, - augmentedSelection.resolvedTarget.name, + augmentedSelection.providerLabel, + augmentedSelection.resolvedProvider.name, ), }, ]; @@ -1605,13 +1619,13 @@ async function prepareFileMetadata(params: { }; } -function buildTaskBundleTargetSelections( +function buildTaskBundleProviderSelections( activeTestFiles: readonly string[], fileMetadata: ReadonlyMap< string, - { readonly selections: readonly { readonly selection: TargetSelection }[] } + { readonly selections: readonly { readonly selection: ProviderSelection }[] } >, -): readonly TaskBundleTargetSelection[] { +): readonly TaskBundleProviderSelection[] { return activeTestFiles.flatMap((testFilePath) => { const meta = fileMetadata.get(testFilePath); if (!meta) { @@ -1619,8 +1633,8 @@ function buildTaskBundleTargetSelections( } return meta.selections.map(({ selection }) => ({ evalFileAbsolutePath: testFilePath, - targetName: selection.targetName, - resolvedTargetName: selection.resolvedTarget.name, + providerLabel: selection.providerLabel, + resolvedProviderName: selection.resolvedProvider.name, definitions: selection.definitions, })); }); @@ -1638,7 +1652,7 @@ async function runSingleEvalFile(params: { readonly progressReporter: ProgressReporter; readonly seenTestCases: Set; readonly displayIdTracker: { getOrAssign(testCaseKey: string): number }; - readonly selection: TargetSelection; + readonly selection: ProviderSelection; readonly inlineTargetLabel: string; readonly testCases: readonly EvalTest[]; readonly trialsConfig?: TrialsConfig; @@ -1676,11 +1690,11 @@ async function runSingleEvalFile(params: { providerFactory, } = params; - const targetName = selection.targetName; + const providerLabel = selection.providerLabel; const replayRecording = options.recordReplay ? { fixturesPath: path.resolve(options.recordReplay), - sourceTarget: targetName, + sourceTarget: providerLabel, variant: options.recordReplayVariant, } : undefined; @@ -1688,19 +1702,19 @@ async function runSingleEvalFile(params: { await ensureFileExists(testFilePath, 'Test file'); // CLI provider verbose logging should only be enabled when --verbose flag is passed - const resolvedTargetSelection = applyVerboseOverride(selection, options.verbose); - const explicitVariant = targetVariantForSelection(resolvedTargetSelection); - const providerLabel = resolvedTargetSelection.resolvedTarget.kind; + const resolvedProviderSelection = applyVerboseOverride(selection, options.verbose); + const explicitVariant = targetVariantForSelection(resolvedProviderSelection); + const providerId = resolvedProviderSelection.resolvedProvider.kind; const targetMessage = options.verbose - ? `Using provider (${resolvedTargetSelection.targetSource}): ${resolvedTargetSelection.targetName} ${buildTargetLabelSuffix(providerLabel, resolvedTargetSelection.resolvedTarget)} via ${resolvedTargetSelection.targetsFilePath}` + ? `Using provider (${resolvedProviderSelection.providerSource}): ${resolvedProviderSelection.providerLabel} ${buildTargetLabelSuffix(providerId, resolvedProviderSelection.resolvedProvider)} via ${resolvedProviderSelection.providersFilePath}` : `Using provider: ${inlineTargetLabel}`; if (!progressReporter.isInteractive || options.verbose) { console.log(`${targetMessage}`); } // Hint about pipeline for CLI agent targets - const targetKind = resolvedTargetSelection.resolvedTarget.kind; - if (targetKind === 'claude-cli' || targetKind === 'copilot-cli') { + const providerKind = resolvedProviderSelection.resolvedProvider.kind; + if (providerKind === 'claude-cli' || providerKind === 'copilot-cli') { console.log(''); console.log(' TIP: For subagent-mode evals, use `agentv pipeline` instead of `eval run`.'); console.log(' The agent orchestrates executor + grader subagents directly.'); @@ -1714,14 +1728,14 @@ async function runSingleEvalFile(params: { // Resolve workers: CLI/config > target setting > default const workerPreference = workersOverride ?? options.workers; let resolvedWorkers = - workerPreference ?? resolvedTargetSelection.resolvedTarget.workers ?? DEFAULT_WORKERS; + workerPreference ?? resolvedProviderSelection.resolvedProvider.workers ?? DEFAULT_WORKERS; if (resolvedWorkers < 1 || resolvedWorkers > 50) { throw new Error(`Workers must be between 1 and 50, got: ${resolvedWorkers}`); } // VSCode providers require window focus, so only 1 worker is allowed const isVSCodeProvider = ['vscode', 'vscode-insiders'].includes( - resolvedTargetSelection.resolvedTarget.kind, + resolvedProviderSelection.resolvedProvider.kind, ); if (isVSCodeProvider && resolvedWorkers > 1) { console.warn( @@ -1732,9 +1746,9 @@ async function runSingleEvalFile(params: { // Auto-provision subagents for VSCode targets if (isVSCodeProvider) { - const vsConfig = resolvedTargetSelection.resolvedTarget.config as { executable?: string }; + const vsConfig = resolvedProviderSelection.resolvedProvider.config as { executable?: string }; await ensureVSCodeSubagents({ - kind: resolvedTargetSelection.resolvedTarget.kind as 'vscode' | 'vscode-insiders', + kind: resolvedProviderSelection.resolvedProvider.kind as 'vscode' | 'vscode-insiders', count: resolvedWorkers, verbose: options.verbose, vscodeCmd: vsConfig.executable, @@ -1744,8 +1758,8 @@ async function runSingleEvalFile(params: { const results = await evaluationRunner({ testFilePath, repoRoot, - target: resolvedTargetSelection.resolvedTarget, - targets: resolvedTargetSelection.definitions, + target: resolvedProviderSelection.resolvedProvider, + targets: resolvedProviderSelection.definitions, env: process.env, maxRetries: Math.max(0, options.maxRetries), agentTimeoutMs, @@ -1754,7 +1768,10 @@ async function runSingleEvalFile(params: { // Skip cache if not enabled if (!cache) return false; // Skip cache when target has temperature > 0 (non-deterministic) - const targetConfig = resolvedTargetSelection.resolvedTarget.config as Record; + const targetConfig = resolvedProviderSelection.resolvedProvider.config as Record< + string, + unknown + >; if (shouldSkipCacheForTemperature(targetConfig)) { if (options.verbose) { console.log('Cache skipped: target temperature > 0'); @@ -1777,7 +1794,7 @@ async function runSingleEvalFile(params: { defaultGraderTarget: options.defaultGraderTarget, model: options.model, threshold: params.threshold, - targetHooks: resolvedTargetSelection.targetHooks, + targetHooks: resolvedProviderSelection.providerHooks, replayRecording, providerFactory, onResult: async (result: EvaluationResult) => { @@ -1790,7 +1807,7 @@ async function runSingleEvalFile(params: { await outputWriter.append(trimmedResult); }, onProgress: async (event) => { - const testCaseKeyId = matrixMode ? `${event.testId}@${targetName}` : event.testId; + const testCaseKeyId = matrixMode ? `${event.testId}@${providerLabel}` : event.testId; const testCaseKey = makeTestCaseKey(testFilePath, testCaseKeyId); if (event.status === 'pending' && !seenTestCases.has(testCaseKey)) { seenTestCases.add(testCaseKey); @@ -1806,7 +1823,7 @@ async function runSingleEvalFile(params: { progressReporter.update(displayId, { workerId: displayId, - testId: matrixMode ? `${event.testId}@${targetName}` : event.testId, + testId: matrixMode ? `${event.testId}@${providerLabel}` : event.testId, status: event.status, startedAt: event.startedAt, completedAt: event.completedAt, @@ -1835,7 +1852,7 @@ export interface RunEvalResult { readonly executionErrorCount: number; readonly outputPath: string; readonly testFiles: readonly string[]; - readonly target?: string; + readonly provider?: string; /** True when --threshold is set and mean score is below the threshold */ readonly thresholdFailed?: boolean; /** True when all tests had execution errors and no evaluation was performed */ @@ -1881,8 +1898,12 @@ export async function runEvalCommand( } let options = normalizeOptions(input.rawOptions, config, yamlConfig?.execution); - if (yamlConfig?.defaults?.provider && options.cliTargets.length === 0 && !options.target) { - options = { ...options, target: yamlConfig.defaults.provider }; + if ( + yamlConfig?.defaults?.provider && + options.cliProviderLabels.length === 0 && + !options.provider + ) { + options = { ...options, provider: yamlConfig.defaults.provider }; } if (yamlConfig?.defaults?.grader) { options = { ...options, defaultGraderTarget: yamlConfig.defaults.grader }; @@ -2146,7 +2167,7 @@ export async function runEvalCommand( readonly testIds: readonly string[]; readonly testCases: readonly EvalTest[]; readonly selections: readonly { - selection: TargetSelection; + selection: ProviderSelection; inlineTargetLabel: string; }[]; readonly trialsConfig?: TrialsConfig; @@ -2248,7 +2269,7 @@ export async function runEvalCommand( for (const meta of fileMetadata.values()) { for (const test of meta.testCases) { for (const { selection } of meta.selections) { - const target = selection.targetName; + const target = selection.providerLabel; const variant = targetVariantForSelection(selection); if (rerunIncludeKeys) { if (resumeIdentityMatches(rerunIncludeKeys, test, target, variant)) { @@ -2323,13 +2344,13 @@ export async function runEvalCommand( for (const testId of meta.testIds) { const testCaseKey = makeTestCaseKey( testFilePath, - meta.selections.length > 1 ? `${testId}@${selection.targetName}` : testId, + meta.selections.length > 1 ? `${testId}@${selection.providerLabel}` : testId, ); seenTestCases.add(testCaseKey); const displayId = displayIdTracker.getOrAssign(testCaseKey); progressReporter.update(displayId, { workerId: displayId, - testId: meta.selections.length > 1 ? `${testId}@${selection.targetName}` : testId, + testId: meta.selections.length > 1 ? `${testId}@${selection.providerLabel}` : testId, status: 'pending', targetLabel: inlineTargetLabel, }); @@ -2478,7 +2499,7 @@ export async function runEvalCommand( input: testCase.input as EvaluationResult['input'], output: [{ role: 'assistant' as const, content: budgetMsg }], finalOutput: budgetMsg, - target: selection.targetName, + target: selection.providerLabel, testId: testCase.testId ?? testCase.id, conversationId: testCase.conversation_id, error: budgetMsg, @@ -2489,7 +2510,7 @@ export async function runEvalCommand( failureStage: 'setup' as const, failureReasonCode: 'budget_exceeded' as const, executionError: { message: budgetMsg, stage: 'setup' as const }, - target: selection.targetName, + target: selection.providerLabel, variant: explicitVariant, })); for (const r of skippedResults) { @@ -2521,7 +2542,7 @@ export async function runEvalCommand( limitTarget(async () => { // Target selection is suite/experiment/CLI runtime policy; every selected // target runs every filtered test case for this eval file. - const targetName = selection.targetName; + const providerLabel = selection.providerLabel; const applicableTestCases = targetPrep.testCases; // --resume skips completed tests; --rerun-failed only includes prior failed/error tests. @@ -2530,14 +2551,14 @@ export async function runEvalCommand( resumeIdentityMatches( rerunIncludeKeys, test, - targetName, + providerLabel, targetVariantForSelection(selection), ), ) : resumeSkipKeys ? applicableTestCases.filter((test) => { const variant = targetVariantForSelection(selection); - return !resumeIdentityMatches(resumeSkipKeys, test, targetName, variant); + return !resumeIdentityMatches(resumeSkipKeys, test, providerLabel, variant); }) : applicableTestCases; @@ -2618,7 +2639,7 @@ export async function runEvalCommand( input: testCase.input as EvaluationResult['input'], output: [{ role: 'assistant' as const, content: message }], finalOutput: message, - target: selection.targetName, + target: selection.providerLabel, testId: testCase.testId ?? testCase.id, conversationId: testCase.conversation_id, error: message, @@ -2630,7 +2651,7 @@ export async function runEvalCommand( failureReasonCode: 'setup_error' as const, durationMs: 0, tokenUsage: { input: 0, output: 0 }, - target: selection.targetName, + target: selection.providerLabel, variant: explicitVariant, }, testFilePath, @@ -2709,7 +2730,7 @@ export async function runEvalCommand( if (allResults.length > 0) { const evalFile = activeTestFiles.length === 1 ? path.relative(cwd, activeTestFiles[0]) : ''; const sourceTests = activeSourceTests; - const taskBundleTargets = buildTaskBundleTargetSelections(activeTestFiles, fileMetadata); + const taskBundleTargets = buildTaskBundleProviderSelections(activeTestFiles, fileMetadata); if (isResumeAppend) { // Resume mode: write per-test artifacts for newly-run tests, then // aggregate the run from its full row manifest (old + new results with @@ -2830,7 +2851,7 @@ export async function runEvalCommand( // Suggest resume commands when execution errors are detected if (summary.executionErrorCount > 0 && !options.retryErrors && !options.resume) { const evalFileArgs = activeTestFiles.map((f) => path.relative(cwd, f)).join(' '); - const targetFlag = options.target ? ` --provider ${options.target}` : ''; + const targetFlag = options.provider ? ` --provider ${options.provider}` : ''; const relativeRunDir = path.relative(cwd, runDir); console.log( `\nTip: ${summary.executionErrorCount} execution error(s) detected. Re-run failed tests with:\n` + @@ -2861,7 +2882,7 @@ export async function runEvalCommand( executionErrorCount: summary.executionErrorCount, outputPath, testFiles: activeTestFiles, - target: options.target, + provider: options.provider, thresholdFailed, allExecutionErrors, budgetExceeded: runBudgetExceeded || undefined, diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 8b3515035..1e43be032 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -36,14 +36,14 @@ function resolveUseTarget( name: string, definitions: readonly ProviderDefinition[], env: NodeJS.ProcessEnv, - targetsFilePath: string, + providersFilePath: string, ): ProviderDefinition { const maxDepth = 5; let current: ProviderDefinition | undefined = definitions.find((d) => d.name === name); if (!current) { const available = listProviderLabels(definitions).join(', '); throw new Error( - `Provider '${name}' not found in ${targetsFilePath}. Available providers: ${available}`, + `Provider '${name}' not found in ${providersFilePath}. Available providers: ${available}`, ); } @@ -64,7 +64,7 @@ function resolveUseTarget( if (!next) { const available = listProviderLabels(definitions).join(', '); throw new Error( - `Provider '${name}' use_target '${resolved.trim()}' not found in ${targetsFilePath}. Available providers: ${available}`, + `Provider '${name}' use_target '${resolved.trim()}' not found in ${providersFilePath}. Available providers: ${available}`, ); } current = next; @@ -73,48 +73,48 @@ function resolveUseTarget( return current; } -export async function readTestSuiteTarget(testFilePath: string): Promise { +export async function readTestSuiteProvider(testFilePath: string): Promise { const metadata = await readTestSuiteMetadata(testFilePath); return metadata.target; } -export async function readTestSuiteTargets( +export async function readTestSuiteProviders( testFilePath: string, ): Promise { const metadata = await readTestSuiteMetadata(testFilePath); return metadata.targets; } -export interface TargetSelection { +export interface ProviderSelection { readonly definitions: readonly ProviderDefinition[]; - readonly resolvedTarget: ResolvedProviderBackend; - readonly targetName: string; - readonly targetLabel?: string; - readonly targetSource: 'cli' | 'test-file' | 'default'; - readonly targetsFilePath: string; - /** Per-target hooks from eval file (eval-level customization) */ - readonly targetHooks?: import('@agentv/core').TargetHooksConfig; + readonly resolvedProvider: ResolvedProviderBackend; + readonly providerLabel: string; + readonly providerDisplayLabel?: string; + readonly providerSource: 'cli' | 'test-file' | 'default'; + readonly providersFilePath: string; + /** Per-provider hooks from eval file (eval-level customization) */ + readonly providerHooks?: import('@agentv/core').TargetHooksConfig; } -export interface TargetSelectionOptions { +export interface ProviderSelectionOptions { readonly testFilePath: string; readonly repoRoot: string; readonly cwd: string; - readonly explicitTargetsPath?: string; + readonly explicitProvidersPath?: string; readonly providerDefinitions?: readonly ProviderDefinition[]; readonly providerDefinitionsSource?: string; readonly requireExplicitProviderCatalog?: boolean; readonly allowLegacyTargetFiles?: boolean; - readonly cliTargetName?: string; - readonly cliTargetNames?: readonly string[]; - readonly fileTargetName?: string; + readonly cliProviderLabel?: string; + readonly cliProviderLabels?: readonly string[]; + readonly fileProviderLabel?: string; readonly fileTargetSpec?: EvalTargetSpec; readonly modelOverride?: string; readonly env: NodeJS.ProcessEnv; } async function readProviderCatalog(options: { - readonly explicitTargetsPath?: string; + readonly explicitProvidersPath?: string; readonly providerDefinitions?: readonly ProviderDefinition[]; readonly providerDefinitionsSource?: string; readonly requireExplicitProviderCatalog?: boolean; @@ -123,38 +123,38 @@ async function readProviderCatalog(options: { readonly cwd: string; readonly allowLegacyTargetFiles?: boolean; }): Promise<{ readonly definitions: readonly ProviderDefinition[]; readonly sourcePath: string }> { - if (!options.explicitTargetsPath && options.providerDefinitions) { + if (!options.explicitProvidersPath && options.providerDefinitions) { return { definitions: options.providerDefinitions, sourcePath: options.providerDefinitionsSource ?? '.agentv/config.yaml:providers', }; } - if (!options.explicitTargetsPath && options.requireExplicitProviderCatalog) { + if (!options.explicitProvidersPath && options.requireExplicitProviderCatalog) { throw new Error( 'No provider catalog configured. Add `providers:` to .agentv/config.yaml, use `providers: file://providers.yaml`, or pass --providers .', ); } - const targetsFilePath = await discoverProvidersFile({ - explicitPath: options.explicitTargetsPath, + const providersFilePath = await discoverProvidersFile({ + explicitPath: options.explicitProvidersPath, testFilePath: options.testFilePath, repoRoot: options.repoRoot, cwd: options.cwd, allowLegacyTargetFiles: options.allowLegacyTargetFiles, }); - await validateProviderCatalogFile(targetsFilePath); + await validateProviderCatalogFile(providersFilePath); return { - definitions: await readProviderDefinitions(targetsFilePath), - sourcePath: targetsFilePath, + definitions: await readProviderDefinitions(providersFilePath), + sourcePath: providersFilePath, }; } -async function validateProviderCatalogFile(targetsFilePath: string): Promise { - const validationResult = await validateTargetsFile(targetsFilePath); +async function validateProviderCatalogFile(providersFilePath: string): Promise { + const validationResult = await validateTargetsFile(providersFilePath); const warnings = validationResult.errors.filter((e) => e.severity === 'warning'); const useColors = isTTY(); if (warnings.length > 0) { - console.warn(`\nWarnings in ${targetsFilePath}:`); + console.warn(`\nWarnings in ${providersFilePath}:`); for (const warning of warnings) { const location = warning.location ? ` [${warning.location}]` : ''; const prefix = useColors ? `${ANSI_YELLOW} ⚠${ANSI_RESET}` : ' ⚠'; @@ -166,7 +166,7 @@ async function validateProviderCatalogFile(targetsFilePath: string): Promise e.severity === 'error'); if (errors.length > 0) { - console.error(`\nErrors in ${targetsFilePath}:`); + console.error(`\nErrors in ${providersFilePath}:`); for (const error of errors) { const location = error.location ? ` [${error.location}]` : ''; const prefix = useColors ? `${ANSI_RED} ✗${ANSI_RESET}` : ' ✗'; @@ -177,16 +177,16 @@ async function validateProviderCatalogFile(targetsFilePath: string): Promise 0) { return { name: fileName, source: 'test-file' }; } @@ -206,14 +206,14 @@ function overlayTargetDefinition(params: { readonly spec: EvalTargetSpec | undefined; readonly definitions: readonly ProviderDefinition[]; readonly env: NodeJS.ProcessEnv; - readonly targetsFilePath: string; + readonly providersFilePath: string; }): ProviderDefinition | undefined { - const { spec, definitions, env, targetsFilePath } = params; + const { spec, definitions, env, providersFilePath } = params; if (!spec?.definition) { return undefined; } if (spec.extends) { - const base = resolveUseTarget(spec.extends, definitions, env, targetsFilePath); + const base = resolveUseTarget(spec.extends, definitions, env, providersFilePath); return { ...base, ...spec.definition, @@ -243,23 +243,25 @@ async function resolveInlineDefinitionEnvironment( return resolved ?? definition; } -export async function selectTarget(options: TargetSelectionOptions): Promise { +export async function selectProvider( + options: ProviderSelectionOptions, +): Promise { const { testFilePath, repoRoot, cwd, - explicitTargetsPath, + explicitProvidersPath, providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog, allowLegacyTargetFiles, - cliTargetName, + cliProviderLabel, modelOverride, env, } = options; const providerCatalog = await readProviderCatalog({ - explicitTargetsPath, + explicitProvidersPath, providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog, @@ -269,21 +271,23 @@ export async function selectTarget(options: TargetSelectionOptions): Promise { +): Promise { const { testFilePath, repoRoot, cwd, - explicitTargetsPath, + explicitProvidersPath, providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog, allowLegacyTargetFiles, env, - targetNames, - targetRefs, + providerLabels, + providerRefs, modelOverride, } = options; // Build a lookup for target hooks from eval target refs const hooksMap = new Map(); const labelsMap = new Map(); - if (targetRefs) { - for (const ref of targetRefs) { + if (providerRefs) { + for (const ref of providerRefs) { if (ref.hooks) { hooksMap.set(ref.name, ref.hooks); } @@ -352,7 +356,7 @@ export async function selectMultipleTargets( } const providerCatalog = await readProviderCatalog({ - explicitTargetsPath, + explicitProvidersPath, providerDefinitions, providerDefinitionsSource, requireExplicitProviderCatalog, @@ -361,13 +365,13 @@ export async function selectMultipleTargets( cwd, allowLegacyTargetFiles, }); - const targetsFilePath = providerCatalog.sourcePath; + const providersFilePath = providerCatalog.sourcePath; const fileDefinitions = providerCatalog.definitions; // Inject synthetic definitions from eval target refs (for use_target delegation) const definitions = [...fileDefinitions]; - if (targetRefs) { - for (const ref of targetRefs) { + if (providerRefs) { + for (const ref of providerRefs) { if (ref.definition && !fileDefinitions.some((d) => d.name === ref.name)) { definitions.push( await resolveInlineDefinitionEnvironment(ref.definition, testFilePath, 'providers'), @@ -378,28 +382,28 @@ export async function selectMultipleTargets( } } - const results: TargetSelection[] = []; + const results: ProviderSelection[] = []; - for (const name of targetNames) { + for (const name of providerLabels) { const targetDefinition = withModelOverride( - resolveUseTarget(name, definitions, env, targetsFilePath), + resolveUseTarget(name, definitions, env, providersFilePath), modelOverride, ); const hooks = hooksMap.get(name); - const targetLabel = labelsMap.get(name); + const providerDisplayLabel = labelsMap.get(name); try { - const resolvedTarget = resolveProviderDefinition(targetDefinition, env, testFilePath, { + const resolvedProvider = resolveProviderDefinition(targetDefinition, env, testFilePath, { emitDeprecationWarnings: false, }); results.push({ definitions, - resolvedTarget, - targetName: name, - ...(targetLabel ? { targetLabel } : {}), - targetSource: options.targetSource ?? 'cli', - targetsFilePath, - ...(hooks && { targetHooks: hooks }), + resolvedProvider, + providerLabel: name, + ...(providerDisplayLabel ? { providerDisplayLabel } : {}), + providerSource: options.providerSource ?? 'cli', + providersFilePath, + ...(hooks && { providerHooks: hooks }), }); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index 99b4fc0d6..ad61611a4 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -73,17 +73,17 @@ const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([ 'retry_status_codes', ]); -export interface TaskBundleTargetSelection { +export interface TaskBundleProviderSelection { readonly evalFileAbsolutePath?: string; - readonly targetName: string; - readonly resolvedTargetName?: string; + readonly providerLabel: string; + readonly resolvedProviderName?: string; readonly definitions: readonly ProviderDefinition[]; } export interface MaterializeTaskBundleOptions { readonly test: EvalTest; - readonly targetName: string; - readonly targetDefinitions: readonly ProviderDefinition[]; + readonly providerLabel: string; + readonly providerDefinitions: readonly ProviderDefinition[]; readonly outputDir: string; readonly cwd?: string; readonly repoRoot?: string; @@ -100,7 +100,7 @@ export interface MaterializedTaskBundlePaths { export interface MaterializeEvalBundleOptions { readonly evalFilePath: string; readonly tests: readonly EvalTest[]; - readonly targetSelections: readonly TaskBundleTargetSelection[]; + readonly providerSelections: readonly TaskBundleProviderSelection[]; readonly outputDir: string; readonly cwd?: string; readonly repoRoot?: string; @@ -580,7 +580,7 @@ function targetReferenceNames(target: ProviderDefinition): readonly string[] { } function selectTargetDefinitions( - targetName: string, + providerLabel: string, definitions: readonly ProviderDefinition[], ): readonly ProviderDefinition[] { const byName = new Map(definitions.map((definition) => [definition.name, definition])); @@ -602,7 +602,7 @@ function selectTargetDefinitions( } } - visit(targetName); + visit(providerLabel); return selected; } @@ -636,7 +636,7 @@ function bundledEvalFileName(evalFilePath: string): string { } function uniqueTargetDefinitions( - selections: readonly TaskBundleTargetSelection[], + selections: readonly TaskBundleProviderSelection[], tests: readonly EvalTest[] = [], ): readonly ProviderDefinition[] { const selected: ProviderDefinition[] = []; @@ -659,7 +659,7 @@ function uniqueTargetDefinitions( const graderTargetNames = collectGraderTargetNames(tests); for (const selection of selections) { - addDefinitions([selection.targetName, ...graderTargetNames], selection.definitions); + addDefinitions([selection.providerLabel, ...graderTargetNames], selection.definitions); } return selected; @@ -703,14 +703,14 @@ function collectGraderTargetNames(tests: readonly EvalTest[]): readonly string[] } function selectTaskBundleTargetDefinitions( - targetName: string, + providerLabel: string, definitions: readonly ProviderDefinition[], tests: readonly EvalTest[], ): readonly ProviderDefinition[] { const selected: ProviderDefinition[] = []; const seen = new Set(); - for (const name of [targetName, ...collectGraderTargetNames(tests)]) { + for (const name of [providerLabel, ...collectGraderTargetNames(tests)]) { for (const definition of selectTargetDefinitions(name, definitions)) { if (seen.has(definition.name)) { continue; @@ -786,15 +786,15 @@ function serializeTargetDefinitions( return definitions.map((definition) => serializeTargetDefinition(definition, rewrites)); } -function uniqueTargetNames(selections: readonly TaskBundleTargetSelection[]): readonly string[] { +function uniqueTargetNames(selections: readonly TaskBundleProviderSelection[]): readonly string[] { const names: string[] = []; const seen = new Set(); for (const selection of selections) { - if (seen.has(selection.targetName)) { + if (seen.has(selection.providerLabel)) { continue; } - seen.add(selection.targetName); - names.push(selection.targetName); + seen.add(selection.providerLabel); + names.push(selection.providerLabel); } return names; } @@ -1258,12 +1258,12 @@ export async function materializeTaskBundle( return undefined; } - const targetDefinitions = selectTaskBundleTargetDefinitions( - options.targetName, - options.targetDefinitions, + const providerDefinitions = selectTaskBundleTargetDefinitions( + options.providerLabel, + options.providerDefinitions, [options.test], ); - if (targetDefinitions.length === 0) { + if (providerDefinitions.length === 0) { return undefined; } @@ -1271,7 +1271,7 @@ export async function materializeTaskBundle( await mkdir(testDir, { recursive: true }); const providerEnvironmentReferences = await collectProviderEnvironmentReferences( - targetDefinitions, + providerDefinitions, path.dirname(options.test.source.evalFileAbsolutePath ?? options.test.source.evalFilePath), ); const copiedReferences = await copyReferences( @@ -1285,12 +1285,12 @@ export async function materializeTaskBundle( const providersPath = path.join(testDir, TASK_PROVIDERS_FILENAME); await writeYamlFile(evalPath, { - providers: [options.targetName], + providers: [options.providerLabel], prompts: [INPUT_PROMPT], tests: [evalCase], }); await writeYamlFile(providersPath, { - providers: serializeTargetDefinitions(targetDefinitions, rewrites), + providers: serializeTargetDefinitions(providerDefinitions, rewrites), }); return { @@ -1320,7 +1320,7 @@ export async function materializeEvalBundle( if (options.tests.length === 0) { throw new Error('Cannot bundle eval with no runnable tests.'); } - if (options.targetSelections.length === 0) { + if (options.providerSelections.length === 0) { throw new Error('Cannot bundle eval without a selected target.'); } @@ -1336,11 +1336,11 @@ export async function materializeEvalBundle( await mkdir(evalsDir, { recursive: true }); const evalFileDir = path.dirname(path.resolve(options.evalFilePath)); - const targetDefinitions = uniqueTargetDefinitions(options.targetSelections, options.tests); + const providerDefinitions = uniqueTargetDefinitions(options.providerSelections, options.tests); const workspaceReferences = await collectWorkspaceReferences(options.tests, evalFileDir); const environmentReferences = await collectEnvironmentReferences(options.tests, evalFileDir); const providerEnvironmentReferences = await collectProviderEnvironmentReferences( - targetDefinitions, + providerDefinitions, evalFileDir, ); const references: BundleSourceReference[] = [ @@ -1358,7 +1358,7 @@ export async function materializeEvalBundle( } const rewrites = buildPathRewrites(copied); - const targetNames = uniqueTargetNames(options.targetSelections); + const targetNames = uniqueTargetNames(options.providerSelections); const evalPath = path.join(evalsDir, bundledEvalFileName(options.evalFilePath)); const providersPath = path.join(outputDir, BUNDLE_PROVIDERS_FILENAME); const configPath = path.join(outputDir, BUNDLE_CONFIG_DIRNAME, BUNDLE_CONFIG_FILENAME); @@ -1371,7 +1371,7 @@ export async function materializeEvalBundle( prompts: [INPUT_PROMPT], tests: options.tests.map((test) => buildPortableEvalCase(test, rewrites)), }); - await writeYamlFile(providersPath, serializeTargetDefinitions(targetDefinitions, rewrites)); + await writeYamlFile(providersPath, serializeTargetDefinitions(providerDefinitions, rewrites)); await mkdir(path.dirname(configPath), { recursive: true }); await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` }); diff --git a/apps/cli/src/commands/grade/index.ts b/apps/cli/src/commands/grade/index.ts index a8fbe1f05..6a1d5070d 100644 --- a/apps/cli/src/commands/grade/index.ts +++ b/apps/cli/src/commands/grade/index.ts @@ -30,7 +30,7 @@ import { command, number, oneOf, option, optional, positional, string } from 'cm import { loadEnvFromHierarchy } from '../eval/env.js'; import { buildDefaultRunDir } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; -import { selectMultipleTargets } from '../eval/targets.js'; +import { selectMultipleProviders } from '../eval/targets.js'; interface PreparedBaseline { readonly status: 'initialized' | 'unavailable'; @@ -540,20 +540,20 @@ async function gradePreparedAttempt(options: { throw new Error(`Test ID '${testId}' not found in ${evalPath}`); } - const selections = await selectMultipleTargets({ + const selections = await selectMultipleProviders({ testFilePath: evalPath, repoRoot, cwd: process.cwd(), env: process.env, - targetNames: [manifest.target], - targetRefs: suite.targetRefs, + providerLabels: [manifest.target], + providerRefs: suite.targetRefs, }); const selection = selections[0]; if (!selection) { throw new Error(`Target '${manifest.target}' could not be resolved`); } const target = { - ...selection.resolvedTarget, + ...selection.resolvedProvider, name: manifest.target, } as ResolvedProviderBackend; diff --git a/apps/cli/src/commands/pipeline/input.ts b/apps/cli/src/commands/pipeline/input.ts index 33c3bdce3..c91939a82 100644 --- a/apps/cli/src/commands/pipeline/input.ts +++ b/apps/cli/src/commands/pipeline/input.ts @@ -43,7 +43,7 @@ import { command, option, optional, positional, string } from 'cmd-ts'; import { buildDefaultRunDir } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; -import { selectTarget } from '../eval/targets.js'; +import { selectProvider } from '../eval/targets.js'; export const evalInputCommand = command({ name: 'input', @@ -120,17 +120,17 @@ export const evalInputCommand = command({ let subagentModeAllowed = true; try { - const selection = await selectTarget({ + const selection = await selectProvider({ testFilePath: resolvedEvalPath, repoRoot, cwd: evalDir, - cliTargetName: provider, - explicitTargetsPath: providers, + cliProviderLabel: provider, + explicitProvidersPath: providers, env: process.env, }); - targetName = selection.targetName; - const resolved = selection.resolvedTarget; + targetName = selection.providerLabel; + const resolved = selection.resolvedProvider; subagentModeAllowed = resolved.subagentModeAllowed !== false; if (resolved.kind === 'cli') { diff --git a/apps/cli/src/commands/pipeline/run.ts b/apps/cli/src/commands/pipeline/run.ts index 531e12108..2f11d1f47 100644 --- a/apps/cli/src/commands/pipeline/run.ts +++ b/apps/cli/src/commands/pipeline/run.ts @@ -23,7 +23,7 @@ import { command, number, oneOf, option, optional, positional, string } from 'cm import { buildDefaultRunDir } from '../eval/result-layout.js'; import { findRepoRoot } from '../eval/shared.js'; -import { selectTarget } from '../eval/targets.js'; +import { selectProvider } from '../eval/targets.js'; import type { GraderTask } from './grade.js'; import { runScriptGraders } from './grade.js'; @@ -162,19 +162,19 @@ export const evalRunCommand = command({ let targetWorkers: number | undefined; try { - const selection = await selectTarget({ + const selection = await selectProvider({ testFilePath: resolvedEvalPath, repoRoot, cwd: evalDir, - cliTargetName: provider, - explicitTargetsPath: providers, + cliProviderLabel: provider, + explicitProvidersPath: providers, env: process.env, }); - targetName = selection.targetName; - targetWorkers = selection.resolvedTarget.workers; - if (selection.resolvedTarget.kind === 'cli') { + targetName = selection.providerLabel; + targetWorkers = selection.resolvedProvider.workers; + if (selection.resolvedProvider.kind === 'cli') { targetKind = 'cli'; - const config = selection.resolvedTarget.config; + const config = selection.resolvedProvider.config; targetInfo = { kind: 'cli', command: config.command, diff --git a/apps/cli/src/commands/prepare/index.ts b/apps/cli/src/commands/prepare/index.ts index 7dfefb00c..23e75fd03 100644 --- a/apps/cli/src/commands/prepare/index.ts +++ b/apps/cli/src/commands/prepare/index.ts @@ -21,7 +21,7 @@ import { command, oneOf, option, optional, positional, string } from 'cmd-ts'; import { loadEnvFromHierarchy } from '../eval/env.js'; import { findRepoRoot } from '../eval/shared.js'; -import { selectMultipleTargets } from '../eval/targets.js'; +import { selectMultipleProviders } from '../eval/targets.js'; type SetupStepStatus = 'ok' | 'skipped' | 'warning'; @@ -261,18 +261,18 @@ async function selectPrepareTarget(options: { readonly evalPath: string; readonly repoRoot: string; readonly target: string; - readonly targetRefs?: readonly EvalTargetRef[]; + readonly providerRefs?: readonly EvalTargetRef[]; }): Promise<{ readonly resolvedTarget: ResolvedProviderBackend; readonly targetHooks?: TargetHooksConfig; }> { - const selections = await selectMultipleTargets({ + const selections = await selectMultipleProviders({ testFilePath: options.evalPath, repoRoot: options.repoRoot, cwd: process.cwd(), env: process.env, - targetNames: [options.target], - targetRefs: options.targetRefs, + providerLabels: [options.target], + providerRefs: options.providerRefs, }); const selection = selections[0]; if (!selection) { @@ -280,10 +280,10 @@ async function selectPrepareTarget(options: { } return { resolvedTarget: { - ...selection.resolvedTarget, + ...selection.resolvedProvider, name: options.target, } as ResolvedProviderBackend, - ...(selection.targetHooks !== undefined && { targetHooks: selection.targetHooks }), + ...(selection.providerHooks !== undefined && { targetHooks: selection.providerHooks }), }; } @@ -307,7 +307,7 @@ async function prepareAttempt(options: { throw new Error(`Test ID '${options.testId}' not found in ${evalPath}`); } - const targetRefs = + const providerRefs = suite.targetSpec?.hooks !== undefined && suite.targetSpec.name === options.target ? [ ...(suite.targetRefs ?? []), @@ -318,7 +318,7 @@ async function prepareAttempt(options: { evalPath, repoRoot, target: options.target, - targetRefs, + providerRefs, }); const prepared = await prepareEvalWorkspace({ diff --git a/apps/cli/src/commands/runs/rerun.ts b/apps/cli/src/commands/runs/rerun.ts index 5e3994abf..5a11445ea 100644 --- a/apps/cli/src/commands/runs/rerun.ts +++ b/apps/cli/src/commands/runs/rerun.ts @@ -530,8 +530,8 @@ export const runsRerunCommand = command({ const result = await runEvalCommand({ testFiles: selected.map((bundle) => bundle.evalPath), rawOptions: { - target: targetOverrides, - targets: args.targets ? path.resolve(cwd, args.targets) : undefined, + provider: targetOverrides, + providers: args.targets ? path.resolve(cwd, args.targets) : undefined, output: outputDir, experiment: args.experiment ?? 'rerun', workers: args.workers, diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 7356e3772..550823246 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -2618,7 +2618,7 @@ describe('writeArtifactsFromResults', () => { taskBundleTargets: [ { evalFileAbsolutePath: evalFile, - targetName: 'gpt-4o', + providerLabel: 'gpt-4o', definitions: [ { name: 'gpt-4o', @@ -2738,7 +2738,7 @@ describe('writeArtifactsFromResults', () => { taskBundleTargets: [ { evalFileAbsolutePath: evalFile, - targetName: 'mock-target', + providerLabel: 'mock-target', definitions: [{ name: 'mock-target', provider: 'mock', response: 'ok' }], }, ], @@ -2790,8 +2790,8 @@ describe('writeArtifactsFromResults', () => { taskBundleTargets: [ { evalFileAbsolutePath: evalFile, - targetName: 'mock-target', - resolvedTargetName: 'mock-target-dry-run', + providerLabel: 'mock-target', + resolvedProviderName: 'mock-target-dry-run', definitions: [{ name: 'mock-target', provider: 'mock', response: 'ok' }], }, ], diff --git a/apps/cli/test/commands/eval/run-command-options.test.ts b/apps/cli/test/commands/eval/run-command-options.test.ts index 79b894733..cb7512d9e 100644 --- a/apps/cli/test/commands/eval/run-command-options.test.ts +++ b/apps/cli/test/commands/eval/run-command-options.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'bun:test'; import { buildEvalRunRawOptions } from '../../../src/commands/eval/commands/run.js'; describe('eval run provider CLI options', () => { - it('lowers public provider flags into existing internal run options', () => { + it('passes public provider flags through provider-named internal run options', () => { const rawOptions = buildEvalRunRawOptions({ provider: ['codex-host', 'claude-docker'], providers: '.agentv/providers.yaml', @@ -11,8 +11,8 @@ describe('eval run provider CLI options', () => { testId: ['case-*'], }); - expect(rawOptions.target).toEqual(['codex-host', 'claude-docker']); - expect(rawOptions.targets).toBe('.agentv/providers.yaml'); + expect(rawOptions.provider).toEqual(['codex-host', 'claude-docker']); + expect(rawOptions.providers).toBe('.agentv/providers.yaml'); expect(rawOptions.graderTarget).toBe('grader-gpt5-mini'); expect(rawOptions.filter).toEqual(['case-*']); }); diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts index 16b0cfe72..45d73299b 100644 --- a/apps/cli/test/commands/eval/targets.test.ts +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -5,7 +5,7 @@ import path from 'node:path'; import { loadTestSuite } from '@agentv/core'; -import { selectMultipleTargets } from '../../../src/commands/eval/targets.js'; +import { selectMultipleProviders } from '../../../src/commands/eval/targets.js'; describe('eval target selection', () => { let tempDir: string; @@ -41,21 +41,21 @@ describe('eval target selection', () => { ); const suite = await loadTestSuite(evalPath, tempDir); - const selections = await selectMultipleTargets({ + const selections = await selectMultipleProviders({ testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, - explicitTargetsPath: providersPath, + explicitProvidersPath: providersPath, env: {}, - targetNames: suite.targets ?? [], - targetRefs: suite.targetRefs, - targetSource: 'test-file', + providerLabels: suite.targets ?? [], + providerRefs: suite.targetRefs, + providerSource: 'test-file', }); expect(selections).toHaveLength(1); - expect(selections[0]?.targetName).toBe('openai:gpt-5.4-mini'); - expect(selections[0]?.targetLabel).toBeUndefined(); - expect(selections[0]?.resolvedTarget.kind).toBe('mock'); + expect(selections[0]?.providerLabel).toBe('openai:gpt-5.4-mini'); + expect(selections[0]?.providerDisplayLabel).toBeUndefined(); + expect(selections[0]?.resolvedProvider.kind).toBe('mock'); }); it('uses an explicit providers.yaml catalog path', async () => { @@ -83,19 +83,19 @@ describe('eval target selection', () => { ); const suite = await loadTestSuite(evalPath, tempDir); - const selections = await selectMultipleTargets({ + const selections = await selectMultipleProviders({ testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, - explicitTargetsPath: providersPath, + explicitProvidersPath: providersPath, env: {}, - targetNames: suite.targets ?? [], - targetRefs: suite.targetRefs, - targetSource: 'test-file', + providerLabels: suite.targets ?? [], + providerRefs: suite.targetRefs, + providerSource: 'test-file', }); - expect(selections[0]?.targetName).toBe('modern'); - expect(selections[0]?.resolvedTarget.config.response).toBe('modern'); + expect(selections[0]?.providerLabel).toBe('modern'); + expect(selections[0]?.resolvedProvider.config.response).toBe('modern'); }); it('uses an explicit directory containing providers.yaml', async () => { @@ -122,19 +122,19 @@ describe('eval target selection', () => { ); const suite = await loadTestSuite(evalPath, tempDir); - const selections = await selectMultipleTargets({ + const selections = await selectMultipleProviders({ testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, - explicitTargetsPath: agentvDir, + explicitProvidersPath: agentvDir, env: {}, - targetNames: suite.targets ?? [], - targetRefs: suite.targetRefs, - targetSource: 'test-file', + providerLabels: suite.targets ?? [], + providerRefs: suite.targetRefs, + providerSource: 'test-file', }); - expect(selections[0]?.targetName).toBe('directory-modern'); - expect(selections[0]?.resolvedTarget.config.response).toBe('directory'); + expect(selections[0]?.providerLabel).toBe('directory-modern'); + expect(selections[0]?.resolvedProvider.config.response).toBe('directory'); }); it('requires config or explicit provider catalog when requested', async () => { @@ -162,15 +162,15 @@ describe('eval target selection', () => { const suite = await loadTestSuite(evalPath, tempDir); await expect( - selectMultipleTargets({ + selectMultipleProviders({ testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, requireExplicitProviderCatalog: true, env: {}, - targetNames: suite.targets ?? [], - targetRefs: suite.targetRefs, - targetSource: 'test-file', + providerLabels: suite.targets ?? [], + providerRefs: suite.targetRefs, + providerSource: 'test-file', }), ).rejects.toThrow(/Add `providers:` to \.agentv\/config\.yaml/); }); @@ -193,7 +193,7 @@ describe('eval target selection', () => { ); const suite = await loadTestSuite(evalPath, tempDir); - const selections = await selectMultipleTargets({ + const selections = await selectMultipleProviders({ testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, @@ -209,14 +209,14 @@ describe('eval target selection', () => { providerDefinitionsSource: '.agentv/config.yaml:providers', requireExplicitProviderCatalog: true, env: {}, - targetNames: suite.targets ?? [], - targetRefs: suite.targetRefs, - targetSource: 'test-file', + providerLabels: suite.targets ?? [], + providerRefs: suite.targetRefs, + providerSource: 'test-file', }); - expect(selections[0]?.targetName).toBe('inline-modern'); - expect(selections[0]?.targetsFilePath).toBe('.agentv/config.yaml:providers'); - expect(selections[0]?.resolvedTarget.config.response).toBe('inline'); + expect(selections[0]?.providerLabel).toBe('inline-modern'); + expect(selections[0]?.providersFilePath).toBe('.agentv/config.yaml:providers'); + expect(selections[0]?.resolvedProvider.config.response).toBe('inline'); }); it('hard-rejects legacy targets.yaml as authored provider config', async () => { @@ -244,14 +244,14 @@ describe('eval target selection', () => { const suite = await loadTestSuite(evalPath, tempDir); await expect( - selectMultipleTargets({ + selectMultipleProviders({ testFilePath: evalPath, repoRoot: tempDir, cwd: tempDir, env: {}, - targetNames: suite.targets ?? [], - targetRefs: suite.targetRefs, - targetSource: 'test-file', + providerLabels: suite.targets ?? [], + providerRefs: suite.targetRefs, + providerSource: 'test-file', }), ).rejects.toThrow(/Authored targets\.yaml files were removed.*providers\.yaml/); }); diff --git a/apps/cli/test/commands/eval/task-bundle.test.ts b/apps/cli/test/commands/eval/task-bundle.test.ts index a8960c45a..ed0064110 100644 --- a/apps/cli/test/commands/eval/task-bundle.test.ts +++ b/apps/cli/test/commands/eval/task-bundle.test.ts @@ -91,8 +91,8 @@ describe('materializeTaskBundle', () => { const paths = await materializeTaskBundle({ test, - targetName: 'selected', - targetDefinitions: [ + providerLabel: 'selected', + providerDefinitions: [ { name: 'selected', provider: 'mock',