diff --git a/apps/cli/src/commands/eval/commands/bundle.ts b/apps/cli/src/commands/eval/commands/bundle.ts index cbe7522b5..c2c8feaf4 100644 --- a/apps/cli/src/commands/eval/commands/bundle.ts +++ b/apps/cli/src/commands/eval/commands/bundle.ts @@ -8,7 +8,7 @@ import { } from '@agentv/core'; import { array, command, multioption, option, optional, positional, string } from 'cmd-ts'; -import { discoverTargetsFile } from '../../../utils/targets.js'; +import { discoverProvidersFile } from '../../../utils/providers.js'; import { loadEnvFromHierarchy } from '../env.js'; import { findRepoRoot, resolveEvalPaths } from '../shared.js'; import { readTestSuiteTarget } from '../targets.js'; @@ -216,7 +216,7 @@ export const evalBundleCommand = command({ definitions = [suite.inlineTarget]; targetNames = unique(args.provider.length > 0 ? args.provider : [suite.inlineTarget.name]); } else { - const targetsFilePath = await discoverTargetsFile({ + const targetsFilePath = await discoverProvidersFile({ explicitPath: args.providers, testFilePath: evalFilePath, repoRoot, diff --git a/apps/cli/src/commands/eval/interactive.ts b/apps/cli/src/commands/eval/interactive.ts index 793d3276c..2ca14c17c 100644 --- a/apps/cli/src/commands/eval/interactive.ts +++ b/apps/cli/src/commands/eval/interactive.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { listProviderLabels, readProviderDefinitions } from '@agentv/core'; import { checkbox, confirm, number, search, select } from '@inquirer/prompts'; -import { PROVIDER_FILE_CANDIDATES, fileExists } from '../../utils/targets.js'; +import { PROVIDER_FILE_CANDIDATES, fileExists } from '../../utils/providers.js'; import { type DiscoveredEvalFile, discoverEvalFiles, @@ -202,14 +202,14 @@ async function promptTargetSelection(cwd: string, firstEvalPath: string): Promis const repoRoot = await findRepoRoot(cwd); // Try to find providers.yaml near the eval file first, then cwd/repoRoot. - const targetsPath = await findTargetsFile(cwd, repoRoot, firstEvalPath); + const providersPath = await findProvidersFile(cwd, repoRoot, firstEvalPath); - if (!targetsPath) { + if (!providersPath) { console.log(`${ANSI_DIM}No providers.yaml found. Using default provider.${ANSI_RESET}`); return 'default'; } - const definitions = await readProviderDefinitions(targetsPath); + const definitions = await readProviderDefinitions(providersPath); const targetNames = listProviderLabels(definitions); if (targetNames.length === 0) { @@ -239,12 +239,12 @@ async function promptTargetSelection(cwd: string, firstEvalPath: string): Promis }); } -async function findTargetsFile( +async function findProvidersFile( cwd: string, repoRoot: string, evalFilePath?: string, ): Promise { - // Build directory chain: eval file dir → cwd → repoRoot (mirrors discoverTargetsFile) + // Build directory chain: eval file dir -> cwd -> repoRoot. const dirsToSearch: string[] = []; if (evalFilePath) { diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 83aa61c06..8b3515035 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -10,7 +10,7 @@ import { resolveProviderDefinitionEnvironments, } from '@agentv/core'; import { validateTargetsFile } from '@agentv/core/evaluation/validation'; -import { discoverTargetsFile } from '../../utils/targets.js'; +import { discoverProvidersFile } from '../../utils/providers.js'; const ANSI_YELLOW = '\u001b[33m'; const ANSI_RED = '\u001b[31m'; @@ -134,7 +134,7 @@ async function readProviderCatalog(options: { 'No provider catalog configured. Add `providers:` to .agentv/config.yaml, use `providers: file://providers.yaml`, or pass --providers .', ); } - const targetsFilePath = await discoverTargetsFile({ + const targetsFilePath = await discoverProvidersFile({ explicitPath: options.explicitTargetsPath, testFilePath: options.testFilePath, repoRoot: options.repoRoot, diff --git a/apps/cli/src/commands/results/eval-runner.ts b/apps/cli/src/commands/results/eval-runner.ts index 8abba035f..e19eee345 100644 --- a/apps/cli/src/commands/results/eval-runner.ts +++ b/apps/cli/src/commands/results/eval-runner.ts @@ -27,7 +27,7 @@ import { listProviderLabels, readProviderDefinitions } from '@agentv/core'; import type { Context } from 'hono'; import type { Hono } from 'hono'; -import { PROVIDER_FILE_CANDIDATES } from '../../utils/targets.js'; +import { PROVIDER_FILE_CANDIDATES } from '../../utils/providers.js'; import { discoverEvalFiles } from '../eval/discover.js'; import { RESULT_INDEX_FILENAME, diff --git a/apps/cli/src/utils/targets.ts b/apps/cli/src/utils/providers.ts similarity index 90% rename from apps/cli/src/utils/targets.ts rename to apps/cli/src/utils/providers.ts index 7324c2d72..405f5a4f1 100644 --- a/apps/cli/src/utils/targets.ts +++ b/apps/cli/src/utils/providers.ts @@ -61,7 +61,9 @@ async function findProviderFileInDirectory(directory: string): Promise { +async function findLegacyGeneratedTargetFileInDirectory( + directory: string, +): Promise { for (const candidate of LEGACY_TARGET_FILE_CANDIDATES) { const fullPath = path.join(directory, candidate); if ((await pathKind(fullPath)) === 'file') { @@ -71,11 +73,12 @@ async function findLegacyTargetFileInDirectory(directory: string): Promise { const { explicitPath, testFilePath, repoRoot, cwd, allowLegacyTargetFiles = false } = options; @@ -95,7 +98,7 @@ export async function discoverTargetsFile(options: { if (providerFile) { return providerFile; } - const legacyTargetFile = await findLegacyTargetFileInDirectory(resolvedExplicit); + const legacyTargetFile = await findLegacyGeneratedTargetFileInDirectory(resolvedExplicit); if (legacyTargetFile) { if (allowLegacyTargetFiles) { return legacyTargetFile; @@ -123,7 +126,7 @@ export async function discoverTargetsFile(options: { } for (const directory of directories) { - const legacyTargetFile = await findLegacyTargetFileInDirectory(directory); + const legacyTargetFile = await findLegacyGeneratedTargetFileInDirectory(directory); if (legacyTargetFile) { if (allowLegacyTargetFiles) { return legacyTargetFile; diff --git a/apps/cli/test/commands/eval/targets.test.ts b/apps/cli/test/commands/eval/targets.test.ts index e15528b9a..16b0cfe72 100644 --- a/apps/cli/test/commands/eval/targets.test.ts +++ b/apps/cli/test/commands/eval/targets.test.ts @@ -98,6 +98,45 @@ describe('eval target selection', () => { expect(selections[0]?.resolvedTarget.config.response).toBe('modern'); }); + it('uses an explicit directory containing providers.yaml', async () => { + const agentvDir = path.join(tempDir, '.agentv'); + await mkdir(agentvDir, { recursive: true }); + await writeFile( + path.join(agentvDir, 'providers.yaml'), + ['- id: mock', ' label: directory-modern', ' response: directory', ''].join('\n'), + ); + const evalPath = path.join(tempDir, 'provider-directory-discovery.eval.yaml'); + await writeFile( + evalPath, + [ + 'providers:', + ' - directory-modern', + 'prompts:', + ' - "{{ input }}"', + 'tests:', + ' - id: provider-case', + ' criteria: ok', + ' vars:', + ' input: hello', + ].join('\n'), + ); + + const suite = await loadTestSuite(evalPath, tempDir); + const selections = await selectMultipleTargets({ + testFilePath: evalPath, + repoRoot: tempDir, + cwd: tempDir, + explicitTargetsPath: agentvDir, + env: {}, + targetNames: suite.targets ?? [], + targetRefs: suite.targetRefs, + targetSource: 'test-file', + }); + + expect(selections[0]?.targetName).toBe('directory-modern'); + expect(selections[0]?.resolvedTarget.config.response).toBe('directory'); + }); + it('requires config or explicit provider catalog when requested', async () => { const agentvDir = path.join(tempDir, '.agentv'); await mkdir(agentvDir, { recursive: true });