From 2287e7168956c0d40f96ab732648a23a5ac7b83b Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 02:25:09 +0200 Subject: [PATCH 1/2] feat(promptfoo): validate exported provider configs --- .github/workflows/ci.yml | 28 ++ .../docs/next/reference/promptfoo-parity.mdx | 10 + package.json | 1 + scripts/export-promptfoo-config.test.ts | 104 ++++++++ scripts/export-promptfoo-config.ts | 249 ++++++++++++++++++ .../environment-unsupported.agentv.yaml | 18 ++ .../provider-surface.agentv.yaml | 29 ++ 7 files changed, 439 insertions(+) create mode 100644 scripts/export-promptfoo-config.test.ts create mode 100644 scripts/export-promptfoo-config.ts create mode 100644 scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml create mode 100644 scripts/fixtures/promptfoo-export/provider-surface.agentv.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30171aa3e..39baa0e86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,6 +169,34 @@ jobs: - name: Run tests run: bun run test + promptfoo-export: + name: Promptfoo Export + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version-file: package.json + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Run Promptfoo export tests + run: bun run validate:promptfoo-export + + - name: Export Promptfoo validation fixture + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/promptfoo-export" + bun scripts/export-promptfoo-config.ts \ + --input scripts/fixtures/promptfoo-export/provider-surface.agentv.yaml \ + --output "$RUNNER_TEMP/promptfoo-export/promptfooconfig.yaml" + + - name: Validate exported config with Promptfoo + run: bunx promptfoo@0.121.15 validate config -c "$RUNNER_TEMP/promptfoo-export/promptfooconfig.yaml" + links: name: Check Links runs-on: ubuntu-latest diff --git a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx index 2890da8d8..4c04fe746 100644 --- a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx +++ b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx @@ -26,6 +26,16 @@ and `inputs`, or provider maps such as Promptfoo may ignore or strip unknown keys and will not execute AgentV environment setup without a transpiler or wrapper. +For AgentV-only provider IDs such as `agentv:codex-cli`, use the Promptfoo +export path instead of running the AgentV YAML directly with Promptfoo. The +exporter lowers supported AgentV-only IDs to Promptfoo-readable provider +references such as +`file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts` +and fails with a diagnostic for unsupported AgentV-only semantics such as +top-level `environment`. The generated TypeScript provider is emitted as a +Promptfoo custom provider module so Promptfoo config validation can load it +without live provider calls. + Use this matrix when translating a Promptfoo-style normal eval into AgentV YAML. It documents which surfaces align directly, which AgentV surfaces are cleaner greenfield extensions, and which Promptfoo surfaces are deferred until AgentV diff --git a/package.json b/package.json index 0137e4f5e..5ffebe9c3 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js", "beads:check": "bun scripts/check-beads-context.ts", "debug:pi-sdk-tools": "bun scripts/debug-pi-sdk-tools.ts", + "validate:promptfoo-export": "bun test scripts/export-promptfoo-config.test.ts", "validate:examples": "EVAL_CRITERIA=placeholder CUSTOM_SYSTEM_PROMPT=placeholder bun scripts/validate-example-evals.ts", "eval:baseline-check": "bun scripts/check-eval-baselines.ts", "release": "bun scripts/release.ts", diff --git a/scripts/export-promptfoo-config.test.ts b/scripts/export-promptfoo-config.test.ts new file mode 100644 index 000000000..7079492b1 --- /dev/null +++ b/scripts/export-promptfoo-config.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'bun:test'; +import { existsSync, mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import YAML from 'yaml'; +import { PromptfooExportDiagnostic, exportPromptfooConfig } from './export-promptfoo-config'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const FIXTURE_DIR = path.join(ROOT, 'scripts', 'fixtures', 'promptfoo-export'); + +function outputPath(name: string): string { + return path.join(mkdtempSync(path.join(tmpdir(), 'agentv-promptfoo-export-')), name); +} + +function parseYamlFile(filePath: string): Record { + return YAML.parse(readFileSync(filePath, 'utf8')) as Record; +} + +describe('exportPromptfooConfig', () => { + it('preserves Promptfoo-native colon provider ids and labels', () => { + const output = outputPath('promptfooconfig.yaml'); + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'provider-surface.agentv.yaml'), + outputPath: output, + }); + + const exported = parseYamlFile(output); + const providers = exported.providers as Array>; + + expect(providers[0]).toMatchObject({ + id: 'openai:responses:gpt-5.4', + label: 'responses-direct', + }); + expect(providers[1]).toMatchObject({ + id: 'openai:codex-sdk', + label: 'codex-sdk-direct', + }); + }); + + it('lowers agentv:codex-cli to a generated Promptfoo file provider and preserves label', () => { + const output = outputPath('promptfooconfig.yaml'); + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'provider-surface.agentv.yaml'), + outputPath: output, + }); + + const exported = parseYamlFile(output); + const providers = exported.providers as Array>; + + expect(providers[2]).toMatchObject({ + id: 'file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts', + label: 'codex-cli-exported', + }); + expect(providers[2]).not.toHaveProperty('runtime'); + expect( + existsSync( + path.join( + path.dirname(output), + '.agentv', + 'generated', + 'promptfoo', + 'providers', + 'codex-cli-provider.ts', + ), + ), + ).toBe(true); + }); + + it('translates AgentV snake_case config keys for Promptfoo validation', () => { + const output = outputPath('promptfooconfig.yaml'); + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'provider-surface.agentv.yaml'), + outputPath: output, + }); + + const exported = parseYamlFile(output); + expect(exported).toHaveProperty('defaultTest'); + expect(exported).toHaveProperty('evaluateOptions'); + expect(exported.evaluateOptions).toEqual({ maxConcurrency: 1 }); + expect(exported).not.toHaveProperty('default_test'); + expect(exported).not.toHaveProperty('evaluate_options'); + }); + + it('fails clearly on unsupported top-level environment semantics', () => { + const output = outputPath('promptfooconfig.yaml'); + expect(() => + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'environment-unsupported.agentv.yaml'), + outputPath: output, + }), + ).toThrow(PromptfooExportDiagnostic); + + try { + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'environment-unsupported.agentv.yaml'), + outputPath: output, + }); + } catch (error) { + expect(error).toBeInstanceOf(PromptfooExportDiagnostic); + expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_environment'); + expect((error as Error).message).toContain("Top-level 'environment' is AgentV-only"); + } + }); +}); diff --git a/scripts/export-promptfoo-config.ts b/scripts/export-promptfoo-config.ts new file mode 100644 index 000000000..69d6b4e22 --- /dev/null +++ b/scripts/export-promptfoo-config.ts @@ -0,0 +1,249 @@ +#!/usr/bin/env bun +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import YAML from 'yaml'; + +type JsonMap = Record; + +export class PromptfooExportDiagnostic extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + this.name = 'PromptfooExportDiagnostic'; + } +} + +export interface ExportPromptfooConfigOptions { + inputPath: string; + outputPath: string; +} + +// Promptfoo validates JavaScript/TypeScript provider modules as class providers +// only when the provider id ends at the file extension. The `:callApi` suffix is +// valid for some Promptfoo file-backed hooks/assertions, but not provider ids. +const GENERATED_CODEX_CLI_PROVIDER_REF = + 'file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts'; + +const AGENTV_PROVIDER_LOWERINGS: Record = { + 'agentv:codex-cli': GENERATED_CODEX_CLI_PROVIDER_REF, +}; + +const TOP_LEVEL_KEY_RENAMES: Record = { + default_test: 'defaultTest', + evaluate_options: 'evaluateOptions', +}; + +const EVALUATE_OPTIONS_KEY_RENAMES: Record = { + max_concurrency: 'maxConcurrency', +}; + +const AGENTV_ONLY_PROVIDER_KEYS = new Set(['runtime']); + +function assertRecord(value: unknown, label: string): JsonMap { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new PromptfooExportDiagnostic( + 'invalid_config', + `${label} must be a YAML object for Promptfoo export.`, + ); + } + return value as JsonMap; +} + +function lowerProviderId(id: string): string { + if (!id.startsWith('agentv:')) { + return id; + } + + const lowered = AGENTV_PROVIDER_LOWERINGS[id]; + if (!lowered) { + throw new PromptfooExportDiagnostic( + 'unsupported_provider', + `Unsupported AgentV-only provider '${id}'. Export currently supports: ${Object.keys( + AGENTV_PROVIDER_LOWERINGS, + ).join(', ')}.`, + ); + } + return lowered; +} + +function lowerProviderObject(provider: JsonMap): JsonMap { + const lowered: JsonMap = {}; + for (const [key, value] of Object.entries(provider)) { + if (AGENTV_ONLY_PROVIDER_KEYS.has(key)) { + continue; + } + lowered[key] = key === 'id' && typeof value === 'string' ? lowerProviderId(value) : value; + } + return lowered; +} + +function looksLikeProviderOptionsMap(provider: JsonMap): boolean { + return !('id' in provider) && Object.keys(provider).length === 1; +} + +function lowerProviderEntry(provider: unknown): unknown { + if (typeof provider === 'string') { + return lowerProviderId(provider); + } + + if (provider && typeof provider === 'object' && !Array.isArray(provider)) { + const providerObject = provider as JsonMap; + if (looksLikeProviderOptionsMap(providerObject)) { + const [[id, options]] = Object.entries(providerObject); + return { [lowerProviderId(id)]: options }; + } + return lowerProviderObject(providerObject); + } + + return provider; +} + +function lowerProviders(providers: unknown): unknown { + if (typeof providers === 'string') { + return lowerProviderId(providers); + } + if (Array.isArray(providers)) { + return providers.map(lowerProviderEntry); + } + if (providers && typeof providers === 'object') { + return lowerProviderEntry(providers); + } + return providers; +} + +function renameObjectKeys(value: unknown, renames: Record): unknown { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return value; + } + + const output: JsonMap = {}; + for (const [key, childValue] of Object.entries(value as JsonMap)) { + output[renames[key] ?? key] = childValue; + } + return output; +} + +function promptfooConfigFromAgentVConfig(config: JsonMap): JsonMap { + if ('environment' in config) { + throw new PromptfooExportDiagnostic( + 'unsupported_environment', + "Top-level 'environment' is AgentV-only and cannot be exported to Promptfoo yet. Remove it for direct Promptfoo validation, or keep running this eval through AgentV.", + ); + } + + const promptfooConfig: JsonMap = {}; + for (const [key, value] of Object.entries(config)) { + const outputKey = TOP_LEVEL_KEY_RENAMES[key] ?? key; + if (key === 'providers') { + promptfooConfig[outputKey] = lowerProviders(value); + continue; + } + if (key === 'evaluate_options') { + promptfooConfig[outputKey] = renameObjectKeys(value, EVALUATE_OPTIONS_KEY_RENAMES); + continue; + } + promptfooConfig[outputKey] = value; + } + return promptfooConfig; +} + +function writeGeneratedProviderFiles(outputPath: string, config: JsonMap): void { + const serialized = YAML.stringify(config); + if (!serialized.includes(GENERATED_CODEX_CLI_PROVIDER_REF)) { + return; + } + + const providerPath = path.join( + path.dirname(outputPath), + '.agentv', + 'generated', + 'promptfoo', + 'providers', + 'codex-cli-provider.ts', + ); + mkdirSync(path.dirname(providerPath), { recursive: true }); + writeFileSync( + providerPath, + [ + 'export async function callApi() {', + ' return {', + " error: 'agentv:codex-cli was exported for Promptfoo config validation. Runtime Promptfoo execution requires a complete AgentV Promptfoo provider wrapper.',", + ' };', + '}', + '', + 'export default class CodexCliPromptfooProvider {', + ' label;', + ' config;', + '', + ' constructor(options = {}) {', + ' this.label = options.label;', + ' this.config = options.config;', + ' }', + '', + ' id() {', + " return this.label || 'agentv:codex-cli';", + ' }', + '', + ' async callApi() {', + ' return callApi();', + ' }', + '}', + '', + ].join('\n'), + 'utf8', + ); +} + +export function exportPromptfooConfig(options: ExportPromptfooConfigOptions): JsonMap { + const input = readFileSync(options.inputPath, 'utf8'); + const parsed = assertRecord(YAML.parse(input), options.inputPath); + const promptfooConfig = promptfooConfigFromAgentVConfig(parsed); + mkdirSync(path.dirname(options.outputPath), { recursive: true }); + writeFileSync(options.outputPath, YAML.stringify(promptfooConfig), 'utf8'); + writeGeneratedProviderFiles(options.outputPath, promptfooConfig); + return promptfooConfig; +} + +function parseArgs(argv: string[]): ExportPromptfooConfigOptions { + const options: Partial = {}; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--input') { + options.inputPath = argv[i + 1]; + i += 1; + continue; + } + if (arg === '--output') { + options.outputPath = argv[i + 1]; + i += 1; + } + } + + if (!options.inputPath || !options.outputPath) { + throw new PromptfooExportDiagnostic( + 'usage', + 'Usage: bun scripts/export-promptfoo-config.ts --input --output ', + ); + } + + return { + inputPath: path.resolve(options.inputPath), + outputPath: path.resolve(options.outputPath), + }; +} + +if (import.meta.main) { + try { + const options = parseArgs(Bun.argv.slice(2)); + exportPromptfooConfig(options); + console.log(`Exported Promptfoo config: ${options.outputPath}`); + } catch (error) { + if (error instanceof PromptfooExportDiagnostic) { + console.error(`error[${error.code}]: ${error.message}`); + process.exit(1); + } + throw error; + } +} diff --git a/scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml b/scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml new file mode 100644 index 000000000..0f3605401 --- /dev/null +++ b/scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml @@ -0,0 +1,18 @@ +description: Unsupported AgentV environment export fixture +environment: + runtime: host + workdir: . + setup: + command: ["bash", "-lc", "echo preparing workspace"] + +providers: + - id: openai:responses:gpt-5.4 + label: responses-direct + +prompts: + - "{{ question }}" + +tests: + - id: capital + vars: + question: What is the capital of France? diff --git a/scripts/fixtures/promptfoo-export/provider-surface.agentv.yaml b/scripts/fixtures/promptfoo-export/provider-surface.agentv.yaml new file mode 100644 index 000000000..6b9f627f7 --- /dev/null +++ b/scripts/fixtures/promptfoo-export/provider-surface.agentv.yaml @@ -0,0 +1,29 @@ +description: Promptfoo provider export validation +providers: + - id: openai:responses:gpt-5.4 + label: responses-direct + config: + api_key: "{{ env.OPENAI_API_KEY }}" + - id: openai:codex-sdk + label: codex-sdk-direct + - id: agentv:codex-cli + label: codex-cli-exported + runtime: host + config: + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + +prompts: + - "Answer briefly: {{ question }}" + +default_test: + assert: + - type: contains + value: Paris + +evaluate_options: + max_concurrency: 1 + +tests: + - id: capital + vars: + question: What is the capital of France? From 947489b2b1710c111c623caaf6d5176521687fd5 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 02:46:36 +0200 Subject: [PATCH 2/2] feat(promptfoo): export host environments --- .github/workflows/ci.yml | 2 +- .../docs/next/reference/promptfoo-parity.mdx | 36 +- scripts/export-promptfoo-config.test.ts | 51 ++- scripts/export-promptfoo-config.ts | 348 +++++++++++++++++- ...docker-environment-unsupported.agentv.yaml | 25 ++ .../environment-unsupported.agentv.yaml | 18 - .../host-environment.agentv.yaml | 34 ++ 7 files changed, 461 insertions(+), 53 deletions(-) create mode 100644 scripts/fixtures/promptfoo-export/docker-environment-unsupported.agentv.yaml delete mode 100644 scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml create mode 100644 scripts/fixtures/promptfoo-export/host-environment.agentv.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39baa0e86..2377301a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,7 +191,7 @@ jobs: set -euo pipefail mkdir -p "$RUNNER_TEMP/promptfoo-export" bun scripts/export-promptfoo-config.ts \ - --input scripts/fixtures/promptfoo-export/provider-surface.agentv.yaml \ + --input scripts/fixtures/promptfoo-export/host-environment.agentv.yaml \ --output "$RUNNER_TEMP/promptfoo-export/promptfooconfig.yaml" - name: Validate exported config with Promptfoo diff --git a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx index 4c04fe746..6be312b02 100644 --- a/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx +++ b/apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx @@ -6,14 +6,12 @@ sidebar: slug: docs/reference/promptfoo-parity --- -AgentV uses a similar eval config contract to Promptfoo for ordinary authored -evals: prompt matrices, test rows, vars, default test data, assertions, and -provider matrices all use the same broad shape. AgentV is a -Promptfoo-compatible superset at the provider declaration layer, not a promise -that every AgentV config executes unchanged in Promptfoo. AgentV keeps the wire -format `snake_case`, keeps provider selection identity explicit with `label`, -and adds repo-native environment recipes and artifact fields for agent -evaluation. +AgentV authored YAML should be Promptfoo-compatible by default for ordinary eval +surfaces: prompt matrices, test rows, vars, default test data, assertions, and +provider matrices all use the same broad shape. The intended authored-config +differences are `environment`, AgentV refs, and built-in AgentV providers. Export +fills those gaps for supported AgentV-native pieces so Promptfoo can validate or +load the resulting config without learning AgentV semantics. Promptfoo-shaped `providers` entries can be strings such as `openai:gpt-4.1-mini`, complete package provider strings such as @@ -21,20 +19,22 @@ Promptfoo-shaped `providers` entries can be strings such as `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as -`{ "openai:gpt-4": { label, config } }`. AgentV-only fields such as top-level -`environment` and provider-local runtime/testbed overlays are AgentV semantics; -Promptfoo may ignore or strip unknown keys and will not execute AgentV -environment setup without a transpiler or wrapper. +`{ "openai:gpt-4": { label, config } }`. Promptfoo-native provider IDs such as +`openai:responses:gpt-5.4`, `openai:codex-sdk`, and +`openai:codex-app-server` remain directly Promptfoo-readable. For AgentV-only provider IDs such as `agentv:codex-cli`, use the Promptfoo export path instead of running the AgentV YAML directly with Promptfoo. The exporter lowers supported AgentV-only IDs to Promptfoo-readable provider references such as `file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts` -and fails with a diagnostic for unsupported AgentV-only semantics such as -top-level `environment`. The generated TypeScript provider is emitted as a -Promptfoo custom provider module so Promptfoo config validation can load it -without live provider calls. +and exports supported host/filesystem `environment` recipes to a generated +Promptfoo extension such as +`file://.agentv/generated/promptfoo/extensions/host-environment.ts:beforeAll`. +The exporter passes the resolved isolated workdir through provider `config`, +default-test vars, and metadata. Docker `environment` export is intentionally +unsupported initially and fails with a diagnostic instead of degrading +isolation, image/context, mounts, services, resources, secrets, or provenance. Use this matrix when translating a Promptfoo-style normal eval into AgentV YAML. It documents which surfaces align directly, which AgentV surfaces are cleaner @@ -66,7 +66,7 @@ implements equivalent semantics directly. | Evaluate options | `evaluateOptions` for runtime controls. | `evaluate_options` for runtime controls. | Align with Promptfoo | AgentV uses `evaluate_options.repeat`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency`. | | Authored concurrency | Common Promptfoo usage includes runtime options such as `maxConcurrency`. | `evaluate_options.max_concurrency`. | Keep AgentV divergence | Do not author `execution.max_concurrency` or top-level `workers` in eval YAML. CLI `--workers` remains an operator override. | | Provider selection | Promptfoo normal evals use `providers`; `targets` can alias providers in unified config. | Use top-level `providers` for one or more systems under test. | Align with Promptfoo | Old AgentV `target`/`targets` authoring is hard-rejected. Use CLI `--provider`/`--providers` for runtime selection. | -| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. | Align with Promptfoo | Package provider strings must include the exported class/function segment after the final colon. AgentV-only fields such as `runtime` and provider-local testbed/runtime overlays are AgentV semantics. Promptfoo may ignore or strip unknown keys and will not execute AgentV environment setup without a transpiler or wrapper. | +| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. | Align with Promptfoo | Package provider strings must include the exported class/function segment after the final colon. Promptfoo-native provider IDs remain directly Promptfoo-readable. Built-in AgentV IDs such as `agentv:codex-cli` need export, which lowers them to a generated file provider for validation. | | Provider-prompt mapping | `providerPromptMap` maps provider identities to prompt subsets. | Rejected. Use explicit AgentV composition: separate eval suites/files for provider-specific prompt subsets, top-level `prompts` plus `tests`/`default_test.vars`, `providers` or CLI `--provider`, and tags/run metadata for grouping. | Removed/rejected surface | Do not author `providerPromptMap` or `provider_prompt_map`. | | Direct authored input | Promptfoo prompt authoring normally goes through `prompts` plus vars. | Top-level `input` and inline `tests[].input` are removed from normal authored eval YAML. External raw-case imports may still carry internal input rows for compatibility. | Removed AgentV extension | Author prompt text, chat/system/user messages, and file-backed prompt content as `prompts`; put row data in `tests[].vars` and shared defaults in `default_test.vars`. | | Authored preprocessors | Not Promptfoo's canonical output-shaping surface. | Rejected in current authored YAML. | Removed/rejected surface | Use `transform` at `default_test.options`, `tests[].options`, or the assertion that needs the shaped output. Historical versioned docs may still show old preprocessor examples. | @@ -78,7 +78,7 @@ implements equivalent semantics directly. | Skill assertions | Promptfoo includes `skill-used` for checking whether an agent invoked a named skill. | `type: skill-used` and `type: not-skill-used` with `value: ` or a matcher object. | Align with Promptfoo | AgentV evaluates these against normalized tool-call and skill-use trace data. | | Trajectory assertions | Promptfoo includes `trajectory:tool-used`, `trajectory:tool-sequence`, `trajectory:tool-args-match`, `trajectory:step-count`, and `trajectory:goal-success`. | AgentV accepts the same assertion names over AgentV-normalized traces. | Align with Promptfoo | Use these for tool presence, order, argument checks, step budgets, and LLM-judged goal success. | | Other Promptfoo trace assertions | Promptfoo also includes `tool-call-f1`, `trace-span-count`, `trace-span-duration`, and `trace-error-spans`. | Not accepted yet. | Defer/future-scope | Use `script` assertions or `execution-metrics` when current AgentV primitives cover the requirement. | -| Coding-agent testbeds | Promptfoo normal evals do not define a typed coding-agent testbed primitive. | `environment`, usually inline or `environment: file://...`, plus distinct top-level `env` and lifecycle `extensions`. | Keep AgentV extension | `environment` is an AgentV extension informed by Margin local-agent ergonomics and Harbor/Terminal-Bench Docker substrate evidence. Use it for host/Docker workdir, setup argv, fixtures, services, and repo materialization. Use top-level `env` for provider/eval variables and `extensions` for lifecycle hooks. | +| Coding-agent testbeds | Promptfoo normal evals do not define a typed coding-agent testbed primitive. | `environment`, usually inline or `environment: file://...`, plus distinct top-level `env` and lifecycle `extensions`. | Keep AgentV extension | `environment` is an AgentV extension informed by Margin local-agent ergonomics and Harbor/Terminal-Bench Docker substrate evidence. Export supports the host/filesystem subset by generating Promptfoo setup extensions and passing the resolved workdir through config, vars, and metadata. Docker export is not supported yet and fails clearly instead of degrading isolation or provenance. | | Run artifacts and inspection | Promptfoo owns its own result viewer and output formats. | AgentV writes `.agentv/results//` bundles with `summary.json`, `.internal/index.jsonl`, sidecars, and local Dashboard support. | Keep AgentV extension | AgentV-owned bundles are the source of truth for compare, Dashboard, CI, and adapters. Phoenix is link-out correlation only through safe external trace metadata. | | Grading result artifacts | Promptfoo `GradingResult` uses aggregate `pass`, `score`, `reason`, recursive `componentResults`, optional `assertion`, optional `namedScores`, and optional `metadata`. | AgentV persists the same concepts as `pass`, `score`, `reason`, recursive `component_results`, optional `assertion`, optional `named_scores`, and optional `metadata`. | Align with Promptfoo | AgentV keeps split filesystem run bundles and `snake_case` persisted fields. Promptfoo import/export adapters may translate `componentResults`/`namedScores` to and from AgentV `component_results`/`named_scores`. | | Compare command | Promptfoo has its own result comparison surfaces. | `agentv results compare `. | Keep AgentV extension | Compare consumes completed AgentV run indexes such as `.agentv/results//.internal/index.jsonl`. | diff --git a/scripts/export-promptfoo-config.test.ts b/scripts/export-promptfoo-config.test.ts index 7079492b1..78b9d5fbb 100644 --- a/scripts/export-promptfoo-config.test.ts +++ b/scripts/export-promptfoo-config.test.ts @@ -81,24 +81,65 @@ describe('exportPromptfooConfig', () => { expect(exported).not.toHaveProperty('evaluate_options'); }); - it('fails clearly on unsupported top-level environment semantics', () => { + it('lowers host environment setup to a generated Promptfoo extension and workdir metadata', () => { + const output = outputPath('promptfooconfig.yaml'); + exportPromptfooConfig({ + inputPath: path.join(FIXTURE_DIR, 'host-environment.agentv.yaml'), + outputPath: output, + }); + + const workdir = path.resolve(FIXTURE_DIR, 'workspaces', 'provider-export'); + const exported = parseYamlFile(output); + const providers = exported.providers as Array>; + const providerConfig = providers[0]?.config as Record; + const defaultTest = exported.defaultTest as Record>; + const metadata = exported.metadata as Record; + const extensionPath = path.join( + path.dirname(output), + '.agentv', + 'generated', + 'promptfoo', + 'extensions', + 'host-environment.ts', + ); + + expect(exported).not.toHaveProperty('environment'); + expect(exported.extensions).toContain( + 'file://.agentv/generated/promptfoo/extensions/host-environment.ts:beforeAll', + ); + expect(defaultTest.vars).toMatchObject({ + locale: 'en-US', + agentv_environment_workdir: workdir, + }); + expect(defaultTest.metadata).toMatchObject({ + agentv_environment: { type: 'host', workdir }, + }); + expect(metadata.agentv_environment).toEqual({ type: 'host', workdir }); + expect(providerConfig.agentv_environment_workdir).toBe(workdir); + expect(providerConfig.agentv_environment).toEqual({ type: 'host', workdir }); + expect(existsSync(extensionPath)).toBe(true); + expect(readFileSync(extensionPath, 'utf8')).toContain('AGENTV_ENVIRONMENT_WORKDIR'); + }); + + it('fails clearly on unsupported Docker environment export', () => { const output = outputPath('promptfooconfig.yaml'); expect(() => exportPromptfooConfig({ - inputPath: path.join(FIXTURE_DIR, 'environment-unsupported.agentv.yaml'), + inputPath: path.join(FIXTURE_DIR, 'docker-environment-unsupported.agentv.yaml'), outputPath: output, }), ).toThrow(PromptfooExportDiagnostic); try { exportPromptfooConfig({ - inputPath: path.join(FIXTURE_DIR, 'environment-unsupported.agentv.yaml'), + inputPath: path.join(FIXTURE_DIR, 'docker-environment-unsupported.agentv.yaml'), outputPath: output, }); } catch (error) { expect(error).toBeInstanceOf(PromptfooExportDiagnostic); - expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_environment'); - expect((error as Error).message).toContain("Top-level 'environment' is AgentV-only"); + expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_docker_environment'); + expect((error as Error).message).toContain('Docker environment export is not supported'); + expect((error as Error).message).toContain('isolation, image/context, mounts, services'); } }); }); diff --git a/scripts/export-promptfoo-config.ts b/scripts/export-promptfoo-config.ts index 69d6b4e22..27b2a3bee 100644 --- a/scripts/export-promptfoo-config.ts +++ b/scripts/export-promptfoo-config.ts @@ -5,6 +5,17 @@ import YAML from 'yaml'; type JsonMap = Record; +interface HostEnvironmentExport { + readonly type: 'host'; + readonly workdir: string; + readonly env?: Record; + readonly setup?: { + readonly command: readonly string[]; + readonly cwd?: string; + readonly timeoutMs?: number; + }; +} + export class PromptfooExportDiagnostic extends Error { constructor( readonly code: string, @@ -25,6 +36,8 @@ export interface ExportPromptfooConfigOptions { // valid for some Promptfoo file-backed hooks/assertions, but not provider ids. const GENERATED_CODEX_CLI_PROVIDER_REF = 'file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts'; +const GENERATED_HOST_ENVIRONMENT_EXTENSION_REF = + 'file://.agentv/generated/promptfoo/extensions/host-environment.ts:beforeAll'; const AGENTV_PROVIDER_LOWERINGS: Record = { 'agentv:codex-cli': GENERATED_CODEX_CLI_PROVIDER_REF, @@ -40,6 +53,16 @@ const EVALUATE_OPTIONS_KEY_RENAMES: Record = { }; const AGENTV_ONLY_PROVIDER_KEYS = new Set(['runtime']); +const SUPPORTED_HOST_ENVIRONMENT_KEYS = new Set(['type', 'workdir', 'setup', 'env']); +const DOCKER_ENVIRONMENT_KEYS = new Set([ + 'context', + 'dockerfile', + 'image', + 'mounts', + 'resources', + 'secrets', + 'services', +]); function assertRecord(value: unknown, label: string): JsonMap { if (!value || typeof value !== 'object' || Array.isArray(value)) { @@ -51,6 +74,179 @@ function assertRecord(value: unknown, label: string): JsonMap { return value as JsonMap; } +function assertString(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new PromptfooExportDiagnostic('invalid_environment', `${label} must be a string.`); + } + return value.trim(); +} + +function assertStringRecord(value: unknown, label: string): Record | undefined { + if (value === undefined) { + return undefined; + } + const record = assertRecord(value, label); + const output: Record = {}; + for (const [key, entry] of Object.entries(record)) { + output[key] = assertString(entry, `${label}.${key}`); + } + return output; +} + +function assertSetup(value: unknown): HostEnvironmentExport['setup'] | undefined { + if (value === undefined) { + return undefined; + } + const setup = assertRecord(value, 'environment.setup'); + for (const key of Object.keys(setup)) { + if (key !== 'command' && key !== 'cwd' && key !== 'timeout_ms' && key !== 'timeoutMs') { + throw new PromptfooExportDiagnostic( + 'unsupported_environment', + `Unsupported host environment setup field '${key}'. Export supports setup.command, setup.cwd, and setup.timeout_ms.`, + ); + } + } + const command = setup.command; + if ( + !Array.isArray(command) || + command.length === 0 || + !command.every((entry) => typeof entry === 'string' && entry.trim().length > 0) + ) { + throw new PromptfooExportDiagnostic( + 'invalid_environment', + 'environment.setup.command must be a non-empty string array.', + ); + } + const timeoutValue = setup.timeout_ms ?? setup.timeoutMs; + if (timeoutValue !== undefined && (typeof timeoutValue !== 'number' || timeoutValue <= 0)) { + throw new PromptfooExportDiagnostic( + 'invalid_environment', + 'environment.setup.timeout_ms must be a positive number of milliseconds.', + ); + } + return { + command: command as string[], + ...(setup.cwd !== undefined && { cwd: assertString(setup.cwd, 'environment.setup.cwd') }), + ...(typeof timeoutValue === 'number' && { timeoutMs: timeoutValue }), + }; +} + +function resolveFileReference(reference: string, baseDir: string): string { + const filePath = reference.slice('file://'.length); + return path.isAbsolute(filePath) ? filePath : path.resolve(baseDir, filePath); +} + +function readEnvironmentInput(value: unknown, inputDir: string): unknown { + if (typeof value !== 'string') { + return value; + } + if (!value.startsWith('file://')) { + throw new PromptfooExportDiagnostic( + 'unsupported_environment_ref', + 'environment must be an inline host recipe or a file:// reference.', + ); + } + const environmentPath = resolveFileReference(value, inputDir); + const parsed = YAML.parse(readFileSync(environmentPath, 'utf8')); + return parsed; +} + +function environmentDiagnosticForDocker(): PromptfooExportDiagnostic { + return new PromptfooExportDiagnostic( + 'unsupported_docker_environment', + 'Docker environment export is not supported yet. Export refuses to silently degrade Docker isolation, image/context, mounts, services, resources, secrets, or provenance.', + ); +} + +function parseHostEnvironment(value: unknown, inputDir: string): HostEnvironmentExport { + const environment = assertRecord(readEnvironmentInput(value, inputDir), 'environment'); + const type = environment.type ?? 'host'; + if ( + type === 'docker' || + Object.keys(environment).some((key) => DOCKER_ENVIRONMENT_KEYS.has(key)) + ) { + throw environmentDiagnosticForDocker(); + } + if (type !== 'host') { + throw new PromptfooExportDiagnostic( + 'unsupported_environment', + 'Only environment.type: host can be exported to Promptfoo in the initial exporter.', + ); + } + for (const key of Object.keys(environment)) { + if (!SUPPORTED_HOST_ENVIRONMENT_KEYS.has(key)) { + throw new PromptfooExportDiagnostic( + 'unsupported_environment', + `Unsupported host environment field '${key}'. Export supports workdir, setup, and env.`, + ); + } + } + + const workdir = assertString(environment.workdir, 'environment.workdir'); + const env = assertStringRecord(environment.env, 'environment.env'); + const setup = assertSetup(environment.setup); + return { + type: 'host', + workdir: path.isAbsolute(workdir) ? workdir : path.resolve(inputDir, workdir), + ...(env !== undefined && { env }), + ...(setup !== undefined && { setup }), + }; +} + +function agentvEnvironmentMetadata(environment: HostEnvironmentExport): JsonMap { + return { + type: environment.type, + workdir: environment.workdir, + }; +} + +function mergeJsonObject(value: unknown, additions: JsonMap): JsonMap { + const base = + value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonMap) : {}; + return { + ...base, + ...additions, + }; +} + +function addEnvironmentToProvider(provider: unknown, environment: HostEnvironmentExport): unknown { + const environmentMetadata = agentvEnvironmentMetadata(environment); + const configAdditions = { + agentv_environment_workdir: environment.workdir, + agentv_environment: environmentMetadata, + }; + + if (typeof provider === 'string') { + return { + id: lowerProviderId(provider), + config: configAdditions, + }; + } + + if (provider && typeof provider === 'object' && !Array.isArray(provider)) { + const providerObject = provider as JsonMap; + if (looksLikeProviderOptionsMap(providerObject)) { + const [[id, options]] = Object.entries(providerObject); + const optionObject = + options && typeof options === 'object' && !Array.isArray(options) + ? (options as JsonMap) + : {}; + return { + [lowerProviderId(id)]: { + ...optionObject, + config: mergeJsonObject(optionObject.config, configAdditions), + }, + }; + } + return { + ...providerObject, + config: mergeJsonObject(providerObject.config, configAdditions), + }; + } + + return provider; +} + function lowerProviderId(id: string): string { if (!id.startsWith('agentv:')) { return id; @@ -83,7 +279,11 @@ function looksLikeProviderOptionsMap(provider: JsonMap): boolean { return !('id' in provider) && Object.keys(provider).length === 1; } -function lowerProviderEntry(provider: unknown): unknown { +function lowerProviderEntry(provider: unknown, environment?: HostEnvironmentExport): unknown { + if (environment) { + return lowerProviderEntry(addEnvironmentToProvider(provider, environment)); + } + if (typeof provider === 'string') { return lowerProviderId(provider); } @@ -100,15 +300,15 @@ function lowerProviderEntry(provider: unknown): unknown { return provider; } -function lowerProviders(providers: unknown): unknown { +function lowerProviders(providers: unknown, environment?: HostEnvironmentExport): unknown { if (typeof providers === 'string') { - return lowerProviderId(providers); + return lowerProviderEntry(providers, environment); } if (Array.isArray(providers)) { - return providers.map(lowerProviderEntry); + return providers.map((provider) => lowerProviderEntry(provider, environment)); } if (providers && typeof providers === 'object') { - return lowerProviderEntry(providers); + return lowerProviderEntry(providers, environment); } return providers; } @@ -125,19 +325,44 @@ function renameObjectKeys(value: unknown, renames: Record): unkn return output; } -function promptfooConfigFromAgentVConfig(config: JsonMap): JsonMap { - if ('environment' in config) { +function defaultTestWithEnvironment(value: unknown, environment: HostEnvironmentExport): JsonMap { + if (typeof value === 'string') { throw new PromptfooExportDiagnostic( - 'unsupported_environment', - "Top-level 'environment' is AgentV-only and cannot be exported to Promptfoo yet. Remove it for direct Promptfoo validation, or keep running this eval through AgentV.", + 'unsupported_environment_default_test_ref', + 'Host environment export cannot inject workdir vars into a default_test file reference yet. Inline default_test before exporting to Promptfoo.', ); } + const defaultTest = + value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonMap) : {}; + return { + ...defaultTest, + vars: mergeJsonObject(defaultTest.vars, { + agentv_environment_workdir: environment.workdir, + }), + metadata: mergeJsonObject(defaultTest.metadata, { + agentv_environment: agentvEnvironmentMetadata(environment), + }), + }; +} +function promptfooConfigFromAgentVConfig( + config: JsonMap, + environment?: HostEnvironmentExport, +): JsonMap { const promptfooConfig: JsonMap = {}; for (const [key, value] of Object.entries(config)) { + if (key === 'environment') { + continue; + } const outputKey = TOP_LEVEL_KEY_RENAMES[key] ?? key; if (key === 'providers') { - promptfooConfig[outputKey] = lowerProviders(value); + promptfooConfig[outputKey] = lowerProviders(value, environment); + continue; + } + if (key === 'default_test') { + promptfooConfig[outputKey] = environment + ? defaultTestWithEnvironment(value, environment) + : value; continue; } if (key === 'evaluate_options') { @@ -146,6 +371,20 @@ function promptfooConfigFromAgentVConfig(config: JsonMap): JsonMap { } promptfooConfig[outputKey] = value; } + if (environment && !('defaultTest' in promptfooConfig)) { + promptfooConfig.defaultTest = defaultTestWithEnvironment(undefined, environment); + } + if (environment) { + promptfooConfig.metadata = mergeJsonObject(promptfooConfig.metadata, { + agentv_environment: agentvEnvironmentMetadata(environment), + }); + promptfooConfig.extensions = [ + ...((Array.isArray(promptfooConfig.extensions) + ? promptfooConfig.extensions + : []) as unknown[]), + GENERATED_HOST_ENVIRONMENT_EXTENSION_REF, + ]; + } return promptfooConfig; } @@ -196,11 +435,98 @@ function writeGeneratedProviderFiles(outputPath: string, config: JsonMap): void ); } +function setupCwdForExport( + environment: HostEnvironmentExport, + inputDir: string, +): string | undefined { + const cwd = environment.setup?.cwd; + if (!environment.setup) { + return undefined; + } + if (!cwd) { + return inputDir; + } + return path.isAbsolute(cwd) ? cwd : path.resolve(environment.workdir, cwd); +} + +function writeGeneratedHostEnvironmentExtension( + outputPath: string, + environment: HostEnvironmentExport | undefined, + inputDir: string, +): void { + if (!environment) { + return; + } + + const extensionPath = path.join( + path.dirname(outputPath), + '.agentv', + 'generated', + 'promptfoo', + 'extensions', + 'host-environment.ts', + ); + mkdirSync(path.dirname(extensionPath), { recursive: true }); + const setup = + environment.setup === undefined + ? null + : { + command: environment.setup.command, + cwd: setupCwdForExport(environment, inputDir), + ...(environment.setup.timeoutMs !== undefined && { + timeoutMs: environment.setup.timeoutMs, + }), + }; + writeFileSync( + extensionPath, + [ + "import { spawnSync } from 'node:child_process';", + "import { mkdirSync } from 'node:fs';", + '', + `const environment = ${JSON.stringify(agentvEnvironmentMetadata(environment), null, 2)};`, + `const setup = ${JSON.stringify(setup, null, 2)};`, + `const environmentEnv = ${JSON.stringify(environment.env ?? {}, null, 2)};`, + '', + 'export function beforeAll(context) {', + ' mkdirSync(environment.workdir, { recursive: true });', + ' if (setup) {', + ' const result = spawnSync(setup.command[0], setup.command.slice(1), {', + ' cwd: setup.cwd,', + ' env: {', + ' ...process.env,', + ' ...environmentEnv,', + ' AGENTV_ENVIRONMENT_WORKDIR: environment.workdir,', + ' },', + ' input: JSON.stringify({ environment }, null, 2),', + " encoding: 'utf8',", + ' timeout: setup.timeoutMs,', + ' });', + ' if (result.error) {', + ' throw result.error;', + ' }', + ' if (result.status && result.status !== 0) {', + ' throw new Error(`AgentV host environment setup failed with exit code ${result.status}: ${result.stderr || result.stdout || ""}`);', + ' }', + ' }', + ' return context;', + '}', + '', + ].join('\n'), + 'utf8', + ); +} + export function exportPromptfooConfig(options: ExportPromptfooConfigOptions): JsonMap { const input = readFileSync(options.inputPath, 'utf8'); const parsed = assertRecord(YAML.parse(input), options.inputPath); - const promptfooConfig = promptfooConfigFromAgentVConfig(parsed); + const inputDir = path.dirname(options.inputPath); + const environment = + parsed.environment !== undefined + ? parseHostEnvironment(parsed.environment, inputDir) + : undefined; + const promptfooConfig = promptfooConfigFromAgentVConfig(parsed, environment); mkdirSync(path.dirname(options.outputPath), { recursive: true }); + writeGeneratedHostEnvironmentExtension(options.outputPath, environment, inputDir); writeFileSync(options.outputPath, YAML.stringify(promptfooConfig), 'utf8'); writeGeneratedProviderFiles(options.outputPath, promptfooConfig); return promptfooConfig; diff --git a/scripts/fixtures/promptfoo-export/docker-environment-unsupported.agentv.yaml b/scripts/fixtures/promptfoo-export/docker-environment-unsupported.agentv.yaml new file mode 100644 index 000000000..129e7fd39 --- /dev/null +++ b/scripts/fixtures/promptfoo-export/docker-environment-unsupported.agentv.yaml @@ -0,0 +1,25 @@ +description: Unsupported Docker environment export fixture +environment: + type: docker + image: node:20 + workdir: /workspace + mounts: + - source: ./fixtures + target: /workspace/fixtures + resources: + cpus: 2 + services: + postgres: + image: postgres:16 + +providers: + - id: openai:responses:gpt-5.4 + label: responses-direct + +prompts: + - "{{ question }}" + +tests: + - id: capital + vars: + question: What is the capital of France? diff --git a/scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml b/scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml deleted file mode 100644 index 0f3605401..000000000 --- a/scripts/fixtures/promptfoo-export/environment-unsupported.agentv.yaml +++ /dev/null @@ -1,18 +0,0 @@ -description: Unsupported AgentV environment export fixture -environment: - runtime: host - workdir: . - setup: - command: ["bash", "-lc", "echo preparing workspace"] - -providers: - - id: openai:responses:gpt-5.4 - label: responses-direct - -prompts: - - "{{ question }}" - -tests: - - id: capital - vars: - question: What is the capital of France? diff --git a/scripts/fixtures/promptfoo-export/host-environment.agentv.yaml b/scripts/fixtures/promptfoo-export/host-environment.agentv.yaml new file mode 100644 index 000000000..9cb61dcb3 --- /dev/null +++ b/scripts/fixtures/promptfoo-export/host-environment.agentv.yaml @@ -0,0 +1,34 @@ +description: Promptfoo host environment export validation +environment: + type: host + workdir: ./workspaces/provider-export + setup: + command: ["bash", "-lc", "echo ready > \"$AGENTV_ENVIRONMENT_WORKDIR/ready.txt\""] + timeout_ms: 120000 + env: + AGENTV_EXPORT_TEST: fixture + +providers: + - id: agentv:codex-cli + label: codex-cli-host-exported + runtime: host + config: + model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}" + +prompts: + - "Answer briefly: {{ question }}" + +default_test: + vars: + locale: en-US + assert: + - type: contains + value: Paris + +evaluate_options: + max_concurrency: 1 + +tests: + - id: capital + vars: + question: What is the capital of France?