feat(promptfoo): validate exported provider configs

.github/workflows/ci.yml

@@ -169,6 +169,34 @@ jobs:
       - name: Run tests
         run: bun run test
 
+  promptfoo-export:
+    name: Promptfoo Export
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version-file: package.json
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Run Promptfoo export tests
+        run: bun run validate:promptfoo-export
+
+      - name: Export Promptfoo validation fixture
+        run: |
+          set -euo pipefail
+          mkdir -p "$RUNNER_TEMP/promptfoo-export"
+          bun scripts/export-promptfoo-config.ts \
+            --input scripts/fixtures/promptfoo-export/host-environment.agentv.yaml \
+            --output "$RUNNER_TEMP/promptfoo-export/promptfooconfig.yaml"
+
+      - name: Validate exported config with Promptfoo
+        run: bunx promptfoo@0.121.15 validate config -c "$RUNNER_TEMP/promptfoo-export/promptfooconfig.yaml"
+
   links:
     name: Check Links
     runs-on: ubuntu-latest

apps/web/src/content/docs/docs/next/reference/promptfoo-parity.mdx

@@ -6,25 +6,35 @@ sidebar:
 slug: docs/reference/promptfoo-parity
 ---
 
-AgentV uses a similar eval config contract to Promptfoo for ordinary authored
-evals: prompt matrices, test rows, vars, default test data, assertions, and
-provider matrices all use the same broad shape. AgentV is a
-Promptfoo-compatible superset at the provider declaration layer, not a promise
-that every AgentV config executes unchanged in Promptfoo. AgentV keeps the wire
-format `snake_case`, keeps provider selection identity explicit with `label`,
-and adds repo-native environment recipes and artifact fields for agent
-evaluation.
+AgentV authored YAML should be Promptfoo-compatible by default for ordinary eval
+surfaces: prompt matrices, test rows, vars, default test data, assertions, and
+provider matrices all use the same broad shape. The intended authored-config
+differences are `environment`, AgentV refs, and built-in AgentV providers. Export
+fills those gaps for supported AgentV-native pieces so Promptfoo can validate or
+load the resulting config without learning AgentV semantics.
 
 Promptfoo-shaped `providers` entries can be strings such as
 `openai:gpt-4.1-mini`, complete package provider strings such as
 `package:@agentv/promptfoo-providers:CodexCliProvider` or
 `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option
 objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`,
 and `inputs`, or provider maps such as
-`{ "openai:gpt-4": { label, config } }`. AgentV-only fields such as top-level
-`environment` and provider-local runtime/testbed overlays are AgentV semantics;
-Promptfoo may ignore or strip unknown keys and will not execute AgentV
-environment setup without a transpiler or wrapper.
+`{ "openai:gpt-4": { label, config } }`. Promptfoo-native provider IDs such as
+`openai:responses:gpt-5.4`, `openai:codex-sdk`, and
+`openai:codex-app-server` remain directly Promptfoo-readable.
+
+For AgentV-only provider IDs such as `agentv:codex-cli`, use the Promptfoo
+export path instead of running the AgentV YAML directly with Promptfoo. The
+exporter lowers supported AgentV-only IDs to Promptfoo-readable provider
+references such as
+`file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts`
+and exports supported host/filesystem `environment` recipes to a generated
+Promptfoo extension such as
+`file://.agentv/generated/promptfoo/extensions/host-environment.ts:beforeAll`.
+The exporter passes the resolved isolated workdir through provider `config`,
+default-test vars, and metadata. Docker `environment` export is intentionally
+unsupported initially and fails with a diagnostic instead of degrading
+isolation, image/context, mounts, services, resources, secrets, or provenance.
 
 Use this matrix when translating a Promptfoo-style normal eval into AgentV YAML.
 It documents which surfaces align directly, which AgentV surfaces are cleaner
@@ -56,7 +66,7 @@ implements equivalent semantics directly.
 | Evaluate options | `evaluateOptions` for runtime controls. | `evaluate_options` for runtime controls. | Align with Promptfoo | AgentV uses `evaluate_options.repeat`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency`. |
 | Authored concurrency | Common Promptfoo usage includes runtime options such as `maxConcurrency`. | `evaluate_options.max_concurrency`. | Keep AgentV divergence | Do not author `execution.max_concurrency` or top-level `workers` in eval YAML. CLI `--workers` remains an operator override. |
 | Provider selection | Promptfoo normal evals use `providers`; `targets` can alias providers in unified config. | Use top-level `providers` for one or more systems under test. | Align with Promptfoo | Old AgentV `target`/`targets` authoring is hard-rejected. Use CLI `--provider`/`--providers` for runtime selection. |
-| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. | Align with Promptfoo | Package provider strings must include the exported class/function segment after the final colon. AgentV-only fields such as `runtime` and provider-local testbed/runtime overlays are AgentV semantics. Promptfoo may ignore or strip unknown keys and will not execute AgentV environment setup without a transpiler or wrapper. |
+| Provider declarations | Provider entries can be strings, complete package provider strings such as `package:@agentv/promptfoo-providers:CodexCliProvider` or `package:@agentv/promptfoo-providers/codex-cli:Provider`, provider option objects with `id`, `label`, `config`, `env`, `prompts`, `transform`, `delay`, and `inputs`, or provider maps such as `{ "openai:gpt-4": { label, config } }`. | AgentV accepts the same provider declaration layer; `id` is the backend/spec and `label` is the stable AgentV selection and result identity. | Align with Promptfoo | Package provider strings must include the exported class/function segment after the final colon. Promptfoo-native provider IDs remain directly Promptfoo-readable. Built-in AgentV IDs such as `agentv:codex-cli` need export, which lowers them to a generated file provider for validation. |
 | Provider-prompt mapping | `providerPromptMap` maps provider identities to prompt subsets. | Rejected. Use explicit AgentV composition: separate eval suites/files for provider-specific prompt subsets, top-level `prompts` plus `tests`/`default_test.vars`, `providers` or CLI `--provider`, and tags/run metadata for grouping. | Removed/rejected surface | Do not author `providerPromptMap` or `provider_prompt_map`. |
 | Direct authored input | Promptfoo prompt authoring normally goes through `prompts` plus vars. | Top-level `input` and inline `tests[].input` are removed from normal authored eval YAML. External raw-case imports may still carry internal input rows for compatibility. | Removed AgentV extension | Author prompt text, chat/system/user messages, and file-backed prompt content as `prompts`; put row data in `tests[].vars` and shared defaults in `default_test.vars`. |
 | Authored preprocessors | Not Promptfoo's canonical output-shaping surface. | Rejected in current authored YAML. | Removed/rejected surface | Use `transform` at `default_test.options`, `tests[].options`, or the assertion that needs the shaped output. Historical versioned docs may still show old preprocessor examples. |
@@ -68,7 +78,7 @@ implements equivalent semantics directly.
 | Skill assertions | Promptfoo includes `skill-used` for checking whether an agent invoked a named skill. | `type: skill-used` and `type: not-skill-used` with `value: <skill>` or a matcher object. | Align with Promptfoo | AgentV evaluates these against normalized tool-call and skill-use trace data. |
 | Trajectory assertions | Promptfoo includes `trajectory:tool-used`, `trajectory:tool-sequence`, `trajectory:tool-args-match`, `trajectory:step-count`, and `trajectory:goal-success`. | AgentV accepts the same assertion names over AgentV-normalized traces. | Align with Promptfoo | Use these for tool presence, order, argument checks, step budgets, and LLM-judged goal success. |
 | Other Promptfoo trace assertions | Promptfoo also includes `tool-call-f1`, `trace-span-count`, `trace-span-duration`, and `trace-error-spans`. | Not accepted yet. | Defer/future-scope | Use `script` assertions or `execution-metrics` when current AgentV primitives cover the requirement. |
-| Coding-agent testbeds | Promptfoo normal evals do not define a typed coding-agent testbed primitive. | `environment`, usually inline or `environment: file://...`, plus distinct top-level `env` and lifecycle `extensions`. | Keep AgentV extension | `environment` is an AgentV extension informed by Margin local-agent ergonomics and Harbor/Terminal-Bench Docker substrate evidence. Use it for host/Docker workdir, setup argv, fixtures, services, and repo materialization. Use top-level `env` for provider/eval variables and `extensions` for lifecycle hooks. |
+| Coding-agent testbeds | Promptfoo normal evals do not define a typed coding-agent testbed primitive. | `environment`, usually inline or `environment: file://...`, plus distinct top-level `env` and lifecycle `extensions`. | Keep AgentV extension | `environment` is an AgentV extension informed by Margin local-agent ergonomics and Harbor/Terminal-Bench Docker substrate evidence. Export supports the host/filesystem subset by generating Promptfoo setup extensions and passing the resolved workdir through config, vars, and metadata. Docker export is not supported yet and fails clearly instead of degrading isolation or provenance. |
 | Run artifacts and inspection | Promptfoo owns its own result viewer and output formats. | AgentV writes `.agentv/results/<run_id>/` bundles with `summary.json`, `.internal/index.jsonl`, sidecars, and local Dashboard support. | Keep AgentV extension | AgentV-owned bundles are the source of truth for compare, Dashboard, CI, and adapters. Phoenix is link-out correlation only through safe external trace metadata. |
 | Grading result artifacts | Promptfoo `GradingResult` uses aggregate `pass`, `score`, `reason`, recursive `componentResults`, optional `assertion`, optional `namedScores`, and optional `metadata`. | AgentV persists the same concepts as `pass`, `score`, `reason`, recursive `component_results`, optional `assertion`, optional `named_scores`, and optional `metadata`. | Align with Promptfoo | AgentV keeps split filesystem run bundles and `snake_case` persisted fields. Promptfoo import/export adapters may translate `componentResults`/`namedScores` to and from AgentV `component_results`/`named_scores`. |
 | Compare command | Promptfoo has its own result comparison surfaces. | `agentv results compare <baseline-index.jsonl> <candidate-index.jsonl>`. | Keep AgentV extension | Compare consumes completed AgentV run indexes such as `.agentv/results/<run_id>/.internal/index.jsonl`. |

package.json

@@ -20,6 +20,7 @@
     "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js",
     "beads:check": "bun scripts/check-beads-context.ts",
     "debug:pi-sdk-tools": "bun scripts/debug-pi-sdk-tools.ts",
+    "validate:promptfoo-export": "bun test scripts/export-promptfoo-config.test.ts",
     "validate:examples": "EVAL_CRITERIA=placeholder CUSTOM_SYSTEM_PROMPT=placeholder bun scripts/validate-example-evals.ts",
     "eval:baseline-check": "bun scripts/check-eval-baselines.ts",
     "release": "bun scripts/release.ts",

scripts/export-promptfoo-config.test.ts

@@ -0,0 +1,145 @@
+import { describe, expect, it } from 'bun:test';
+import { existsSync, mkdtempSync, readFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import YAML from 'yaml';
+import { PromptfooExportDiagnostic, exportPromptfooConfig } from './export-promptfoo-config';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const FIXTURE_DIR = path.join(ROOT, 'scripts', 'fixtures', 'promptfoo-export');
+
+function outputPath(name: string): string {
+  return path.join(mkdtempSync(path.join(tmpdir(), 'agentv-promptfoo-export-')), name);
+}
+
+function parseYamlFile(filePath: string): Record<string, unknown> {
+  return YAML.parse(readFileSync(filePath, 'utf8')) as Record<string, unknown>;
+}
+
+describe('exportPromptfooConfig', () => {
+  it('preserves Promptfoo-native colon provider ids and labels', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    exportPromptfooConfig({
+      inputPath: path.join(FIXTURE_DIR, 'provider-surface.agentv.yaml'),
+      outputPath: output,
+    });
+
+    const exported = parseYamlFile(output);
+    const providers = exported.providers as Array<Record<string, unknown>>;
+
+    expect(providers[0]).toMatchObject({
+      id: 'openai:responses:gpt-5.4',
+      label: 'responses-direct',
+    });
+    expect(providers[1]).toMatchObject({
+      id: 'openai:codex-sdk',
+      label: 'codex-sdk-direct',
+    });
+  });
+
+  it('lowers agentv:codex-cli to a generated Promptfoo file provider and preserves label', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    exportPromptfooConfig({
+      inputPath: path.join(FIXTURE_DIR, 'provider-surface.agentv.yaml'),
+      outputPath: output,
+    });
+
+    const exported = parseYamlFile(output);
+    const providers = exported.providers as Array<Record<string, unknown>>;
+
+    expect(providers[2]).toMatchObject({
+      id: 'file://.agentv/generated/promptfoo/providers/codex-cli-provider.ts',
+      label: 'codex-cli-exported',
+    });
+    expect(providers[2]).not.toHaveProperty('runtime');
+    expect(
+      existsSync(
+        path.join(
+          path.dirname(output),
+          '.agentv',
+          'generated',
+          'promptfoo',
+          'providers',
+          'codex-cli-provider.ts',
+        ),
+      ),
+    ).toBe(true);
+  });
+
+  it('translates AgentV snake_case config keys for Promptfoo validation', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    exportPromptfooConfig({
+      inputPath: path.join(FIXTURE_DIR, 'provider-surface.agentv.yaml'),
+      outputPath: output,
+    });
+
+    const exported = parseYamlFile(output);
+    expect(exported).toHaveProperty('defaultTest');
+    expect(exported).toHaveProperty('evaluateOptions');
+    expect(exported.evaluateOptions).toEqual({ maxConcurrency: 1 });
+    expect(exported).not.toHaveProperty('default_test');
+    expect(exported).not.toHaveProperty('evaluate_options');
+  });
+
+  it('lowers host environment setup to a generated Promptfoo extension and workdir metadata', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    exportPromptfooConfig({
+      inputPath: path.join(FIXTURE_DIR, 'host-environment.agentv.yaml'),
+      outputPath: output,
+    });
+
+    const workdir = path.resolve(FIXTURE_DIR, 'workspaces', 'provider-export');
+    const exported = parseYamlFile(output);
+    const providers = exported.providers as Array<Record<string, unknown>>;
+    const providerConfig = providers[0]?.config as Record<string, unknown>;
+    const defaultTest = exported.defaultTest as Record<string, Record<string, unknown>>;
+    const metadata = exported.metadata as Record<string, unknown>;
+    const extensionPath = path.join(
+      path.dirname(output),
+      '.agentv',
+      'generated',
+      'promptfoo',
+      'extensions',
+      'host-environment.ts',
+    );
+
+    expect(exported).not.toHaveProperty('environment');
+    expect(exported.extensions).toContain(
+      'file://.agentv/generated/promptfoo/extensions/host-environment.ts:beforeAll',
+    );
+    expect(defaultTest.vars).toMatchObject({
+      locale: 'en-US',
+      agentv_environment_workdir: workdir,
+    });
+    expect(defaultTest.metadata).toMatchObject({
+      agentv_environment: { type: 'host', workdir },
+    });
+    expect(metadata.agentv_environment).toEqual({ type: 'host', workdir });
+    expect(providerConfig.agentv_environment_workdir).toBe(workdir);
+    expect(providerConfig.agentv_environment).toEqual({ type: 'host', workdir });
+    expect(existsSync(extensionPath)).toBe(true);
+    expect(readFileSync(extensionPath, 'utf8')).toContain('AGENTV_ENVIRONMENT_WORKDIR');
+  });
+
+  it('fails clearly on unsupported Docker environment export', () => {
+    const output = outputPath('promptfooconfig.yaml');
+    expect(() =>
+      exportPromptfooConfig({
+        inputPath: path.join(FIXTURE_DIR, 'docker-environment-unsupported.agentv.yaml'),
+        outputPath: output,
+      }),
+    ).toThrow(PromptfooExportDiagnostic);
+
+    try {
+      exportPromptfooConfig({
+        inputPath: path.join(FIXTURE_DIR, 'docker-environment-unsupported.agentv.yaml'),
+        outputPath: output,
+      });
+    } catch (error) {
+      expect(error).toBeInstanceOf(PromptfooExportDiagnostic);
+      expect((error as PromptfooExportDiagnostic).code).toBe('unsupported_docker_environment');
+      expect((error as Error).message).toContain('Docker environment export is not supported');
+      expect((error as Error).message).toContain('isolation, image/context, mounts, services');
+    }
+  });
+});