From 8dd0a9275b0b4c9c3040e4de890d38f04d526535 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 15:44:02 +0200 Subject: [PATCH 01/10] feat(config): add composable AgentV config graph --- apps/cli/src/commands/eval/run-eval.ts | 2 +- .../docs/docs/next/evaluation/eval-files.mdx | 5 +- .../docs/docs/next/evaluation/experiments.mdx | 6 +- .../docs/next/evaluation/running-evals.mdx | 84 +++- .../content/docs/docs/next/evaluation/sdk.mdx | 2 +- .../docs/docs/next/targets/configuration.mdx | 115 +++--- .../docs/docs/next/tools/dashboard.mdx | 11 - .../readme-quickstart/.agentv/config.yaml | 6 + .../readme-quickstart/.agentv/defaults.yaml | 2 + .../readme-quickstart/.agentv/graders.yaml | 7 + .../readme-quickstart/.agentv/targets.yaml | 8 + .../readme-quickstart/.agentv/tests.yaml | 2 + examples/features/readme-quickstart/README.md | 16 + packages/core/src/evaluation/config.ts | 30 +- packages/core/src/evaluation/experiment.ts | 2 +- .../src/evaluation/loaders/config-graph.ts | 370 ++++++++++++++++++ .../src/evaluation/loaders/config-loader.ts | 116 +++++- .../evaluation/validation/config-validator.ts | 60 ++- .../evaluation/validation/eval-file.schema.ts | 44 ++- .../evaluation/validation/eval-validator.ts | 60 ++- .../core/src/evaluation/workspace/setup.ts | 2 +- packages/core/src/evaluation/yaml-parser.ts | 17 +- packages/core/src/projects.ts | 28 +- packages/core/test/evaluation/config.test.ts | 15 +- .../evaluation/eval-inline-experiment.test.ts | 4 +- .../evaluation/loaders/config-loader.test.ts | 269 ++++++++++++- .../validation/config-validator.test.ts | 123 +++++- .../validation/eval-file-schema.test.ts | 40 +- .../validation/eval-validator.test.ts | 68 +++- packages/core/test/projects.test.ts | 39 ++ .../references/eval.schema.json | 70 +++- 31 files changed, 1486 insertions(+), 137 deletions(-) create mode 100644 examples/features/readme-quickstart/.agentv/config.yaml create mode 100644 examples/features/readme-quickstart/.agentv/defaults.yaml create mode 100644 examples/features/readme-quickstart/.agentv/graders.yaml create mode 100644 examples/features/readme-quickstart/.agentv/targets.yaml create mode 100644 examples/features/readme-quickstart/.agentv/tests.yaml create mode 100644 packages/core/src/evaluation/loaders/config-graph.ts diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 9c25d8dbf..959a59a83 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -636,7 +636,7 @@ function normalizeOptions( yamlExecution?: ExecutionDefaults, ): NormalizedOptions { const cliWorkers = normalizeOptionalNumber(rawOptions.workers); - const configWorkers = config?.execution?.workers ?? yamlExecution?.workers; + const configWorkers = config?.execution?.maxConcurrency ?? yamlExecution?.max_concurrency; const workers = cliWorkers ?? configWorkers ?? 0; const cliOutputDir = normalizeString(rawOptions.output); diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index c3d4e4eec..05fd933c1 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -8,7 +8,10 @@ sidebar: Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. The reserved `tags.experiment` key is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated attempts and gates. Workspace lifetime belongs under `workspace.scope`; repository provenance belongs under `workspace.repos`; Docker/container binding belongs under `workspace.docker`. Non-provisioning setup commands belong in top-level `extensions`; reset policy stays under `workspace.hooks.after_each.reset`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. -Eval files describe the task, target binding, and run controls. Use `evaluate_options.max_concurrency` for authored suite concurrency. Operators can still override concurrency with `--workers` or set defaults with `execution.workers` in `agentv.config.*` / `.agentv/config.yaml`; do not author legacy `workers` fields in eval YAML. +Eval files describe the task, target binding, and run controls. Use +`execution.max_concurrency` or `evaluate_options.max_concurrency` for authored +suite concurrency. Operators can still override concurrency with +`agentv eval --workers N`; do not author legacy `workers` fields in eval YAML. ## Authoring Shapes diff --git a/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx b/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx index e19bccd7e..eb5f513b0 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/experiments.mdx @@ -10,9 +10,9 @@ AgentV eval files are the runnable authoring artifact. Use top-level label, `target` for the system under test, and flat top-level run controls such as `timeout_seconds` and `threshold`. Use `evaluate_options` for evaluation runtime options such as `repeat`, `budget_usd`, and `max_concurrency`. -Use `agentv eval --workers N` or project config defaults such as -`agentv.config.*` / `.agentv/config.yaml` `execution.workers` for operator-side -overrides. +Use `execution.max_concurrency` in eval YAML or `.agentv/config.yaml` for the +AgentV config graph concurrency field. `agentv eval --workers N` remains an +operator-side override. ```yaml name: support-regression diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index 782c6885a..05d1785dc 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -455,10 +455,78 @@ Project-local YAML config takes precedence over home/global YAML config. AgentV Within one config directory, AgentV reads `config.yaml` first and `config.local.yaml` second. The local overlay wins: plain objects deep-merge, arrays replace, and scalar values from `config.local.yaml` override `config.yaml`. -Use `config.yaml` for portable defaults that can be committed with the eval project. Use `config.local.yaml` for machine-local overrides such as private paths, local result remotes, Dashboard project registry entries, or temporary execution defaults. Project-local `config.local.yaml` is gitignored by default. +Use `config.yaml` for portable defaults and shared eval-definition fields that can be committed with the eval project. Use `config.local.yaml` for machine-local overrides such as private paths, local result remotes, Dashboard project registry entries, or temporary execution defaults. Project-local `config.local.yaml` is gitignored by default. ### YAML config (`config.yaml` plus optional `config.local.yaml`) +Project config and eval YAML share the same eval-definition graph for targets, +graders, tests, defaults, and execution policy. Small projects can keep that +graph inline: + +```yaml +targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini + +tests: + - id: smoke + input: Fix the failing test + +defaults: + target: codex-local + grader: openai-grader + +execution: + max_concurrency: 3 +``` + +Larger projects can decompose any supported top-level field with a direct +`file://...` reference. The referenced file contains that field's value +directly: + +```yaml +targets: file://targets.yaml +graders: file://graders.yaml +tests: file://tests.yaml +defaults: file://defaults.yaml +execution: file://execution.yaml +``` + +```yaml +# .agentv/targets.yaml +- id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] +``` + +```yaml +# .agentv/defaults.yaml +target: codex-local +grader: openai-grader +``` + +Do not wrap referenced field files in another object. For example, +`targets: file://targets.yaml` expects `targets.yaml` to contain a bare array, +not `{ targets: [...] }`. + +`execution.max_concurrency` is AgentV's general eval parallelism field for this +config graph. It is AgentV's run-policy shape, aligned with the general +max-concurrency concept in eval runners, not a copied Promptfoo YAML path. + +Other project defaults can live beside the graph: + ```yaml execution: verbose: true @@ -481,6 +549,7 @@ eval_patterns: | Field | CLI equivalent | Type | Default | Description | |-------|---------------|------|---------|-------------| | `verbose` | `--verbose` | boolean | `false` | Enable verbose logging | +| `max_concurrency` | `--workers` | integer | none | Default eval parallelism for the composable config graph | | `keep_workspaces` | `--keep-workspaces` | boolean | `false` | Always keep temp workspaces after eval | | `workspace_path` | `--workspace-path` | string | none | Machine-local existing workspace directory | | `refs` | none | object | none | Project-defined named references for fields that support `ref://name`, such as shared `default_test` files | @@ -594,6 +663,19 @@ projects: branch: agentv/results/v1 ``` +For larger registries, `$AGENTV_HOME/config.yaml` can point `projects` at a +bare project array: + +```yaml +projects: file://projects.yaml +``` + +```yaml +# $AGENTV_HOME/projects.yaml +- id: agentv + path: /home/user/projects/agentv +``` + When running AgentV from a worktree that needs environment from a primary checkout, load the primary `.env` through the runtime instead of shell-sourcing it: ```bash diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index 9f3411f83..262521348 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -405,7 +405,7 @@ import { defineConfig } from '@agentv/core'; export default defineConfig({ execution: { - workers: 5, + maxConcurrency: 5, maxRetries: 2, verbose: true, }, diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index ed974c498..18b5516eb 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -5,35 +5,59 @@ sidebar: order: 1 --- -Targets define which agent or LLM provider to evaluate. They are configured in `.agentv/targets.yaml` to decouple eval files from provider details. +Targets define which agent or LLM provider to evaluate. Project manifests can +keep targets inline in `.agentv/config.yaml` or decompose them into a direct +field reference such as `targets: file://targets.yaml`. Both forms normalize to +the same config graph. ## Structure ```yaml targets: - - label: azure-base - provider: azure + - id: local-openai + provider: openai + runtime: host config: - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - - - label: vscode_dev - provider: vscode - grader_target: azure-base + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex - - label: local_agent - provider: cli +graders: + - id: openai-grader + provider: openai config: - command: 'python agent.py --prompt {PROMPT}' - grader_target: azure-base + model: gpt-5-mini + +defaults: + target: codex-local + grader: openai-grader +``` + +Use `id` for the stable AgentV target identity. `provider` selects the adapter +or control boundary. `runtime` describes where the provider runs; use `host` as +the shorthand for the current machine, or object form when you need +`mode: host | profile | sandbox` plus runtime-specific settings. Provider +settings belong under `config`. Process-backed providers use +`config.command` as a non-empty argv array. + +Any supported top-level field can be moved to a file reference: + +```yaml +targets: file://targets.yaml +graders: file://graders.yaml +defaults: file://defaults.yaml ``` -Use `label` for AgentV target references and comparison names. Use `id` only when you need to carry a promptfoo provider/backend identifier. The -`provider` field selects the backend kind. Provider-specific settings belong in -`config`; AgentV target extensions such as `grader_target`, `use_target`, -`fallback_targets`, `workers`, and `batch_requests` remain top-level fields on -the target object. +Referenced field files contain the field value directly. `targets.yaml` contains +a bare array, not an object wrapped in `targets:`. ## Environment Variables @@ -43,8 +67,9 @@ eval directory hierarchy when present: ```yaml targets: - - label: my_target + - id: my-target provider: anthropic + runtime: host config: api_key: ${{ ANTHROPIC_API_KEY }} model: ${{ ANTHROPIC_MODEL }} @@ -60,9 +85,13 @@ already-exported secrets into `.env`. | `azure` | LLM | Azure OpenAI | | `anthropic` | LLM | Anthropic Claude API | | `gemini` | LLM | Google Gemini | -| `claude` | Agent | Claude Agent SDK | -| `codex` | Agent | Codex CLI | +| `claude-cli` | Agent | Claude CLI subprocess | +| `claude-sdk` | Agent | Claude Agent SDK | +| `codex-cli` | Agent | Codex CLI subprocess | +| `codex-app-server` | Agent | Codex app-server subprocess | +| `codex-sdk` | Agent | Codex SDK provider | | `pi-coding-agent` | Agent | Pi Coding Agent | +| `pi-cli` | Agent | Pi CLI subprocess | | `vscode` | Agent | VS Code with Copilot | | `vscode-insiders` | Agent | VS Code Insiders | | `cli` | Agent | Any CLI command — see [CLI Provider](/docs/targets/cli-provider) | @@ -70,45 +99,35 @@ already-exported secrets into `.env`. ## Referencing Targets in Evals -Select the system under test with top-level `target` or CLI `--target`. -Test cases do not choose targets; split target-specific cases into separate eval -suites, select them with tags/filters, or run the same eval with different -`--target` values. +Select the system under test with `defaults.target`, top-level `target`, or CLI +`--target`, depending on the command flow. Test cases do not choose targets; +split target-specific cases into separate eval suites, select them with +tags/filters, or run the same eval with different `--target` values. ```yaml -target: azure-base +target: local-openai tests: - id: test-1 - id: test-2 ``` -The string is a target `label` from `.agentv/targets.yaml` or `targets.yaml`. -Use object form when an eval needs a local target variant: +The string is a configured target `id`. Use object form when an eval needs a +local target variant: ```yaml target: - extends: azure-base - label: azure-high-reasoning - model: gpt-5.4 - reasoning_effort: high + id: codex-high-reasoning + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high ``` -`extends` names the base target label. `label` names the eval-local variant for -results and comparison. If `label` is omitted, the eval overrides the base target -under the same label. If `extends` is omitted, the object is a complete inline -target definition and must include enough provider configuration to run. - -## Grader Target - -Agent targets that need LLM-based evaluation specify a `grader_target` — the LLM used to run LLM grader graders: - -```yaml -targets: - - label: codex_target - provider: codex - grader_target: azure-base # LLM used for grading -``` +Use `defaults.grader` for the project default grader. A specific evaluator can +still choose its own grader target when the evaluator supports that override. ### Lifecycle Extensions diff --git a/apps/web/src/content/docs/docs/next/tools/dashboard.mdx b/apps/web/src/content/docs/docs/next/tools/dashboard.mdx index 4d7a6c4fd..f76ddd593 100644 --- a/apps/web/src/content/docs/docs/next/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/next/tools/dashboard.mdx @@ -87,17 +87,6 @@ dashboard: Legacy `studio.threshold`, `studio.pass_threshold`, and root-level `pass_threshold` values are still read for existing projects. When Dashboard saves settings, it writes the canonical `dashboard.threshold` field and preserves unrelated config. -## White label - -Dashboard shows AgentV by default. Override the displayed name with `dashboard.app_name` in project-local `.agentv/config.yaml`: - -```yaml -dashboard: - app_name: ai evals -``` - -You can also set the same field globally in `$AGENTV_HOME/config.yaml` or `~/.agentv/config.yaml`. Project-local config takes precedence over the global value. - ## Run Detail Click any run to see a breakdown by suite, per-test scores, target, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `index.jsonl`—including per-result test bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. diff --git a/examples/features/readme-quickstart/.agentv/config.yaml b/examples/features/readme-quickstart/.agentv/config.yaml new file mode 100644 index 000000000..18be46b82 --- /dev/null +++ b/examples/features/readme-quickstart/.agentv/config.yaml @@ -0,0 +1,6 @@ +targets: file://targets.yaml +graders: file://graders.yaml +tests: file://tests.yaml +defaults: file://defaults.yaml +execution: + max_concurrency: 1 diff --git a/examples/features/readme-quickstart/.agentv/defaults.yaml b/examples/features/readme-quickstart/.agentv/defaults.yaml new file mode 100644 index 000000000..012ed6296 --- /dev/null +++ b/examples/features/readme-quickstart/.agentv/defaults.yaml @@ -0,0 +1,2 @@ +target: local-openai +grader: local-openai-grader diff --git a/examples/features/readme-quickstart/.agentv/graders.yaml b/examples/features/readme-quickstart/.agentv/graders.yaml new file mode 100644 index 000000000..2411d814d --- /dev/null +++ b/examples/features/readme-quickstart/.agentv/graders.yaml @@ -0,0 +1,7 @@ +- id: local-openai-grader + provider: openai + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} diff --git a/examples/features/readme-quickstart/.agentv/targets.yaml b/examples/features/readme-quickstart/.agentv/targets.yaml new file mode 100644 index 000000000..b5bb93e17 --- /dev/null +++ b/examples/features/readme-quickstart/.agentv/targets.yaml @@ -0,0 +1,8 @@ +- id: local-openai + provider: openai + runtime: host + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} diff --git a/examples/features/readme-quickstart/.agentv/tests.yaml b/examples/features/readme-quickstart/.agentv/tests.yaml new file mode 100644 index 000000000..543bb6028 --- /dev/null +++ b/examples/features/readme-quickstart/.agentv/tests.yaml @@ -0,0 +1,2 @@ +- id: fizzbuzz + input: Write FizzBuzz in Python. Use lowercase output strings "fizz", "buzz", and "fizzbuzz". diff --git a/examples/features/readme-quickstart/README.md b/examples/features/readme-quickstart/README.md index fd9ffd0f0..b8bba8be7 100644 --- a/examples/features/readme-quickstart/README.md +++ b/examples/features/readme-quickstart/README.md @@ -2,6 +2,22 @@ This example mirrors the root README quickstart and is used for smoke testing the documented `llm-rubric` and `default_test.options.rubric_prompt` flow. +It includes a composable `.agentv/config.yaml` that decomposes the base config +graph into direct field refs: + +```yaml +targets: file://targets.yaml +graders: file://graders.yaml +tests: file://tests.yaml +defaults: file://defaults.yaml +execution: + max_concurrency: 1 +``` + +Each referenced file contains that field's value directly, such as a bare target +array in `.agentv/targets.yaml` and a bare defaults object in +`.agentv/defaults.yaml`. + Run it against a local OpenAI-compatible endpoint: ```bash diff --git a/packages/core/src/evaluation/config.ts b/packages/core/src/evaluation/config.ts index 34ebbe1cb..136606247 100644 --- a/packages/core/src/evaluation/config.ts +++ b/packages/core/src/evaluation/config.ts @@ -11,7 +11,7 @@ * * export default defineConfig({ * execution: { - * workers: 5, + * maxConcurrency: 5, * maxRetries: 2, * agentTimeoutMs: 120_000, * }, @@ -28,8 +28,8 @@ import { z } from 'zod'; const ExecutionConfigSchema = z .object({ - /** Number of parallel workers (default: 3) */ - workers: z.number().int().min(1).max(50).optional(), + /** General eval parallelism (default: 3) */ + maxConcurrency: z.number().int().min(1).max(50).optional(), /** Maximum retries on failure (default: 2) */ maxRetries: z.number().int().min(0).optional(), /** Agent timeout in milliseconds. No timeout if not set. */ @@ -41,6 +41,14 @@ const ExecutionConfigSchema = z }) .passthrough() .superRefine((value, ctx) => { + const supportedFields = new Set([ + 'maxConcurrency', + 'maxRetries', + 'agentTimeoutMs', + 'verbose', + 'keepWorkspaces', + ]); + if ('otelFile' in value) { ctx.addIssue({ code: z.ZodIssueCode.custom, @@ -49,9 +57,19 @@ const ExecutionConfigSchema = z 'execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata.', }); } + + for (const key of Object.keys(value)) { + if (!supportedFields.has(key) && key !== 'otelFile') { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [key], + message: `Unsupported execution field '${key}'`, + }); + } + } }) - .transform(({ workers, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({ - ...(workers !== undefined && { workers }), + .transform(({ maxConcurrency, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({ + ...(maxConcurrency !== undefined && { maxConcurrency }), ...(maxRetries !== undefined && { maxRetries }), ...(agentTimeoutMs !== undefined && { agentTimeoutMs }), ...(verbose !== undefined && { verbose }), @@ -132,7 +150,7 @@ export type AgentVConfig = z.infer; * import { defineConfig } from '@agentv/core'; * * export default defineConfig({ - * execution: { workers: 5 }, + * execution: { maxConcurrency: 5 }, * output: { dir: './results' }, * limits: { maxCostUsd: 10.0 }, * }); diff --git a/packages/core/src/evaluation/experiment.ts b/packages/core/src/evaluation/experiment.ts index a59cde7dc..60e76ef33 100644 --- a/packages/core/src/evaluation/experiment.ts +++ b/packages/core/src/evaluation/experiment.ts @@ -404,7 +404,7 @@ function rejectExperimentWorkers(raw: unknown): void { return; } throw new Error( - 'Experiment workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.', + 'Experiment workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.', ); } diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts new file mode 100644 index 000000000..d533356b0 --- /dev/null +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -0,0 +1,370 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { isPlainConfigObject } from '../../config-overlays.js'; +import { parseYamlValue } from '../yaml-loader.js'; + +const FILE_PROTOCOL = 'file://'; +const ARRAY_FIELDS = new Set(['targets', 'graders', 'tests', 'projects']); +const OBJECT_FIELDS = new Set([ + 'defaults', + 'execution', + 'results', + 'hooks', + 'refs', + 'tags', + 'dashboard', +]); +const SCALAR_OR_ARRAY_FIELDS = new Set(['eval_patterns']); +const SCALAR_FIELDS = new Set(['required_version', '$schema']); +const SUPPORTED_FILE_REF_FIELDS = new Set([ + ...ARRAY_FIELDS, + ...OBJECT_FIELDS, + ...SCALAR_OR_ARRAY_FIELDS, + ...SCALAR_FIELDS, +]); +const RUNTIME_MODES = new Set(['host', 'profile', 'sandbox']); +const AMBIGUOUS_PROVIDER_ALIASES = new Set(['codex', 'claude', 'copilot', 'pi']); +const REMOVED_TARGET_FIELDS = new Map([ + ['label', "target identity uses 'id'; remove 'label'."], + ['name', "target identity uses 'id'; remove 'name'."], + ['executable', "put process argv under 'config.command'."], + ['binary', "put process argv under 'config.command'."], + ['args', "put process argv under 'config.command'."], + ['arguments', "put process argv under 'config.command'."], + ['grader_target', "grader selection belongs in 'defaults.grader' or evaluator config."], + ['workers', "target-level 'workers' is not general run policy; use 'execution.max_concurrency'."], + [ + 'batch_requests', + "target-level 'batch_requests' is not part of the base config contract; keep provider batching under provider-specific config only when needed.", + ], + [ + 'subagent_mode_allowed', + "target-level 'subagent_mode_allowed' is not part of the base config contract.", + ], +]); + +export type RuntimeMode = 'host' | 'profile' | 'sandbox'; + +export type NormalizedRuntimeConfig = { + readonly mode: RuntimeMode; + readonly [key: string]: unknown; +}; + +export type NormalizedTargetConfig = { + readonly id: string; + readonly provider: string; + readonly runtime: NormalizedRuntimeConfig; + readonly config: Record; +}; + +export type NormalizedGraderConfig = { + readonly id: string; + readonly provider: string; + readonly config: Record; +}; + +export type ConfigDefaults = { + readonly target?: string; + readonly grader?: string; +}; + +export type ConfigExecution = { + readonly max_concurrency?: number; +}; + +export type ComposableConfigGraph = { + readonly targets?: readonly NormalizedTargetConfig[]; + readonly graders?: readonly NormalizedGraderConfig[]; + readonly tests?: readonly unknown[]; + readonly defaults?: ConfigDefaults; + readonly execution?: ConfigExecution; +}; + +type NormalizeOptions = { + readonly allowExecutionDefaultFields?: boolean; +}; + +export async function resolveConfigFieldReferences( + rawConfig: Record, + configPath: string, +): Promise> { + const resolvedEntries = await Promise.all( + Object.entries(rawConfig).map(async ([field, value]) => { + if (!isFileReference(value)) { + return [field, value] as const; + } + if (!SUPPORTED_FILE_REF_FIELDS.has(field)) { + throw new Error( + `Field '${field}' in ${configPath} cannot use a file:// reference because it is not a supported top-level config field.`, + ); + } + return [field, await loadReferencedFieldValue(field, value, configPath)] as const; + }), + ); + return Object.fromEntries(resolvedEntries); +} + +export async function loadComposableConfigGraph( + configPath: string, +): Promise { + const raw = parseYamlValue(await readFile(configPath, 'utf8')); + if (!isPlainConfigObject(raw)) { + throw new Error(`Config graph at ${configPath} must be a YAML object.`); + } + const resolved = await resolveConfigFieldReferences(raw, configPath); + return normalizeComposableConfigGraph(resolved, configPath); +} + +export function normalizeComposableConfigGraph( + rawConfig: Record, + configPath: string, + options: NormalizeOptions = {}, +): ComposableConfigGraph { + const graph: ComposableConfigGraph = { + ...(rawConfig.targets !== undefined + ? { targets: parseTargets(rawConfig.targets, `${configPath}:targets`) } + : {}), + ...(rawConfig.graders !== undefined + ? { graders: parseGraders(rawConfig.graders, `${configPath}:graders`) } + : {}), + ...(rawConfig.tests !== undefined + ? { tests: parseArray(rawConfig.tests, `${configPath}:tests`) } + : {}), + ...(rawConfig.defaults !== undefined + ? { defaults: parseDefaults(rawConfig.defaults, `${configPath}:defaults`) } + : {}), + ...(rawConfig.execution !== undefined + ? { + execution: parseExecution( + rawConfig.execution, + `${configPath}:execution`, + options.allowExecutionDefaultFields ?? false, + ), + } + : {}), + }; + + validateDefaultSelections(graph, configPath); + return graph; +} + +function isFileReference(value: unknown): value is string { + return typeof value === 'string' && value.startsWith(FILE_PROTOCOL); +} + +async function loadReferencedFieldValue( + field: string, + reference: string, + ownerPath: string, +): Promise { + const referencedPath = resolveReferencePath(reference, ownerPath); + const parsed = parseYamlValue(await readFile(referencedPath, 'utf8')); + if (isPlainConfigObject(parsed) && Object.prototype.hasOwnProperty.call(parsed, field)) { + throw new Error( + `Invalid ${field} file reference in ${ownerPath}: ${referencedPath} must contain the '${field}' value directly, not an object wrapped in '${field}'.`, + ); + } + validateReferencedFieldShape(field, parsed, referencedPath); + return parsed; +} + +function resolveReferencePath(reference: string, ownerPath: string): string { + const filePath = reference.slice(FILE_PROTOCOL.length); + return path.isAbsolute(filePath) + ? filePath + : path.resolve(path.dirname(path.resolve(ownerPath)), filePath); +} + +function validateReferencedFieldShape(field: string, value: unknown, referencedPath: string): void { + if (ARRAY_FIELDS.has(field) && !Array.isArray(value)) { + throw new Error(`Referenced ${field} file ${referencedPath} must contain a YAML array.`); + } + if (OBJECT_FIELDS.has(field) && !isPlainConfigObject(value)) { + throw new Error(`Referenced ${field} file ${referencedPath} must contain a YAML object.`); + } + if (SCALAR_OR_ARRAY_FIELDS.has(field) && typeof value !== 'string' && !Array.isArray(value)) { + throw new Error( + `Referenced ${field} file ${referencedPath} must contain a scalar or array value.`, + ); + } +} + +function parseArray(value: unknown, location: string): readonly unknown[] { + if (!Array.isArray(value)) { + throw new Error(`Invalid ${location}: expected an array.`); + } + return value; +} + +function parseTargets(value: unknown, location: string): readonly NormalizedTargetConfig[] { + return parseArray(value, location).map((entry, index) => + parseTarget(entry, `${location}[${index}]`), + ); +} + +function parseTarget(value: unknown, location: string): NormalizedTargetConfig { + if (!isPlainConfigObject(value)) { + throw new Error(`Invalid ${location}: target must be an object.`); + } + for (const [field, message] of REMOVED_TARGET_FIELDS) { + if (Object.prototype.hasOwnProperty.call(value, field)) { + throw new Error(`Invalid ${location}.${field}: ${message}`); + } + } + + const id = readRequiredString(value.id, `${location}.id`); + const provider = readRequiredString(value.provider, `${location}.provider`); + if (AMBIGUOUS_PROVIDER_ALIASES.has(provider)) { + throw new Error( + `Invalid ${location}.provider: '${provider}' is ambiguous; choose an explicit provider such as '${provider}-cli' or '${provider}-sdk'.`, + ); + } + + const config = readOptionalObject(value.config, `${location}.config`) ?? {}; + validateCommand(config.command, `${location}.config.command`); + + return { + id, + provider, + runtime: parseRuntime(value.runtime, `${location}.runtime`), + config, + }; +} + +function parseRuntime(value: unknown, location: string): NormalizedRuntimeConfig { + if (typeof value === 'string') { + const mode = value.trim(); + if (RUNTIME_MODES.has(mode)) { + return { mode: mode as RuntimeMode }; + } + } + if (isPlainConfigObject(value)) { + const mode = typeof value.mode === 'string' ? value.mode.trim() : ''; + if (RUNTIME_MODES.has(mode)) { + return { ...value, mode: mode as RuntimeMode }; + } + } + throw new Error(`Invalid ${location}: use 'host' or an object with mode: host|profile|sandbox.`); +} + +function parseGraders(value: unknown, location: string): readonly NormalizedGraderConfig[] { + return parseArray(value, location).map((entry, index) => { + const graderLocation = `${location}[${index}]`; + if (!isPlainConfigObject(entry)) { + throw new Error(`Invalid ${graderLocation}: grader must be an object.`); + } + const id = readRequiredString(entry.id, `${graderLocation}.id`); + const provider = readRequiredString(entry.provider, `${graderLocation}.provider`); + const config = readOptionalObject(entry.config, `${graderLocation}.config`) ?? {}; + validateCommand(config.command, `${graderLocation}.config.command`); + return { id, provider, config }; + }); +} + +function parseDefaults(value: unknown, location: string): ConfigDefaults { + const defaults = readOptionalObject(value, location); + if (!defaults) { + throw new Error(`Invalid ${location}: expected an object.`); + } + const target = readOptionalString(defaults.target, `${location}.target`); + const grader = readOptionalString(defaults.grader, `${location}.grader`); + return { + ...(target !== undefined ? { target } : {}), + ...(grader !== undefined ? { grader } : {}), + }; +} + +function parseExecution( + value: unknown, + location: string, + allowDefaultFields: boolean, +): ConfigExecution { + const execution = readOptionalObject(value, location); + if (!execution) { + throw new Error(`Invalid ${location}: expected an object.`); + } + for (const key of Object.keys(execution)) { + if (key !== 'max_concurrency') { + if (allowDefaultFields) { + continue; + } + throw new Error( + `Invalid ${location}.${key}: unsupported execution field. Use execution.max_concurrency for eval parallelism.`, + ); + } + } + const rawMaxConcurrency = execution.max_concurrency; + if (rawMaxConcurrency === undefined) { + return {}; + } + if ( + typeof rawMaxConcurrency !== 'number' || + !Number.isInteger(rawMaxConcurrency) || + rawMaxConcurrency < 1 || + rawMaxConcurrency > 50 + ) { + throw new Error(`Invalid ${location}.max_concurrency: expected an integer between 1 and 50.`); + } + return { max_concurrency: rawMaxConcurrency }; +} + +function validateDefaultSelections(graph: ComposableConfigGraph, configPath: string): void { + if (graph.defaults?.target !== undefined) { + const targetIds = new Set((graph.targets ?? []).map((target) => target.id)); + if (!targetIds.has(graph.defaults.target)) { + throw new Error( + `Invalid defaults.target in ${configPath}: '${graph.defaults.target}' does not match a configured target id.`, + ); + } + } + if (graph.defaults?.grader !== undefined) { + const graderIds = new Set((graph.graders ?? []).map((grader) => grader.id)); + if (!graderIds.has(graph.defaults.grader)) { + throw new Error( + `Invalid defaults.grader in ${configPath}: '${graph.defaults.grader}' does not match a configured grader id.`, + ); + } + } +} + +function readRequiredString(value: unknown, location: string): string { + const result = readOptionalString(value, location); + if (result === undefined) { + throw new Error(`Invalid ${location}: expected a non-empty string.`); + } + return result; +} + +function readOptionalString(value: unknown, location: string): string | undefined { + if (value === undefined || value === null) { + return undefined; + } + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`Invalid ${location}: expected a non-empty string.`); + } + return value.trim(); +} + +function readOptionalObject(value: unknown, location: string): Record | undefined { + if (value === undefined || value === null) { + return undefined; + } + if (!isPlainConfigObject(value)) { + throw new Error(`Invalid ${location}: expected an object.`); + } + return value; +} + +function validateCommand(value: unknown, location: string): void { + if (value === undefined) { + return; + } + if ( + !Array.isArray(value) || + value.length === 0 || + !value.every((entry) => typeof entry === 'string' && entry.trim().length > 0) + ) { + throw new Error(`Invalid ${location}: expected a non-empty argv array of strings.`); + } +} diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index c7d9f2538..7251e3be2 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -23,6 +23,11 @@ import type { } from '../types.js'; import { isJsonObject } from '../types.js'; import { parseYamlValue } from '../yaml-loader.js'; +import { + type ComposableConfigGraph, + normalizeComposableConfigGraph, + resolveConfigFieldReferences, +} from './config-graph.js'; import { buildDirectoryChain, fileExists } from './file-resolver.js'; const ANSI_YELLOW = '\u001b[33m'; @@ -37,7 +42,7 @@ export const DEFAULT_EVAL_PATTERNS: readonly string[] = [ ]; export type ExecutionDefaults = { - readonly workers?: number; + readonly max_concurrency?: number; readonly verbose?: boolean; readonly keep_workspaces?: boolean; readonly workspace_path?: string; @@ -76,7 +81,7 @@ export type AgentVConfig = { * reserved key `experiment` participates in experiment-namespace resolution. */ readonly tags?: Record; -}; +} & ComposableConfigGraph; /** * Load optional AgentV YAML configuration. @@ -145,9 +150,15 @@ async function readConfigFilePair( repoRoot: string, ): Promise { const localConfigPath = getLocalConfigPath(configPath); - const base = stripLocalOnlyExecutionDefaults(await readConfigObjectFile(configPath), configPath); + const base = stripLocalOnlyExecutionDefaults( + await resolveConfigObjectFileReferences(await readConfigObjectFile(configPath), configPath), + configPath, + ); const local = stripLocalOnlyExecutionDefaults( - await readConfigObjectFile(localConfigPath), + await resolveConfigObjectFileReferences( + await readConfigObjectFile(localConfigPath), + localConfigPath, + ), localConfigPath, ); const rawMerged = base && local ? mergeConfigObjects(base, local) : (local ?? base); @@ -157,6 +168,16 @@ async function readConfigFilePair( return parseConfigObject(rawMerged, local ? localConfigPath : configPath, repoRoot); } +async function resolveConfigObjectFileReferences( + rawConfig: Record | undefined, + configPath: string, +): Promise | undefined> { + if (!rawConfig) { + return undefined; + } + return resolveConfigFieldReferences(rawConfig, configPath); +} + function parseConfigObject( rawConfig: Record, configPath: string, @@ -189,23 +210,36 @@ function parseConfigObject( return null; } - const executionDefaults = parseExecutionDefaults( - (parsed as Record).execution, - configPath, - ); + const rawExecution = (parsed as Record).execution; + if (isJsonObject(rawExecution) && rawExecution.workers !== undefined) { + logWarning( + `Invalid execution.workers in ${configPath}; use execution.max_concurrency for eval parallelism.`, + ); + return null; + } + + const executionDefaults = parseExecutionDefaults(rawExecution, configPath); const results = parseResultsConfig((parsed as Record).results, configPath); const hooks = parseHooksConfig((parsed as Record).hooks, configPath); const tags = parseTagsConfig((parsed as Record).tags, configPath); const refs = parseRefsConfig((parsed as Record).refs, configPath); + const graph = normalizeComposableConfigGraph(parsed as Record, configPath, { + allowExecutionDefaultFields: true, + }); + const execution = mergeExecutionConfig(executionDefaults, graph.execution); return { required_version: requiredVersion as string | undefined, eval_patterns: evalPatterns as readonly string[] | undefined, - execution: executionDefaults, + ...(execution && { execution }), results, ...(hooks && { hooks }), ...(refs && { refs }), ...(tags && { tags }), + ...(graph.targets && { targets: graph.targets }), + ...(graph.graders && { graders: graph.graders }), + ...(graph.tests && { tests: graph.tests }), + ...(graph.defaults && { defaults: graph.defaults }), }; } catch (error) { const message = (error as Error).message; @@ -217,6 +251,14 @@ function parseConfigObject( } } +function mergeExecutionConfig( + defaults: ExecutionDefaults | undefined, + graph: ComposableConfigGraph['execution'], +): ExecutionDefaults | undefined { + const merged = { ...(defaults ?? {}), ...(graph ?? {}) }; + return Object.keys(merged).length > 0 ? merged : undefined; +} + function parseRefsConfig(raw: unknown, configPath: string): ReferenceMap | undefined { if (raw === undefined || raw === null) { return undefined; @@ -317,8 +359,31 @@ function rejectAuthoredRuntimeContainers(suite: JsonObject): void { throw new Error("Top-level 'budget_usd' has been removed. Use evaluate_options.budget_usd."); } if (suite.execution !== undefined) { + assertAllowedSuiteExecution(suite.execution); + } +} + +function assertAllowedSuiteExecution(rawExecution: JsonValue): void { + if (!isJsonObject(rawExecution)) { + throw new Error("Invalid top-level 'execution': expected an object."); + } + for (const key of Object.keys(rawExecution)) { + if (key !== 'max_concurrency') { + throw new Error( + `Top-level 'execution.${key}' is not part of eval YAML. Use execution.max_concurrency for AgentV eval parallelism; keep target and other run controls at their supported top-level or evaluate_options fields.`, + ); + } + } + const maxConcurrency = rawExecution.max_concurrency; + if ( + maxConcurrency !== undefined && + (typeof maxConcurrency !== 'number' || + !Number.isInteger(maxConcurrency) || + maxConcurrency < 1 || + maxConcurrency > 50) + ) { throw new Error( - "Top-level 'execution' is not part of eval YAML. Put target and run controls at the top level, authored concurrency under evaluate_options.max_concurrency, and operational defaults in CLI flags or project config.", + "Invalid top-level 'execution.max_concurrency': expected an integer between 1 and 50.", ); } } @@ -538,11 +603,15 @@ export function parseTargetHooks(raw: unknown): TargetHooksConfig | undefined { /** * Extract suite-level max concurrency from eval YAML. * - * Preferred authoring uses evaluate_options.max_concurrency, matching the - * lowest-common-denominator naming used by other eval runners. Internal - * TypeScript continues to pass this as workers/maxConcurrency at runtime. + * AgentV eval YAML accepts execution.max_concurrency as the config-graph field + * and evaluate_options.max_concurrency for promptfoo-shaped eval options. The + * runner still receives the resolved value through its historical workers slot. */ export function extractWorkersFromSuite(suite: JsonObject): number | undefined { + rejectAuthoredRuntimeContainers(suite); + if (isJsonObject(suite.execution) && typeof suite.execution.max_concurrency === 'number') { + return suite.execution.max_concurrency; + } return getSuiteEvaluateOptionsNumber( suite, 'max_concurrency', @@ -681,11 +750,22 @@ export function parseExecutionDefaults( const obj = raw as Record; const result: Record = {}; - const workers = obj.workers; - if (typeof workers === 'number' && Number.isInteger(workers) && workers >= 1 && workers <= 50) { - result.workers = workers; - } else if (workers !== undefined) { - logWarning(`Invalid execution.workers in ${configPath}, expected integer 1-50`); + if (obj.workers !== undefined) { + logWarning( + `execution.workers in ${configPath} has been removed; use execution.max_concurrency`, + ); + } + + const maxConcurrency = obj.max_concurrency; + if ( + typeof maxConcurrency === 'number' && + Number.isInteger(maxConcurrency) && + maxConcurrency >= 1 && + maxConcurrency <= 50 + ) { + result.max_concurrency = maxConcurrency; + } else if (maxConcurrency !== undefined) { + logWarning(`Invalid execution.max_concurrency in ${configPath}, expected integer 1-50`); } if (typeof obj.verbose === 'boolean') { diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index e6b5cf401..95bb42d2a 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -4,6 +4,10 @@ import path from 'node:path'; import { getLocalConfigPath } from '../../config-overlays.js'; import { getAgentvConfigDir } from '../../paths.js'; import { interpolateEnv } from '../interpolation.js'; +import { + normalizeComposableConfigGraph, + resolveConfigFieldReferences, +} from '../loaders/config-graph.js'; import { parseYamlValue } from '../yaml-loader.js'; import type { ValidationError, ValidationResult } from './types.js'; @@ -31,7 +35,17 @@ export async function validateConfigFile( return { valid: false, filePath, fileType: 'config', errors }; } - const config = parsed as Record; + let config: Record; + try { + config = await resolveConfigFieldReferences(parsed as Record, filePath); + } catch (error) { + errors.push({ + severity: 'error', + filePath, + message: (error as Error).message, + }); + return { valid: false, filePath, fileType: 'config', errors }; + } // Validate eval_patterns if present const evalPatterns = config.eval_patterns; @@ -77,6 +91,8 @@ export async function validateConfigFile( validateResultsConfig(errors, filePath, config.results, 'results'); validateRepoResolversConfig(errors, filePath, config.repo_resolvers); validateRefsConfig(errors, filePath, config.refs); + validateComposableGraph(errors, filePath, config); + validateDashboardConfig(errors, filePath, config.dashboard); const projects = config.projects; if (projects !== undefined) { @@ -108,6 +124,10 @@ export async function validateConfigFile( 'results', 'repo_resolvers', 'refs', + 'targets', + 'graders', + 'tests', + 'defaults', 'projects', 'dashboard', 'studio', @@ -138,6 +158,40 @@ export async function validateConfigFile( } } +function validateComposableGraph( + errors: ValidationError[], + filePath: string, + config: Record, +): void { + try { + normalizeComposableConfigGraph(config, filePath); + } catch (error) { + addError(errors, filePath, undefined, (error as Error).message); + } +} + +function validateDashboardConfig( + errors: ValidationError[], + filePath: string, + rawDashboard: unknown, +): void { + if (rawDashboard === undefined) { + return; + } + if (!isPlainObject(rawDashboard)) { + addError(errors, filePath, 'dashboard', "Field 'dashboard' must be an object"); + return; + } + if (Object.prototype.hasOwnProperty.call(rawDashboard, 'app_name')) { + addError( + errors, + filePath, + 'dashboard.app_name', + "Field 'dashboard.app_name' has been removed; the Dashboard app name is not user-configurable.", + ); + } +} + function validateRefsConfig(errors: ValidationError[], filePath: string, rawRefs: unknown): void { if (rawRefs === undefined) { return; @@ -291,10 +345,10 @@ function validateProjects(errors: ValidationError[], filePath: string, projects: function addError( errors: ValidationError[], filePath: string, - location: string, + location: string | undefined, message: string, ): void { - errors.push({ severity: 'error', filePath, location, message }); + errors.push({ severity: 'error', filePath, ...(location ? { location } : {}), message }); } function isPlainObject(value: unknown): value is Record { diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 3d1c8fe4a..494d9eb34 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -648,6 +648,45 @@ const TestsSchema = z.union([ z.string().min(1), ]); +const ConfigRuntimeSchema = z.union([ + z.enum(['host', 'profile', 'sandbox']), + z + .object({ + mode: z.enum(['host', 'profile', 'sandbox']), + }) + .passthrough(), +]); + +const ConfigTargetSchema = z + .object({ + id: z.string().min(1), + provider: z.string().min(1), + runtime: ConfigRuntimeSchema, + config: JsonRecordSchema.optional(), + }) + .strict(); + +const ConfigGraderSchema = z + .object({ + id: z.string().min(1), + provider: z.string().min(1), + config: JsonRecordSchema.optional(), + }) + .strict(); + +const ConfigDefaultsSchema = z + .object({ + target: z.string().min(1).optional(), + grader: z.string().min(1).optional(), + }) + .strict(); + +const ConfigExecutionSchema = z + .object({ + max_concurrency: z.number().int().min(1).max(50).optional(), + }) + .strict(); + const ScenarioConfigSchema = z .object({ vars: JsonObjectSchema.optional(), @@ -710,6 +749,9 @@ export const EvalFileSchema: z.ZodType = z imports: ImportsSchema.optional(), // Tests (inline raw cases, legacy include entries, or external raw-case path) tests: TestsSchema.optional(), + // Shared composable config graph fields + graders: z.union([z.array(ConfigGraderSchema), z.string().min(1)]).optional(), + defaults: z.union([ConfigDefaultsSchema, z.string().min(1)]).optional(), // Deprecated aliases eval_cases: TestsSchema.optional(), // Target @@ -735,7 +777,7 @@ export const EvalFileSchema: z.ZodType = z extensions: z.array(ExtensionSchema).optional(), on_run_complete: z.never().optional(), policy: z.never().optional(), - execution: z.never().optional(), + execution: z.union([ConfigExecutionSchema, z.string().min(1)]).optional(), // Suite-level assert entries assert: z.array(AssertionItemSchema).optional(), // Suite-level content preprocessors shared by evaluators diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index 759d8e694..06c4a9cdf 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -92,6 +92,8 @@ const KNOWN_TOP_LEVEL_FIELDS = new Set([ 'prompts', 'imports', 'tests', + 'graders', + 'defaults', 'target', 'targets', 'model', @@ -166,17 +168,13 @@ const KNOWN_TEST_EXECUTION_FIELDS = new Set([ const REMOVED_TOP_LEVEL_FIELDS = new Map([ [ 'workers', - "'workers' has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.", + "'workers' has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.", ], ['model', "Top-level 'model' is not part of eval YAML. Put model inside the target object."], [ 'policy', "Top-level 'policy' is not part of eval YAML. Put repeat under evaluate_options.repeat, timeout_seconds and threshold at the top level, and budget_usd under evaluate_options.", ], - [ - 'execution', - "Top-level 'execution' is not part of eval YAML. Put target and run controls at the top level, authored concurrency under evaluate_options.max_concurrency, and operational defaults in CLI flags or project config.", - ], [ 'providers', "Top-level 'providers' is not a runtime alias in AgentV eval YAML. Use 'targets' for systems under test; provider names backend kind inside each target.", @@ -406,6 +404,7 @@ export async function validateEvalFile(filePath: string): Promise 50) + ) { + errors.push({ + severity: 'error', + filePath, + location: `${location}.max_concurrency`, + message: "Invalid 'execution.max_concurrency' field (must be an integer between 1 and 50)", + }); + } +} + function rejectWorkersField( raw: JsonValue | undefined, location: string, @@ -705,7 +749,7 @@ function rejectWorkersField( severity: 'error', filePath, location: `${location}.workers`, - message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + message: `${location}.workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`, }); } rejectTargetWorkers(raw.targets, `${location}.targets`, filePath, errors); @@ -728,7 +772,7 @@ function rejectTargetWorkers( severity: 'error', filePath, location: `${location}[${index}].workers`, - message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + message: `${location}[${index}].workers has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`, }); }); } diff --git a/packages/core/src/evaluation/workspace/setup.ts b/packages/core/src/evaluation/workspace/setup.ts index 8e7b24a9b..47fd27972 100644 --- a/packages/core/src/evaluation/workspace/setup.ts +++ b/packages/core/src/evaluation/workspace/setup.ts @@ -454,7 +454,7 @@ export async function prepareSharedWorkspaceSetup( [ `Warning: This eval uses a shared workspace with ${workers} workers.`, 'If the agent under test makes file edits, concurrent runs may corrupt each other.', - 'To limit concurrency, pass --workers 1 on the command line or set execution.workers in agentv.config.* / .agentv/config.yaml.', + 'To limit concurrency, pass --workers 1 on the command line or set execution.max_concurrency in eval YAML or .agentv/config.yaml.', ].join('\n'), ); } diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 8f63ca457..3d1135b72 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -1624,7 +1624,7 @@ function rejectAuthoredWorkers(parsed: JsonObject): void { } throw new Error( - `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with evaluate_options.max_concurrency, or operational defaults with --workers, agentv.config.*, .agentv/config.yaml execution.workers, or target-level runtime config.`, + `${locations[0]} has been removed from eval YAML. Set authored eval concurrency with execution.max_concurrency or evaluate_options.max_concurrency.`, ); } @@ -2182,9 +2182,18 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO ); } if (suite.execution !== undefined) { - throw new Error( - `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' is not part of eval YAML. Put target and run controls at the top level, authored concurrency under evaluate_options.max_concurrency, and operational defaults in CLI flags or project config.`, - ); + if (!isJsonObject(suite.execution)) { + throw new Error( + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution' must be an object with max_concurrency.`, + ); + } + for (const key of Object.keys(suite.execution)) { + if (key !== 'max_concurrency') { + throw new Error( + `Invalid eval runtime config in ${evalFilePath}: top-level 'execution.${key}' is not part of eval YAML. Use execution.max_concurrency for eval parallelism.`, + ); + } + } } if (suite.providers !== undefined) { throw new Error( diff --git a/packages/core/src/projects.ts b/packages/core/src/projects.ts index 48ef108d3..df5fe0b9c 100644 --- a/packages/core/src/projects.ts +++ b/packages/core/src/projects.ts @@ -213,14 +213,36 @@ function readHomeConfig(configPath: string): Record { if (!existsSync(configPath)) return {}; try { const parsed = parseYamlValue(readFileSync(configPath, 'utf-8')) as unknown; - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) - ? (parsed as Record) - : {}; + const config = + parsed && typeof parsed === 'object' && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + return resolveProjectsFieldReference(config, configPath); } catch { return {}; } } +function resolveProjectsFieldReference( + config: Record, + configPath: string, +): Record { + if (typeof config.projects !== 'string' || !config.projects.startsWith('file://')) { + return config; + } + const referencePath = config.projects.slice('file://'.length); + const projectsPath = path.isAbsolute(referencePath) + ? referencePath + : path.resolve(path.dirname(configPath), referencePath); + const parsed = parseYamlValue(readFileSync(projectsPath, 'utf-8')) as unknown; + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed) && 'projects' in parsed) { + throw new Error( + `Invalid projects file reference in ${configPath}: ${projectsPath} must contain the projects array directly, not an object wrapped in 'projects'.`, + ); + } + return { ...config, projects: parsed }; +} + function readMergedHomeConfig(configPath: string): Record { const base = readHomeConfig(configPath); const local = readHomeConfig(getLocalConfigPath(configPath)); diff --git a/packages/core/test/evaluation/config.test.ts b/packages/core/test/evaluation/config.test.ts index 636051361..93e6c2a8b 100644 --- a/packages/core/test/evaluation/config.test.ts +++ b/packages/core/test/evaluation/config.test.ts @@ -24,7 +24,7 @@ describe('defineConfig execution defaults', () => { it('accepts all execution fields together', () => { const config = defineConfig({ execution: { - workers: 5, + maxConcurrency: 5, maxRetries: 2, agentTimeoutMs: 120_000, verbose: true, @@ -32,7 +32,7 @@ describe('defineConfig execution defaults', () => { }, }); expect(config.execution).toEqual({ - workers: 5, + maxConcurrency: 5, maxRetries: 2, agentTimeoutMs: 120_000, verbose: true, @@ -44,9 +44,14 @@ describe('defineConfig execution defaults', () => { expect(() => defineConfig({ execution: { verbose: 'yes' } } as never)).toThrow(); }); - it('drops legacy traceFile fields from typed config', () => { - const config = defineConfig({ execution: { traceFile: 'trace.jsonl' } } as never); - expect(config.execution).toEqual({}); + it('rejects removed execution.workers', () => { + expect(() => defineConfig({ execution: { workers: 5 } } as never)).toThrow(/workers/); + }); + + it('rejects legacy traceFile fields from typed config', () => { + expect(() => defineConfig({ execution: { traceFile: 'trace.jsonl' } } as never)).toThrow( + /traceFile/, + ); }); it('rejects removed output.format', () => { diff --git a/packages/core/test/evaluation/eval-inline-experiment.test.ts b/packages/core/test/evaluation/eval-inline-experiment.test.ts index 86b961dc2..cb66e3d2d 100644 --- a/packages/core/test/evaluation/eval-inline-experiment.test.ts +++ b/packages/core/test/evaluation/eval-inline-experiment.test.ts @@ -561,7 +561,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ].join('\n'), ); - await expect(loadTestSuite(legacyPath, tempDir)).rejects.toThrow(/top-level 'execution'/); + await expect(loadTestSuite(legacyPath, tempDir)).rejects.toThrow(/execution\.target/); const removedPath = path.join(tempDir, 'removed.eval.yaml'); await writeFile( @@ -1139,7 +1139,7 @@ describe('eval.yaml flat runtime controls and tests imports', () => { ].join('\n'), ); - await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow(/top-level 'execution'/); + await expect(loadTestSuite(parentPath, tempDir)).rejects.toThrow(/execution\.workspace/); }); it('does not apply imported child run controls when parent has no run controls', async () => { diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index e3ee1aeb4..b79025772 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -3,6 +3,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { loadComposableConfigGraph } from '../../../src/evaluation/loaders/config-graph.js'; import { extractBudgetUsd, extractFailOnError, @@ -41,6 +42,245 @@ function withOptionalEnv( } describe('loadConfig', () => { + it('loads inline composable config graph fields from .agentv/config.yaml', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-inline-')); + try { + const projectDir = path.join(tempDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + const localConfigDir = path.join(projectDir, '.agentv'); + mkdirSync(evalDir, { recursive: true }); + mkdirSync(localConfigDir, { recursive: true }); + writeFileSync( + path.join(localConfigDir, 'config.yaml'), + [ + 'targets:', + ' - id: codex-local', + ' provider: codex-app-server', + ' runtime: host', + ' config:', + ' command: ["codex", "app-server"]', + ' model: gpt-5-codex', + 'graders:', + ' - id: openai-grader', + ' provider: openai', + ' config:', + ' model: gpt-5-mini', + 'tests:', + ' - id: smoke', + ' input: Fix the failing test', + 'defaults:', + ' target: codex-local', + ' grader: openai-grader', + 'execution:', + ' max_concurrency: 3', + '', + ].join('\n'), + ); + + const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); + + expect(config?.targets).toEqual([ + { + id: 'codex-local', + provider: 'codex-app-server', + runtime: { mode: 'host' }, + config: { command: ['codex', 'app-server'], model: 'gpt-5-codex' }, + }, + ]); + expect(config?.graders).toEqual([ + { id: 'openai-grader', provider: 'openai', config: { model: 'gpt-5-mini' } }, + ]); + expect(config?.tests).toEqual([{ id: 'smoke', input: 'Fix the failing test' }]); + expect(config?.defaults).toEqual({ target: 'codex-local', grader: 'openai-grader' }); + expect(config?.execution?.max_concurrency).toBe(3); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('normalizes split file refs the same as inline fields', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-split-')); + try { + const inlinePath = path.join(tempDir, 'inline.eval.yaml'); + const splitPath = path.join(tempDir, 'split.eval.yaml'); + writeFileSync( + inlinePath, + [ + 'targets:', + ' - id: codex-local', + ' provider: codex-app-server', + ' runtime:', + ' mode: profile', + ' home: .agentv/profiles/codex-local', + ' config:', + ' command: ["codex"]', + 'graders:', + ' - id: openai-grader', + ' provider: openai', + ' config: {}', + 'tests:', + ' - id: smoke', + ' input: Fix the failing test', + 'defaults:', + ' target: codex-local', + ' grader: openai-grader', + 'execution:', + ' max_concurrency: 2', + '', + ].join('\n'), + ); + writeFileSync( + splitPath, + [ + 'targets: file://targets.yaml', + 'graders: file://graders.yaml', + 'tests: file://tests.yaml', + 'defaults: file://defaults.yaml', + 'execution: file://execution.yaml', + '', + ].join('\n'), + ); + writeFileSync( + path.join(tempDir, 'targets.yaml'), + [ + '- id: codex-local', + ' provider: codex-app-server', + ' runtime:', + ' mode: profile', + ' home: .agentv/profiles/codex-local', + ' config:', + ' command: ["codex"]', + '', + ].join('\n'), + ); + writeFileSync( + path.join(tempDir, 'graders.yaml'), + ['- id: openai-grader', ' provider: openai', ' config: {}', ''].join('\n'), + ); + writeFileSync( + path.join(tempDir, 'tests.yaml'), + ['- id: smoke', ' input: Fix the failing test', ''].join('\n'), + ); + writeFileSync( + path.join(tempDir, 'defaults.yaml'), + ['target: codex-local', 'grader: openai-grader', ''].join('\n'), + ); + writeFileSync(path.join(tempDir, 'execution.yaml'), 'max_concurrency: 2\n'); + + await expect(loadComposableConfigGraph(splitPath)).resolves.toEqual( + await loadComposableConfigGraph(inlinePath), + ); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('rejects wrapped referenced field files', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-wrapped-')); + try { + const configPath = path.join(tempDir, 'config.yaml'); + writeFileSync(configPath, 'targets: file://targets.yaml\n'); + writeFileSync( + path.join(tempDir, 'targets.yaml'), + ['targets:', ' - id: codex-local', ' provider: codex-app-server', ''].join('\n'), + ); + + await expect(loadComposableConfigGraph(configPath)).rejects.toThrow(/wrapped in 'targets'/); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('validates command arrays, defaults, max_concurrency, and removed target fields', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-invalid-')); + try { + const invalidCases = [ + { + name: 'command', + yaml: [ + 'targets:', + ' - id: codex-local', + ' provider: codex-app-server', + ' runtime: host', + ' config:', + ' command: codex', + '', + ].join('\n'), + message: /config\.command/, + }, + { + name: 'default-target', + yaml: [ + 'targets:', + ' - id: codex-local', + ' provider: codex-app-server', + ' runtime: host', + ' config: {}', + 'defaults:', + ' target: missing', + '', + ].join('\n'), + message: /defaults\.target/, + }, + { + name: 'concurrency', + yaml: 'execution:\n max_concurrency: 0\n', + message: /max_concurrency/, + }, + { + name: 'execution-workers', + yaml: 'execution:\n workers: 2\n', + message: /execution\.workers/, + }, + { + name: 'legacy-label', + yaml: [ + 'targets:', + ' - label: codex-local', + ' provider: codex-app-server', + ' runtime: host', + ' config: {}', + '', + ].join('\n'), + message: /label/, + }, + { + name: 'bare-provider', + yaml: [ + 'targets:', + ' - id: codex-local', + ' provider: codex', + ' runtime: host', + ' config: {}', + '', + ].join('\n'), + message: /ambiguous/, + }, + { + name: 'target-workers', + yaml: [ + 'targets:', + ' - id: codex-local', + ' provider: codex-app-server', + ' runtime: host', + ' workers: 3', + ' config: {}', + '', + ].join('\n'), + message: /workers/, + }, + ]; + + for (const testCase of invalidCases) { + const configPath = path.join(tempDir, `${testCase.name}.yaml`); + writeFileSync(configPath, testCase.yaml); + await expect(loadComposableConfigGraph(configPath)).rejects.toThrow(testCase.message); + } + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('falls back to AGENTV_HOME/config.yaml when no project-local config exists', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-global-config-')); try { @@ -583,7 +823,7 @@ describe('extractTargetFromSuite', () => { it('rejects authored top-level execution blocks', () => { const suite: JsonObject = { execution: { target: 'my-target' } }; - expect(() => extractTargetFromSuite(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractTargetFromSuite(suite)).toThrow(/execution\.target/); }); }); @@ -629,8 +869,8 @@ describe('extractTargetsFromSuite and extractTargetRefsFromSuite', () => { it('reject top-level target arrays through execution', () => { const suite: JsonObject = { execution: { targets: ['copilot', 'claude'] } }; - expect(() => extractTargetsFromSuite(suite)).toThrow(/Top-level 'execution'/); - expect(() => extractTargetRefsFromSuite(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractTargetsFromSuite(suite)).toThrow(/execution\.targets/); + expect(() => extractTargetRefsFromSuite(suite)).toThrow(/execution\.targets/); }); }); @@ -672,7 +912,7 @@ describe('extractBudgetUsd', () => { it('rejects authored execution blocks', () => { const suite: JsonObject = { execution: { budget_usd: 10.0 } }; - expect(() => extractBudgetUsd(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractBudgetUsd(suite)).toThrow(/execution\.budget_usd/); }); }); @@ -687,6 +927,11 @@ describe('extractWorkersFromSuite', () => { expect(extractWorkersFromSuite(suite)).toBe(5); }); + it('parses valid execution.max_concurrency', () => { + const suite: JsonObject = { execution: { max_concurrency: 3 } }; + expect(extractWorkersFromSuite(suite)).toBe(3); + }); + it('returns undefined for invalid max_concurrency', () => { const suite: JsonObject = { evaluate_options: { max_concurrency: 0 } }; expect(extractWorkersFromSuite(suite)).toBeUndefined(); @@ -694,7 +939,7 @@ describe('extractWorkersFromSuite', () => { it('rejects authored execution blocks', () => { const suite: JsonObject = { execution: { workers: 5 } }; - expect(() => extractWorkersFromSuite(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractWorkersFromSuite(suite)).toThrow(/execution\.workers/); }); }); @@ -706,7 +951,7 @@ describe('extractFailOnError', () => { it('rejects authored execution blocks', () => { const suite: JsonObject = { execution: { fail_on_error: true } }; - expect(() => extractFailOnError(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractFailOnError(suite)).toThrow(/execution\.fail_on_error/); }); }); @@ -748,7 +993,7 @@ describe('extractThreshold', () => { it('rejects authored execution blocks', () => { const suite: JsonObject = { execution: { threshold: 0.8 } }; - expect(() => extractThreshold(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractThreshold(suite)).toThrow(/execution\.threshold/); }); }); @@ -766,17 +1011,13 @@ describe('parseExecutionDefaults', () => { expect(result?.verbose).toBe(true); }); - it('parses workers as an operator-side execution default', () => { - const result = parseExecutionDefaults({ workers: 4 }, '/test/config.yaml'); - expect(result?.workers).toBe(4); - }); - - it('ignores invalid workers defaults', () => { + it('rejects execution.workers defaults', () => { const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); try { - const result = parseExecutionDefaults({ workers: 0 }, '/test/config.yaml'); + const result = parseExecutionDefaults({ workers: 4 }, '/test/config.yaml'); expect(result).toBeUndefined(); expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('execution.workers')); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('execution.max_concurrency')); } finally { warnSpy.mockRestore(); } diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index e4ef6036a..56db4911f 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -36,7 +36,7 @@ describe('validateConfigFile', () => { await writeFile( filePath, `execution: - timeout: 30000 + max_concurrency: 3 `, ); @@ -46,6 +46,127 @@ describe('validateConfigFile', () => { expect(result.errors).toHaveLength(0); }); + it('rejects execution.workers in config graph execution policy', async () => { + const filePath = path.join(tempDir, 'config-execution-workers.yaml'); + await writeFile( + filePath, + `execution: + workers: 3 +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + message: expect.stringContaining('execution.workers'), + }), + ); + }); + + it('accepts composable config graph fields and direct file refs', async () => { + const graphDir = path.join(tempDir, 'composable-config'); + const filePath = path.join(graphDir, '.agentv', 'config.yaml'); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile( + filePath, + [ + 'targets: file://targets.yaml', + 'graders: file://graders.yaml', + 'tests: file://tests.yaml', + 'defaults: file://defaults.yaml', + 'execution: file://execution.yaml', + '', + ].join('\n'), + ); + await writeFile( + path.join(path.dirname(filePath), 'targets.yaml'), + [ + '- id: codex-local', + ' provider: codex-app-server', + ' runtime: host', + ' config:', + ' command: ["codex", "app-server"]', + '', + ].join('\n'), + ); + await writeFile( + path.join(path.dirname(filePath), 'graders.yaml'), + ['- id: openai-grader', ' provider: openai', ' config: {}', ''].join('\n'), + ); + await writeFile( + path.join(path.dirname(filePath), 'tests.yaml'), + ['- id: smoke', ' input: Fix the failing test', ''].join('\n'), + ); + await writeFile( + path.join(path.dirname(filePath), 'defaults.yaml'), + ['target: codex-local', 'grader: openai-grader', ''].join('\n'), + ); + await writeFile(path.join(path.dirname(filePath), 'execution.yaml'), 'max_concurrency: 3\n'); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('rejects removed composable config target fields and dashboard.app_name', async () => { + const filePath = path.join(tempDir, 'config-invalid-composable.yaml'); + await writeFile( + filePath, + [ + 'targets:', + ' - name: codex-local', + ' provider: codex', + ' runtime: host', + ' executable: codex', + ' config: {}', + 'dashboard:', + ' app_name: Custom AgentV', + '', + ].join('\n'), + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + severity: 'error', + message: expect.stringContaining('name'), + }), + expect.objectContaining({ + severity: 'error', + location: 'dashboard.app_name', + }), + ]), + ); + }); + + it('rejects wrapped referenced config field files', async () => { + const graphDir = path.join(tempDir, 'wrapped-composable-config'); + const filePath = path.join(graphDir, '.agentv', 'config.yaml'); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, 'targets: file://targets.yaml\n'); + await writeFile( + path.join(path.dirname(filePath), 'targets.yaml'), + ['targets:', ' - id: codex-local', ' provider: codex-app-server', ''].join('\n'), + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + message: expect.stringContaining("wrapped in 'targets'"), + }), + ); + }); + it('accepts results field without warnings', async () => { const filePath = path.join(tempDir, 'config-results.yaml'); await writeFile( diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index 43a7aa251..b4a3b3c85 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -50,7 +50,18 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); - it('rejects eval-level execution blocks', () => { + it('accepts eval-level execution.max_concurrency', () => { + const result = EvalFileSchema.safeParse({ + execution: { + max_concurrency: 2, + }, + tests: [baseTest], + }); + + expect(result.success).toBe(true); + }); + + it('rejects removed eval-level execution fields', () => { const result = EvalFileSchema.safeParse({ execution: { trials: { @@ -64,6 +75,33 @@ describe('EvalFileSchema input shorthand', () => { expect(result.success).toBe(false); }); + it('accepts shared composable graph fields in eval YAML', () => { + const result = EvalFileSchema.safeParse({ + targets: [ + { + id: 'codex-local', + provider: 'codex-app-server', + runtime: 'host', + config: { command: ['codex', 'app-server'] }, + }, + ], + graders: [ + { + id: 'openai-grader', + provider: 'openai', + config: { model: 'gpt-5-mini' }, + }, + ], + defaults: { + target: 'codex-local', + grader: 'openai-grader', + }, + tests: [baseTest], + }); + + expect(result.success).toBe(true); + }); + it('rejects removed top-level runs and early_exit controls', () => { const result = EvalFileSchema.safeParse({ target: 'codex', diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index d0bb9d27a..227bf7a73 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -104,6 +104,70 @@ tests: ).toBe(true); }); + it('validates composable execution.max_concurrency and defaults in eval YAML', async () => { + const filePath = path.join(tempDir, 'composable-eval-graph.yaml'); + await writeFile( + filePath, + `targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini +defaults: + target: codex-local + grader: openai-grader +execution: + max_concurrency: 3 +tests: + - id: local-case + input: "Hello" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('rejects removed top-level execution fields in eval YAML', async () => { + const filePath = path.join(tempDir, 'invalid-composable-execution.yaml'); + await writeFile( + filePath, + `execution: + target: claude + workers: 4 + max_concurrency: 3 +tests: + - id: local-case + input: "Hello" +`, + ); + + const result = await validateEvalFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'execution.target', + }), + ); + expect(result.errors).toContainEqual( + expect.objectContaining({ + severity: 'error', + location: 'execution.workers', + message: expect.stringContaining("Unsupported execution field 'workers'"), + }), + ); + }); + it('validates default_test.threshold', async () => { const filePath = path.join(tempDir, 'default-test-threshold.yaml'); await writeFile( @@ -808,8 +872,8 @@ tests: expect( result.errors.some( (error) => - error.location === 'execution' && - error.message.includes("Top-level 'execution' is not part of eval YAML"), + error.location === 'execution.target' && + error.message.includes("Unsupported execution field 'target'"), ), ).toBe(true); }); diff --git a/packages/core/test/projects.test.ts b/packages/core/test/projects.test.ts index 7c8d42f03..fe552b0f8 100644 --- a/packages/core/test/projects.test.ts +++ b/packages/core/test/projects.test.ts @@ -109,6 +109,45 @@ describe('projects registry', () => { expect(yamlOnDisk).toContain('projects:'); }); + it('loads global projects from a direct projects file reference', () => { + const registryPath = getProjectsRegistryPath(); + mkdirSync(path.dirname(registryPath), { recursive: true }); + writeFileSync(registryPath, 'projects: file://projects.yaml\n', 'utf-8'); + writeFileSync( + path.join(path.dirname(registryPath), 'projects.yaml'), + [ + '- id: split-project', + ' path: /srv/agentv/split-project', + ' added_at: "2026-01-01T00:00:00Z"', + ' last_opened_at: "2026-01-01T00:00:00Z"', + '', + ].join('\n'), + 'utf-8', + ); + + expect(loadProjectRegistry().projects).toEqual([ + { + id: 'split-project', + path: '/srv/agentv/split-project', + addedAt: '2026-01-01T00:00:00Z', + lastOpenedAt: '2026-01-01T00:00:00Z', + }, + ]); + }); + + it('rejects wrapped projects file references by returning an empty registry', () => { + const registryPath = getProjectsRegistryPath(); + mkdirSync(path.dirname(registryPath), { recursive: true }); + writeFileSync(registryPath, 'projects: file://projects.yaml\n', 'utf-8'); + writeFileSync( + path.join(path.dirname(registryPath), 'projects.yaml'), + ['projects:', ' - id: wrapped', ' path: /srv/agentv/wrapped', ''].join('\n'), + 'utf-8', + ); + + expect(loadProjectRegistry().projects).toEqual([]); + }); + it('round-trips flat source repo fields through YAML', () => { const registryPath = getProjectsRegistryPath(); mkdirSync(path.dirname(registryPath), { recursive: true }); diff --git a/skills-data/agentv-eval-writer/references/eval.schema.json b/skills-data/agentv-eval-writer/references/eval.schema.json index f92b7164a..d776f90ee 100644 --- a/skills-data/agentv-eval-writer/references/eval.schema.json +++ b/skills-data/agentv-eval-writer/references/eval.schema.json @@ -2917,6 +2917,58 @@ } ] }, + "graders": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "config": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["id", "provider"], + "additionalProperties": false + } + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "defaults": { + "anyOf": [ + { + "type": "object", + "properties": { + "target": { + "type": "string", + "minLength": 1 + }, + "grader": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + { + "type": "string", + "minLength": 1 + } + ] + }, "eval_cases": { "anyOf": [ { @@ -10801,7 +10853,23 @@ "not": {} }, "execution": { - "not": {} + "anyOf": [ + { + "type": "object", + "properties": { + "max_concurrency": { + "type": "integer", + "minimum": 1, + "maximum": 50 + } + }, + "additionalProperties": false + }, + { + "type": "string", + "minLength": 1 + } + ] }, "assert": { "type": "array", From de6e5165c65e7346518a32e7e020cc8e70653f47 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 16:42:42 +0200 Subject: [PATCH 02/10] feat(results): add target execution envelopes --- apps/cli/src/commands/eval/statistics.ts | 16 + apps/dashboard/src/components/ResultTable.tsx | 34 +- apps/dashboard/src/lib/types.ts | 45 ++ .../docs/next/reference/result-artifacts.mdx | 17 + packages/core/src/evaluation/orchestrator.ts | 37 +- packages/core/src/evaluation/providers/cli.ts | 577 ++++++++++++++++-- .../core/src/evaluation/providers/types.ts | 76 +++ .../core/src/evaluation/result-row-schema.ts | 4 + packages/core/src/evaluation/run-artifacts.ts | 255 +++++++- packages/core/src/evaluation/types.ts | 3 + .../test/evaluation/providers/cli.test.ts | 127 +++- .../target-execution-artifacts.test.ts | 108 ++++ 12 files changed, 1223 insertions(+), 76 deletions(-) create mode 100644 packages/core/test/evaluation/target-execution-artifacts.test.ts diff --git a/apps/cli/src/commands/eval/statistics.ts b/apps/cli/src/commands/eval/statistics.ts index 13d64f508..eba1e4c0a 100644 --- a/apps/cli/src/commands/eval/statistics.ts +++ b/apps/cli/src/commands/eval/statistics.ts @@ -22,6 +22,7 @@ export interface EvaluationSummary { readonly passedCount: number; readonly byFailureStage: Readonly>; readonly byFailureReason: Readonly>; + readonly byTargetErrorKind: Readonly>; } const HISTOGRAM_BREAKPOINTS = [0, 0.2, 0.4, 0.6, 0.8, 1]; @@ -112,6 +113,7 @@ export function calculateEvaluationSummary( passedCount: 0, byFailureStage: {}, byFailureReason: {}, + byTargetErrorKind: {}, }; } @@ -150,6 +152,7 @@ export function calculateEvaluationSummary( // Aggregate by failure stage and reason (execution errors only) const byFailureStage: Record = {}; const byFailureReason: Record = {}; + const byTargetErrorKind: Record = {}; for (const result of executionErrors) { if (result.failureStage) { byFailureStage[result.failureStage] = (byFailureStage[result.failureStage] ?? 0) + 1; @@ -158,6 +161,10 @@ export function calculateEvaluationSummary( byFailureReason[result.failureReasonCode] = (byFailureReason[result.failureReasonCode] ?? 0) + 1; } + const targetErrorKind = result.targetExecution?.errorKind; + if (targetErrorKind) { + byTargetErrorKind[targetErrorKind] = (byTargetErrorKind[targetErrorKind] ?? 0) + 1; + } } return { @@ -177,6 +184,7 @@ export function calculateEvaluationSummary( passedCount, byFailureStage, byFailureReason, + byTargetErrorKind, }; } @@ -297,6 +305,14 @@ export function formatEvaluationSummary( } } + const targetErrorEntries = Object.entries(summary.byTargetErrorKind); + if (targetErrorEntries.length > 0) { + lines.push('\nTarget runtime errors by kind:'); + for (const [kind, count] of targetErrorEntries) { + lines.push(` ${kind}: ${count}`); + } + } + return lines.join('\n'); } diff --git a/apps/dashboard/src/components/ResultTable.tsx b/apps/dashboard/src/components/ResultTable.tsx index ba6c85483..9c118a60b 100644 --- a/apps/dashboard/src/components/ResultTable.tsx +++ b/apps/dashboard/src/components/ResultTable.tsx @@ -615,6 +615,26 @@ function primaryTrialArtifactPath(trial: EvalCaseTrial): string | null { ); } +function formatTargetError( + error: string | undefined, + kind: string | undefined, +): string | undefined { + if (!error) return kind ? `Target error: ${kind}` : undefined; + return kind ? `[target:${kind}] ${error}` : error; +} + +function targetErrorKind(value: { + targetExecution?: { error_kind?: string; errorKind?: string }; + target_execution?: { error_kind?: string; errorKind?: string }; +}): string | undefined { + return ( + value.target_execution?.error_kind ?? + value.target_execution?.errorKind ?? + value.targetExecution?.error_kind ?? + value.targetExecution?.errorKind + ); +} + function TrialResultCell({ column, row, @@ -656,7 +676,12 @@ function TrialResultCell({ case 'review': return -; case 'error': - return ; + return ( + + ); default: return -; } @@ -840,7 +865,12 @@ function ResultCell({ case 'cost_tokens': return ; case 'error': - return ; + return ( + + ); default: return -; } diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index cf23b472f..5a6c71d5c 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -70,6 +70,41 @@ export interface TokenUsage { cached?: number; } +export interface TargetExecutionEnvelope { + schema_version?: 'agentv.target_execution.v1'; + status?: 'success' | 'error' | string; + target_id?: string; + provider_id?: string; + provider_kind?: string; + runtime_mode?: string; + runtimeMode?: string; + error_kind?: string; + errorKind?: string; + message?: string; + exit_code?: number | null; + exitCode?: number | null; + signal?: string | null; + duration_ms?: number; + durationMs?: number; + command?: { + argv?: string[]; + command_line?: string; + cwd?: string; + }; + artifacts?: { + target_execution_path?: string; + stdout_path?: string; + stderr_path?: string; + transcript_path?: string; + transcript_raw_path?: string; + summary_path?: string; + metrics_path?: string; + file_changes_path?: string; + output_path?: string; + answer_path?: string; + }; +} + export interface ScoreEntry { name?: string; type?: string; @@ -100,6 +135,11 @@ export interface EvalCaseTrial { scores?: ScoreEntry[]; assertions?: AssertionEntry[]; error?: string; + targetExecution?: TargetExecutionEnvelope; + target_execution?: TargetExecutionEnvelope; + target_execution_path?: string; + stdout_path?: string; + stderr_path?: string; execution_status?: string; cost_usd?: number; total_tokens?: number; @@ -228,6 +268,11 @@ export interface EvalResult { score: number; executionStatus?: string; error?: string; + targetExecution?: TargetExecutionEnvelope; + target_execution?: TargetExecutionEnvelope; + target_execution_path?: string; + stdout_path?: string; + stderr_path?: string; costUsd?: number; durationMs?: number; tokenUsage?: TokenUsage; diff --git a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx index 6594efaac..3196a4bbe 100644 --- a/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx +++ b/apps/web/src/content/docs/docs/next/reference/result-artifacts.mdx @@ -46,6 +46,9 @@ The default local layout is: result.json grading.json metrics.json + target-execution.json # optional target runtime envelope + stdout.txt # optional captured target stdout + stderr.txt # optional captured target stderr transcript.json transcript-raw.jsonl outputs/ @@ -55,6 +58,9 @@ The default local layout is: result.json grading.json metrics.json + target-execution.json + stdout.txt + stderr.txt transcript.json transcript-raw.jsonl outputs/ @@ -92,6 +98,8 @@ reserved for rebuildable local state and are skipped by run discovery. | `result.json` | Compact per-attempt manifest for one attempt directory, including AgentV `execution_status` and `verdict`. | Loading one attempt without scanning the whole run index. | | `grading.json` | Grader outputs, `assertion_results`, rubric evidence, execution-metric grader facts, and scoring provenance. | Explaining why a row passed or failed. | | `metrics.json` | Duration, token usage, cost, execution status, trajectory, and derived executor behavior such as tool calls, files touched, shell commands, errors, turns, and output sizes. | Dashboard behavior views, cost/latency reporting, metric-style graders, adapter projections, and lightweight analysis. | +| `target-execution.json` | Provider-neutral target runtime envelope, including command, cwd, timeout, exit code or signal, error kind, timestamps, log truncation metadata, and artifact paths. | Distinguishing target task failures, target crashes, timeouts, cancellation, malformed provider output, and sandbox/runner failures from AgentV orchestrator failures. | +| `stdout.txt` / `stderr.txt` | Captured target process logs when the runtime exposes them, with truncation metadata recorded in `target-execution.json`. | Debugging crashed, timed-out, cancelled, or malformed target runs without treating raw logs as the canonical row index. | | `outputs/file_changes.diff` | Full unified diff of workspace file changes when file changes are captured. | Human review and external artifact inspection; LLM and script graders still receive the same full diff through `file_changes`. | | `transcript.json` | AgentV-normalized transcript/timeline document with canonical `tool_name` values and `transcript_summary`. | Portable human review, transcript-aware graders, and tool-trajectory analysis. | | `transcript-raw.jsonl` | Native provider or harness evidence when available. | Parser debugging, forensic review, and preserving source bytes without making provider schemas public AgentV fields. | @@ -193,6 +201,15 @@ Example row: "summary_path": "refund-eligibility--4f9a7c2d1b6e/summary.json", "grading_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/grading.json", "metrics_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/metrics.json", + "target_execution_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/target-execution.json", + "stdout_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/stdout.txt", + "stderr_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/stderr.txt", + "target_execution": { + "schema_version": "agentv.target_execution.v1", + "status": "success", + "provider_kind": "cli", + "target_id": "codex-gpt5" + }, "transcript_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/transcript.json", "transcript_raw_path": "refund-eligibility--4f9a7c2d1b6e/sample-1/transcript-raw.jsonl", "transcript_summary": { diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 6899e6da1..2e38d7192 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -146,6 +146,14 @@ function extractProviderRawLogPath(response: ProviderResponse): string | undefin return trimmed.length > 0 ? trimmed : undefined; } +function targetExecutionFailureReasonCode(response: ProviderResponse): string { + const kind = response.targetExecution?.errorKind; + if (!kind || kind === 'agentv_orchestrator_failure') { + return 'provider_error'; + } + return `target_${kind}`; +} + function mergeMetadata( base: Record | undefined, overlay: JsonObject | Record | undefined, @@ -1691,18 +1699,22 @@ async function runBatchEvaluation(options: { }); if (providerError) { + const failureReasonCode = targetExecutionFailureReasonCode(providerResponse); result = { ...result, + targetExecution: providerResponse.targetExecution, trace: appendErrorEventToTrace(result.trace, providerError, { failure_stage: 'agent', - failure_reason_code: 'provider_error', + failure_reason_code: failureReasonCode, }), error: providerError, executionStatus: 'execution_error' as const, failureStage: 'agent' as const, - failureReasonCode: 'provider_error', + failureReasonCode: failureReasonCode, executionError: { message: providerError, stage: 'agent' as const }, }; + } else if (providerResponse.targetExecution) { + result = { ...result, targetExecution: providerResponse.targetExecution }; } } catch (error) { const errorResult = buildErrorResult( @@ -2269,19 +2281,25 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise 0) { + return message; + } + return targetExecution.errorKind ?? 'target execution failed'; + } + const raw = response.raw; if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { return undefined; diff --git a/packages/core/src/evaluation/providers/cli.ts b/packages/core/src/evaluation/providers/cli.ts index 4afbc60fc..408f8077f 100644 --- a/packages/core/src/evaluation/providers/cli.ts +++ b/packages/core/src/evaluation/providers/cli.ts @@ -1,8 +1,7 @@ -import { type ExecException, type ExecOptions, exec as execWithCallback } from 'node:child_process'; +import { spawn } from 'node:child_process'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { promisify } from 'node:util'; import { z } from 'zod'; @@ -16,7 +15,11 @@ import type { ProviderRequest, ProviderResponse, ProviderTokenUsage, + TargetExecutionEnvelope, + TargetExecutionErrorKind, + TargetExecutionLogCapture, } from './types.js'; +import { extractLastAssistantContent } from './types.js'; // --------------------------------------------------------------------------- // Zod Schemas for CLI Output Parsing @@ -70,6 +73,7 @@ const TokenUsageSchema = z.object({ */ const CliOutputSchema = z.object({ text: z.unknown().optional(), + error: z.string().optional(), output: z.array(MessageInputSchema).optional(), output_messages: z.array(MessageInputSchema).optional(), token_usage: TokenUsageSchema.optional(), @@ -149,7 +153,6 @@ function convertMessages( })); } -const execAsync = promisify(execWithCallback); const DEFAULT_MAX_BUFFER = 10 * 1024 * 1024; // 10 MB to accommodate verbose CLI output export interface CommandRunOptions { @@ -166,6 +169,7 @@ export interface CommandRunResult { readonly failed: boolean; readonly timedOut?: boolean; readonly signal?: NodeJS.Signals | null; + readonly spawnErrorCode?: string; } export type CommandRunner = ( @@ -177,43 +181,215 @@ async function defaultCommandRunner( command: string, options: CommandRunOptions, ): Promise { - const execOptions: ExecOptions = { - cwd: options.cwd, - env: options.env, - timeout: options.timeoutMs, - signal: options.signal, - maxBuffer: DEFAULT_MAX_BUFFER, - shell: process.platform === 'win32' ? 'powershell.exe' : undefined, - }; - - try { - const { stdout, stderr } = await execAsync(command, execOptions); + return new Promise((resolve) => { + const useWindowsShell = process.platform === 'win32'; + const shell = useWindowsShell ? 'powershell.exe' : '/bin/sh'; + const args = useWindowsShell ? ['-NoProfile', '-Command', command] : ['-lc', command]; + const child = spawn(shell, args, { + cwd: options.cwd, + env: options.env, + detached: !useWindowsShell, + windowsHide: true, + }); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + let cancelled = false; + let settled = false; + + const append = (current: string, chunk: Buffer) => { + if (Buffer.byteLength(current, 'utf8') >= DEFAULT_MAX_BUFFER) { + return current; + } + return `${current}${chunk.toString('utf8')}`; + }; - return { - stdout, - stderr, - exitCode: 0, - failed: false, - timedOut: false, - signal: null, + const terminate = (signal: NodeJS.Signals) => { + if (child.pid === undefined) { + return; + } + try { + if (useWindowsShell) { + child.kill(signal); + } else { + process.kill(-child.pid, signal); + } + } catch { + child.kill(signal); + } }; - } catch (error) { - const execError = error as ExecException & { - stdout?: string; - stderr?: string; - timedOut?: boolean; - killed?: boolean; + + const timeout = options.timeoutMs + ? setTimeout(() => { + timedOut = true; + terminate('SIGTERM'); + setTimeout(() => terminate('SIGKILL'), 2_000).unref?.(); + }, options.timeoutMs) + : undefined; + timeout?.unref?.(); + + const abort = () => { + cancelled = true; + terminate('SIGTERM'); + setTimeout(() => terminate('SIGKILL'), 2_000).unref?.(); }; + if (options.signal) { + if (options.signal.aborted) { + abort(); + } else { + options.signal.addEventListener('abort', abort, { once: true }); + } + } + + child.stdout?.on('data', (chunk: Buffer) => { + stdout = append(stdout, chunk); + }); + child.stderr?.on('data', (chunk: Buffer) => { + stderr = append(stderr, chunk); + }); + child.on('error', (error: NodeJS.ErrnoException) => { + if (settled) { + return; + } + settled = true; + if (timeout) clearTimeout(timeout); + if (options.signal) options.signal.removeEventListener('abort', abort); + resolve({ + stdout, + stderr: stderr || error.message, + exitCode: null, + failed: true, + timedOut, + signal: null, + spawnErrorCode: error.code, + }); + }); + + child.on('close', (code, signal) => { + if (settled) { + return; + } + settled = true; + if (timeout) clearTimeout(timeout); + if (options.signal) options.signal.removeEventListener('abort', abort); + resolve({ + stdout, + stderr, + exitCode: code, + failed: code !== 0 || signal !== null || timedOut || cancelled, + timedOut, + signal, + }); + }); + }); +} + +const INLINE_LOG_LIMIT_BYTES = 128 * 1024; + +function captureLog(text: string): TargetExecutionLogCapture { + const bytes = Buffer.byteLength(text, 'utf8'); + if (bytes <= INLINE_LOG_LIMIT_BYTES) { return { - stdout: execError.stdout ?? '', - stderr: execError.stderr ?? '', - exitCode: typeof execError.code === 'number' ? execError.code : null, - failed: true, - timedOut: execError.timedOut === true || execError.killed === true, - signal: execError.signal ?? null, + text, + truncated: false, + bytes, + storedBytes: bytes, }; } + + let stored = text; + while (Buffer.byteLength(stored, 'utf8') > INLINE_LOG_LIMIT_BYTES) { + stored = stored.slice(0, Math.max(0, stored.length - 1024)); + } + const storedBytes = Buffer.byteLength(stored, 'utf8'); + return { + text: stored, + truncated: true, + bytes, + storedBytes, + }; +} + +function commandEnvelopeBase(params: { + targetName: string; + providerId: string; + providerKind: string; + command: string; + cwd?: string; + timeoutMs?: number; + startedAt: number; + endedAt: number; + result?: CommandRunResult; +}): Omit { + const argv = + process.platform === 'win32' + ? ['powershell.exe', '-NoProfile', '-Command', params.command] + : ['/bin/sh', '-lc', params.command]; + return { + schemaVersion: 'agentv.target_execution.v1', + targetId: params.targetName, + providerId: params.providerId, + providerKind: params.providerKind, + runtimeMode: 'host', + command: { + argv, + commandLine: params.command, + cwd: params.cwd, + }, + timeoutMs: params.timeoutMs, + startedAt: new Date(params.startedAt).toISOString(), + endedAt: new Date(params.endedAt).toISOString(), + durationMs: params.endedAt - params.startedAt, + exitCode: params.result?.exitCode, + signal: params.result?.signal ?? null, + logs: { + stdout: captureLog(params.result?.stdout ?? ''), + stderr: captureLog(params.result?.stderr ?? ''), + }, + }; +} + +function classifyCommandFailure( + result: CommandRunResult, + signalAborted: boolean | undefined, +): TargetExecutionErrorKind { + if (signalAborted) { + return 'cancelled'; + } + if (result.timedOut) { + return 'timeout'; + } + if (result.spawnErrorCode) { + return 'spawn_failure'; + } + if (result.signal && result.exitCode === null) { + return 'signal_crash'; + } + return 'nonzero_exit'; +} + +function commandFailureMessage(result: CommandRunResult, errorKind: TargetExecutionErrorKind) { + if (errorKind === 'cancelled') { + return 'CLI provider request was aborted'; + } + if (errorKind === 'timeout') { + return 'CLI provider timed out'; + } + if (errorKind === 'spawn_failure') { + return ( + result.stderr.trim() || + result.stdout.trim() || + `CLI failed to spawn (${result.spawnErrorCode})` + ); + } + if (errorKind === 'signal_crash') { + return `CLI terminated by signal ${result.signal ?? 'unknown'}`; + } + const codeText = result.exitCode !== null ? result.exitCode : 'unknown'; + const detail = result.stderr.trim() || result.stdout.trim(); + return detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`; } export class CliProvider implements Provider { @@ -277,34 +453,127 @@ export class CliProvider implements Provider { const measuredDurationMs = Date.now() - startTime; if (result.failed || (result.exitCode ?? 0) !== 0) { - if (request.signal?.aborted) { - throw new Error('CLI provider request was aborted'); - } - if (result.timedOut) { - throw new Error( - `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`, - ); - } - const codeText = result.exitCode !== null ? result.exitCode : 'unknown'; - const detail = result.stderr.trim() || result.stdout.trim(); - const message = detail - ? `${detail} (exit code ${codeText})` - : `CLI exited with code ${codeText}`; - throw new Error(message); + const errorKind = classifyCommandFailure(result, request.signal?.aborted); + const baseMessage = commandFailureMessage(result, errorKind); + const message = + errorKind === 'timeout' + ? `${baseMessage}${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}` + : baseMessage; + return { + output: [{ role: 'assistant', content: `Error: ${message}` }], + durationMs: measuredDurationMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: renderedCommand, + cwd: effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: startTime, + endedAt: Date.now(), + result, + }), + status: 'error', + errorKind, + message, + transcript: { + messages: [{ role: 'assistant', content: `Error: ${message}` }], + finalOutput: `Error: ${message}`, + }, + details: { + outputFile: outputFilePath, + spawnErrorCode: result.spawnErrorCode, + }, + }, + raw: { + command: renderedCommand, + stderr: result.stderr, + stdout: result.stdout, + exitCode: result.exitCode, + signal: result.signal, + cwd: effectiveCwd, + outputFile: outputFilePath, + error: message, + }, + }; } // Read from output file and parse as JSON if possible - const responseContent = await this.readAndCleanupOutputFile(outputFilePath); + let responseContent: string; + try { + responseContent = await this.readAndCleanupOutputFile(outputFilePath); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + output: [{ role: 'assistant', content: `Error: ${message}` }], + durationMs: measuredDurationMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: renderedCommand, + cwd: effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: startTime, + endedAt: Date.now(), + result, + }), + status: 'error', + errorKind: 'malformed_output', + message, + transcript: { + messages: [{ role: 'assistant', content: `Error: ${message}` }], + finalOutput: `Error: ${message}`, + }, + details: { outputFile: outputFilePath }, + }, + raw: { + command: renderedCommand, + stderr: result.stderr, + stdout: result.stdout, + exitCode: result.exitCode ?? 0, + cwd: effectiveCwd, + outputFile: outputFilePath, + error: message, + }, + }; + } const parsed = this.parseOutputContent(responseContent); + const finalOutput = extractLastAssistantContent(parsed.output); + const targetTaskFailure = parsed.error?.trim(); return { output: parsed.output, tokenUsage: parsed.tokenUsage, costUsd: parsed.costUsd, durationMs: parsed.durationMs ?? measuredDurationMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: renderedCommand, + cwd: effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: startTime, + endedAt: Date.now(), + result, + }), + status: targetTaskFailure ? 'error' : 'success', + errorKind: targetTaskFailure ? 'target_task_failure' : undefined, + message: targetTaskFailure, + transcript: { + messages: parsed.output, + finalOutput, + }, + details: { outputFile: outputFilePath }, + }, raw: { command: renderedCommand, stderr: result.stderr, + stdout: result.stdout, exitCode: result.exitCode ?? 0, cwd: effectiveCwd, outputFile: outputFilePath, @@ -378,24 +647,77 @@ export class CliProvider implements Provider { const measuredDurationMs = Date.now() - startTime; if (result.failed || (result.exitCode ?? 0) !== 0) { - if (controller.signal.aborted) { - throw new Error('CLI provider request was aborted'); - } - if (result.timedOut) { - throw new Error( - `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`, - ); - } - const codeText = result.exitCode !== null ? result.exitCode : 'unknown'; - const detail = result.stderr.trim() || result.stdout.trim(); - const message = detail - ? `${detail} (exit code ${codeText})` - : `CLI exited with code ${codeText}`; - throw new Error(message); + const errorKind = classifyCommandFailure(result, controller.signal.aborted); + const baseMessage = commandFailureMessage(result, errorKind); + const message = + errorKind === 'timeout' + ? `${baseMessage}${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}` + : baseMessage; + return requests.map((request) => + this.buildBatchErrorResponse({ + request, + renderedCommand, + effectiveCwd, + outputFilePath, + result, + startedAt: startTime, + durationMs: measuredDurationMs, + perRequestFallbackMs: Math.round(measuredDurationMs / requests.length), + errorKind, + message, + }), + ); } - const responseContent = await this.readAndCleanupOutputFile(outputFilePath); - const recordsById = this.parseJsonlBatchOutput(responseContent); + let responseContent: string; + try { + responseContent = await this.readAndCleanupOutputFile(outputFilePath); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return requests.map((request) => + this.buildBatchErrorResponse({ + request, + renderedCommand, + effectiveCwd, + outputFilePath, + result, + startedAt: startTime, + durationMs: measuredDurationMs, + perRequestFallbackMs: Math.round(measuredDurationMs / requests.length), + errorKind: 'malformed_output', + message, + }), + ); + } + let recordsById: Map< + string, + { + output: readonly Message[]; + error?: string; + tokenUsage?: ProviderTokenUsage; + costUsd?: number; + durationMs?: number; + } + >; + try { + recordsById = this.parseJsonlBatchOutput(responseContent); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return requests.map((request) => + this.buildBatchErrorResponse({ + request, + renderedCommand, + effectiveCwd, + outputFilePath, + result, + startedAt: startTime, + durationMs: measuredDurationMs, + perRequestFallbackMs: Math.round(measuredDurationMs / requests.length), + errorKind: 'malformed_output', + message, + }), + ); + } // Calculate per-request fallback duration (total time / number of requests) const perRequestFallbackMs = Math.round(measuredDurationMs / requests.length); @@ -406,9 +728,29 @@ export class CliProvider implements Provider { return { output: [], durationMs: perRequestFallbackMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: renderedCommand, + cwd: effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: startTime, + endedAt: Date.now(), + result, + }), + status: 'success', + transcript: { + messages: [], + finalOutput: '', + }, + details: { outputFile: outputFilePath }, + }, raw: { command: renderedCommand, stderr: result.stderr, + stdout: result.stdout, exitCode: result.exitCode ?? 0, cwd: effectiveCwd, outputFile: outputFilePath, @@ -427,9 +769,31 @@ export class CliProvider implements Provider { return { output: [{ role: 'assistant', content: `Error: ${errorMessage}` }], durationMs: perRequestFallbackMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: renderedCommand, + cwd: effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: startTime, + endedAt: Date.now(), + result, + }), + status: 'error', + errorKind: 'malformed_output', + message: errorMessage, + transcript: { + messages: [{ role: 'assistant', content: `Error: ${errorMessage}` }], + finalOutput: `Error: ${errorMessage}`, + }, + details: { outputFile: outputFilePath, recordId: evalCaseId }, + }, raw: { command: renderedCommand, stderr: result.stderr, + stdout: result.stdout, exitCode: result.exitCode ?? 0, cwd: effectiveCwd, outputFile: outputFilePath, @@ -443,9 +807,31 @@ export class CliProvider implements Provider { tokenUsage: parsed.tokenUsage, costUsd: parsed.costUsd, durationMs: parsed.durationMs ?? perRequestFallbackMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: renderedCommand, + cwd: effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: startTime, + endedAt: Date.now(), + result, + }), + status: parsed.error ? 'error' : 'success', + errorKind: parsed.error ? 'target_task_failure' : undefined, + message: parsed.error, + transcript: { + messages: parsed.output, + finalOutput: extractLastAssistantContent(parsed.output), + }, + details: { outputFile: outputFilePath, recordId: evalCaseId }, + }, raw: { command: renderedCommand, stderr: result.stderr, + stdout: result.stdout, exitCode: result.exitCode ?? 0, cwd: effectiveCwd, outputFile: outputFilePath, @@ -473,6 +859,7 @@ export class CliProvider implements Provider { */ private parseOutputContent(content: string): { output: readonly Message[]; + error?: string; tokenUsage?: ProviderTokenUsage; costUsd?: number; durationMs?: number; @@ -504,6 +891,7 @@ export class CliProvider implements Provider { if (output && output.length > 0) { return { output, + error: obj.error, tokenUsage: obj.token_usage, costUsd: metrics.costUsd, durationMs: metrics.durationMs, @@ -515,6 +903,7 @@ export class CliProvider implements Provider { const text = typeof obj.text === 'string' ? obj.text : String(obj.text); return { output: [{ role: 'assistant', content: text }], + error: obj.error, tokenUsage: obj.token_usage, costUsd: metrics.costUsd, durationMs: metrics.durationMs, @@ -522,13 +911,14 @@ export class CliProvider implements Provider { } // No output or text, treat original content as plain text - return { output: [{ role: 'assistant', content }] }; + return { output: [{ role: 'assistant', content }], error: obj.error }; } private parseJsonlBatchOutput(content: string): Map< string, { output: readonly Message[]; + error?: string; tokenUsage?: ProviderTokenUsage; costUsd?: number; durationMs?: number; @@ -538,6 +928,7 @@ export class CliProvider implements Provider { string, { output: readonly Message[]; + error?: string; tokenUsage?: ProviderTokenUsage; costUsd?: number; durationMs?: number; @@ -595,6 +986,7 @@ export class CliProvider implements Provider { records.set(obj.id, { output: finalMessages, + error: obj.error, tokenUsage: obj.token_usage, costUsd: metrics.costUsd, durationMs: metrics.durationMs, @@ -604,6 +996,61 @@ export class CliProvider implements Provider { return records; } + private buildBatchErrorResponse(params: { + readonly request: ProviderRequest; + readonly renderedCommand: string; + readonly effectiveCwd?: string; + readonly outputFilePath: string; + readonly result: CommandRunResult; + readonly startedAt: number; + readonly durationMs: number; + readonly perRequestFallbackMs: number; + readonly errorKind: TargetExecutionErrorKind; + readonly message: string; + }): ProviderResponse { + const content = `Error: ${params.message}`; + return { + output: [{ role: 'assistant', content }], + durationMs: params.perRequestFallbackMs, + targetExecution: { + ...commandEnvelopeBase({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + command: params.renderedCommand, + cwd: params.effectiveCwd, + timeoutMs: this.config.timeoutMs, + startedAt: params.startedAt, + endedAt: params.startedAt + params.durationMs, + result: params.result, + }), + status: 'error', + errorKind: params.errorKind, + message: params.message, + transcript: { + messages: [{ role: 'assistant', content }], + finalOutput: content, + }, + details: { + outputFile: params.outputFilePath, + recordId: params.request.evalCaseId, + spawnErrorCode: params.result.spawnErrorCode, + }, + }, + raw: { + command: params.renderedCommand, + stderr: params.result.stderr, + stdout: params.result.stdout, + exitCode: params.result.exitCode, + signal: params.result.signal, + cwd: params.effectiveCwd, + outputFile: params.outputFilePath, + recordId: params.request.evalCaseId, + error: params.message, + }, + }; + } + private async readAndCleanupOutputFile(filePath: string): Promise { try { const content = await readTextFile(filePath); diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 82642bb38..8186ff112 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -233,6 +233,81 @@ export interface Message { readonly tokenUsage?: ProviderTokenUsage; } +export type TargetExecutionErrorKind = + | 'target_task_failure' + | 'spawn_failure' + | 'nonzero_exit' + | 'signal_crash' + | 'timeout' + | 'cancelled' + | 'malformed_output' + | 'sandbox_infra_failure' + | 'agentv_orchestrator_failure'; + +export interface TargetExecutionLogCapture { + readonly text: string; + readonly truncated: boolean; + readonly bytes: number; + readonly storedBytes: number; +} + +export interface TargetExecutionCommand { + readonly argv?: readonly string[]; + readonly commandLine?: string; + readonly cwd?: string; +} + +export interface TargetExecutionArtifacts { + readonly targetExecutionPath?: string; + readonly stdoutPath?: string; + readonly stderrPath?: string; + readonly transcriptPath?: string; + readonly transcriptRawPath?: string; + readonly summaryPath?: string; + readonly metricsPath?: string; + readonly fileChangesPath?: string; + readonly outputPath?: string; + readonly answerPath?: string; +} + +/** + * Provider-neutral target runtime envelope. Providers report target/process + * outcomes here so AgentV can serialize target crashes, timeouts, malformed + * protocol output, and partial transcripts without treating them as AgentV + * orchestrator failures. Provider-specific detail belongs in `details`. + */ +export interface TargetExecutionEnvelope { + readonly schemaVersion: 'agentv.target_execution.v1'; + readonly status: 'success' | 'error'; + readonly targetId: string; + readonly providerId: string; + readonly providerKind: string; + readonly runtimeMode?: 'host' | 'profile' | 'sandbox' | string; + readonly command?: TargetExecutionCommand; + readonly timeoutMs?: number; + readonly startedAt: string; + readonly endedAt: string; + readonly durationMs: number; + readonly exitCode?: number | null; + readonly signal?: string | null; + readonly errorKind?: TargetExecutionErrorKind; + readonly message?: string; + readonly logs?: { + readonly stdout?: TargetExecutionLogCapture; + readonly stderr?: TargetExecutionLogCapture; + }; + readonly transcript?: { + readonly messages?: readonly Message[]; + readonly finalOutput?: string; + }; + readonly fileChanges?: { + readonly available: boolean; + readonly summary?: string; + }; + readonly artifacts?: TargetExecutionArtifacts; + readonly details?: Record; +} + /** * Token usage metrics reported by provider. */ @@ -261,6 +336,7 @@ export interface ProviderStepInfo { export interface ProviderResponse { readonly raw?: unknown; readonly usage?: JsonObject; + readonly targetExecution?: TargetExecutionEnvelope; /** Output messages from agent execution (primary source for tool trajectory) */ readonly output?: readonly Message[]; /** Token usage metrics (optional) */ diff --git a/packages/core/src/evaluation/result-row-schema.ts b/packages/core/src/evaluation/result-row-schema.ts index 5179601d2..a2ed37261 100644 --- a/packages/core/src/evaluation/result-row-schema.ts +++ b/packages/core/src/evaluation/result-row-schema.ts @@ -39,6 +39,8 @@ const RESULT_ROW_ALIASES = { responsePath: 'response_path', startTime: 'start_time', summaryPath: 'summary_path', + targetExecution: 'target_execution', + targetExecutionPath: 'target_execution_path', targetsPath: 'targets_path', taskDir: 'task_dir', testDir: 'test_dir', @@ -50,6 +52,8 @@ const RESULT_ROW_ALIASES = { transcriptRawPath: 'transcript_raw_path', transcriptSummary: 'transcript_summary', workspacePath: 'workspace_path', + stdoutPath: 'stdout_path', + stderrPath: 'stderr_path', } as const; const NEW_SNAKE_CASE_ONLY_FIELDS = { diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index bb26a6ae4..b5a30f1db 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -17,7 +17,11 @@ import { traceEnvelopeToNormalizedTranscriptJson, traceEnvelopeToTranscriptJsonLines, } from '../import/types.js'; -import { parseEvaluationResultBoundary, toCamelCaseDeep } from './case-conversion.js'; +import { + parseEvaluationResultBoundary, + toCamelCaseDeep, + toSnakeCaseDeep, +} from './case-conversion.js'; import type { ExperimentArtifactMetadata } from './experiment.js'; import { type ExternalTraceMetadataWire, @@ -35,7 +39,11 @@ import { toProjectionIdentityIssueWire, toProjectionIdentityWire, } from './projection-identity.js'; -import type { Message } from './providers/types.js'; +import type { + Message, + TargetExecutionArtifacts, + TargetExecutionEnvelope, +} from './providers/types.js'; import { extractLastAssistantContent } from './providers/types.js'; import { CANONICAL_FILE_CHANGES_ARTIFACT_PATH, @@ -65,6 +73,17 @@ export const RUN_INTERNAL_DIRNAME = '.internal'; export const CROSS_RUN_INDEX_DIRNAME = '.indexes'; export const CROSS_RUN_RUNS_INDEX_FILENAME = 'runs.jsonl'; export const CROSS_RUN_CASES_INDEX_FILENAME = 'cases.jsonl'; +const TARGET_EXECUTION_ARTIFACT_PATH = 'target-execution.json'; +const TARGET_STDOUT_ARTIFACT_PATH = 'stdout.txt'; +const TARGET_STDERR_ARTIFACT_PATH = 'stderr.txt'; + +type TargetExecutionWire = Record; + +function toTargetExecutionWire( + envelope: TargetExecutionEnvelope | undefined, +): TargetExecutionWire | undefined { + return envelope ? (toSnakeCaseDeep(envelope) as TargetExecutionWire) : undefined; +} export function runInternalPath(runDir: string, filename: string): string { return path.join(runDir, RUN_INTERNAL_DIRNAME, filename); @@ -535,6 +554,7 @@ export interface RunSummaryArtifact { } >; readonly per_grader_summary?: Record; + readonly target_error_summary?: Record; readonly metrics: TimingArtifact; readonly notes: readonly string[]; } @@ -587,6 +607,10 @@ export interface IndexArtifactEntry { readonly error?: string; readonly failure_stage?: string; readonly failure_reason_code?: string; + readonly target_execution?: TargetExecutionWire; + readonly target_execution_path?: string; + readonly stdout_path?: string; + readonly stderr_path?: string; readonly workspace_path?: string; readonly result_dir?: string; readonly grading_path?: string; @@ -648,6 +672,10 @@ export interface AgentVRunResultArtifact { readonly verdict: TrialResult['verdict']; readonly sample_index?: number; readonly retry_index?: number; + readonly target_execution?: TargetExecutionWire; + readonly target_execution_path?: string; + readonly stdout_path?: string; + readonly stderr_path?: string; readonly duration_ms?: number; readonly duration_seconds: number; readonly model: string; @@ -1111,6 +1139,10 @@ function buildAgentVRunResultArtifact(params: { readonly hasTranscript: boolean; readonly hasOutput: boolean; readonly hasFileChanges: boolean; + readonly targetExecution?: TargetExecutionEnvelope; + readonly targetExecutionPath?: string; + readonly stdoutPath?: string; + readonly stderrPath?: string; }): AgentVRunResultArtifact { const metrics = params.metricsArtifact.metrics; const fileChangesPath = params.hasFileChanges @@ -1121,6 +1153,12 @@ function buildAgentVRunResultArtifact(params: { verdict: params.trial.verdict, sample_index: params.result.sampleIndex, retry_index: params.result.retryIndex, + target_execution: toTargetExecutionWire(params.targetExecution), + target_execution_path: params.targetExecutionPath + ? `./${params.targetExecutionPath}` + : undefined, + stdout_path: params.stdoutPath ? `./${params.stdoutPath}` : undefined, + stderr_path: params.stderrPath ? `./${params.stderrPath}` : undefined, duration_ms: resultDurationMs(params.result), duration_seconds: resultDurationSeconds(params.result), model: params.result.target ?? 'unknown', @@ -1174,6 +1212,74 @@ function materializedRunTrials(result: EvaluationResult): readonly TrialResult[] return persisted.length > 0 ? persisted : [singleRunTrial(result)]; } +function withTargetExecutionArtifacts( + envelope: TargetExecutionEnvelope | undefined, + artifacts: TargetExecutionArtifacts, +): TargetExecutionEnvelope | undefined { + if (!envelope) { + return undefined; + } + return { + ...envelope, + artifacts: { + ...(envelope.artifacts ?? {}), + ...(dropUndefined( + artifacts as unknown as Record, + ) as TargetExecutionArtifacts), + }, + }; +} + +async function writeTargetExecutionArtifacts(params: { + readonly result: EvaluationResult; + readonly sampleDir: string; + readonly hasTranscript: boolean; + readonly hasOutput: boolean; + readonly hasFileChanges: boolean; +}): Promise<{ + readonly targetExecution?: TargetExecutionEnvelope; + readonly targetExecutionPath?: string; + readonly stdoutPath?: string; + readonly stderrPath?: string; +}> { + const envelope = params.result.targetExecution; + if (!envelope) { + return {}; + } + + const stdoutPath = path.join(params.sampleDir, TARGET_STDOUT_ARTIFACT_PATH); + const stderrPath = path.join(params.sampleDir, TARGET_STDERR_ARTIFACT_PATH); + const targetExecutionPath = path.join(params.sampleDir, TARGET_EXECUTION_ARTIFACT_PATH); + const artifactPaths = { + targetExecutionPath: TARGET_EXECUTION_ARTIFACT_PATH, + stdoutPath: TARGET_STDOUT_ARTIFACT_PATH, + stderrPath: TARGET_STDERR_ARTIFACT_PATH, + transcriptPath: params.hasTranscript ? CANONICAL_TRANSCRIPT_ARTIFACT_PATH : undefined, + transcriptRawPath: params.hasTranscript ? 'transcript-raw.jsonl' : undefined, + summaryPath: RUN_SUMMARY_FILENAME, + metricsPath: CANONICAL_METRICS_ARTIFACT_PATH, + fileChangesPath: params.hasFileChanges ? CANONICAL_FILE_CHANGES_ARTIFACT_PATH : undefined, + outputPath: params.hasOutput ? 'outputs/answer.md' : undefined, + answerPath: params.hasOutput ? 'outputs/answer.md' : undefined, + }; + const targetExecution = withTargetExecutionArtifacts(envelope, artifactPaths); + + await writeFile(stdoutPath, envelope.logs?.stdout?.text ?? '', 'utf8'); + await writeFile(stderrPath, envelope.logs?.stderr?.text ?? '', 'utf8'); + await writeFile( + targetExecutionPath, + `${JSON.stringify(toTargetExecutionWire(targetExecution), null, 2)}\n`, + 'utf8', + ); + + return { + targetExecution, + targetExecutionPath, + stdoutPath, + stderrPath, + }; +} + async function writeTrialRunArtifacts(params: { readonly trial: TrialResult; readonly parentTestDir: string; @@ -1232,6 +1338,13 @@ async function writeTrialRunArtifacts(params: { await writeNormalizedTranscriptJson(transcriptPath, envelope, result); await writeRawTranscriptJsonl(transcriptRawPath, result, envelope); } + const targetExecutionArtifacts = await writeTargetExecutionArtifacts({ + result, + sampleDir: runDir, + hasTranscript, + hasOutput: result.output.length > 0, + hasFileChanges: result.fileChanges !== undefined && result.fileChanges.length > 0, + }); const metricsArtifact = await writeMetricsArtifact({ filePath: metricsPath, result, @@ -1253,6 +1366,12 @@ async function writeTrialRunArtifacts(params: { hasTranscript, hasOutput: result.output.length > 0, hasFileChanges: result.fileChanges !== undefined && result.fileChanges.length > 0, + targetExecution: targetExecutionArtifacts.targetExecution, + targetExecutionPath: targetExecutionArtifacts.targetExecutionPath + ? TARGET_EXECUTION_ARTIFACT_PATH + : undefined, + stdoutPath: targetExecutionArtifacts.stdoutPath ? TARGET_STDOUT_ARTIFACT_PATH : undefined, + stderrPath: targetExecutionArtifacts.stderrPath ? TARGET_STDERR_ARTIFACT_PATH : undefined, }), null, 2, @@ -1606,11 +1725,21 @@ export function buildRunSummaryArtifact( } const errorCount = results.filter((r) => r.executionStatus === 'execution_error').length; + const targetErrorSummary: Record = {}; + for (const result of results) { + const kind = result.targetExecution?.errorKind; + if (kind) { + targetErrorSummary[kind] = (targetErrorSummary[kind] ?? 0) + 1; + } + } if (errorCount > 0) { notes.push( `${errorCount} test(s) had execution errors and are excluded from quality pass_rate`, ); } + if (Object.keys(targetErrorSummary).length > 0) { + notes.push('Target runtime errors are reported separately from AgentV orchestrator errors'); + } if (results.length === 0) { notes.push('No results to summarize'); } @@ -1757,6 +1886,8 @@ export function buildRunSummaryArtifact( }, run_summary: runSummary, per_grader_summary: perEvaluatorSummary, + target_error_summary: + Object.keys(targetErrorSummary).length > 0 ? targetErrorSummary : undefined, metrics: runMetrics, notes, }; @@ -2047,6 +2178,9 @@ export function buildIndexArtifactEntry( transcriptRawPath?: string; metricsPath?: string; fileChangesPath?: string; + targetExecutionPath?: string; + stdoutPath?: string; + stderrPath?: string; artifactPointers?: ResultArtifactPointersWire; rawProviderLogPath?: string; extraIndexFields?: AdditionalResultIndexFields; @@ -2055,6 +2189,39 @@ export function buildIndexArtifactEntry( duplicatePolicy?: ExportDuplicatePolicy; }, ): IndexArtifactEntry { + const targetExecution = withTargetExecutionArtifacts(result.targetExecution, { + targetExecutionPath: options.targetExecutionPath + ? toRelativeArtifactPath(options.outputDir, options.targetExecutionPath) + : undefined, + stdoutPath: options.stdoutPath + ? toRelativeArtifactPath(options.outputDir, options.stdoutPath) + : undefined, + stderrPath: options.stderrPath + ? toRelativeArtifactPath(options.outputDir, options.stderrPath) + : undefined, + transcriptPath: options.transcriptPath + ? toRelativeArtifactPath(options.outputDir, options.transcriptPath) + : undefined, + transcriptRawPath: options.transcriptRawPath + ? toRelativeArtifactPath(options.outputDir, options.transcriptRawPath) + : undefined, + summaryPath: options.summaryPath + ? toRelativeArtifactPath(options.outputDir, options.summaryPath) + : undefined, + metricsPath: options.metricsPath + ? toRelativeArtifactPath(options.outputDir, options.metricsPath) + : undefined, + fileChangesPath: options.fileChangesPath + ? toRelativeArtifactPath(options.outputDir, options.fileChangesPath) + : undefined, + outputPath: options.outputPath + ? toRelativeArtifactPath(options.outputDir, options.outputPath) + : undefined, + answerPath: options.answerPath + ? toRelativeArtifactPath(options.outputDir, options.answerPath) + : undefined, + }); + return { timestamp: result.timestamp, test_id: result.testId ?? 'unknown', @@ -2079,6 +2246,16 @@ export function buildIndexArtifactEntry( error: result.error, failure_stage: result.failureStage, failure_reason_code: result.failureReasonCode, + target_execution: toTargetExecutionWire(targetExecution), + target_execution_path: options.targetExecutionPath + ? toRelativeArtifactPath(options.outputDir, options.targetExecutionPath) + : undefined, + stdout_path: options.stdoutPath + ? toRelativeArtifactPath(options.outputDir, options.stdoutPath) + : undefined, + stderr_path: options.stderrPath + ? toRelativeArtifactPath(options.outputDir, options.stderrPath) + : undefined, workspace_path: result.workspacePath, result_dir: options.resultDir ? toRelativeArtifactPath(options.outputDir, options.resultDir) @@ -2146,6 +2323,37 @@ export function buildResultIndexArtifact( const hasTranscript = resultHasExecutionTraceTranscript(result); const isSingleRun = !hasPersistedTrialRuns(result); const singleRunDir = path.posix.join(artifactSubdir, sampleDirName(0)); + const targetExecution = result.targetExecution + ? withTargetExecutionArtifacts(result.targetExecution, { + targetExecutionPath: path.posix.join(singleRunDir, TARGET_EXECUTION_ARTIFACT_PATH), + stdoutPath: path.posix.join(singleRunDir, TARGET_STDOUT_ARTIFACT_PATH), + stderrPath: path.posix.join(singleRunDir, TARGET_STDERR_ARTIFACT_PATH), + transcriptPath: + isSingleRun && hasTranscript + ? path.posix.join(singleRunDir, CANONICAL_TRANSCRIPT_ARTIFACT_PATH) + : undefined, + transcriptRawPath: + isSingleRun && hasTranscript + ? path.posix.join(singleRunDir, 'transcript-raw.jsonl') + : undefined, + summaryPath: path.posix.join(artifactSubdir, RUN_SUMMARY_FILENAME), + metricsPath: isSingleRun + ? path.posix.join(singleRunDir, CANONICAL_METRICS_ARTIFACT_PATH) + : undefined, + fileChangesPath: + isSingleRun && hasFileChanges + ? path.posix.join(singleRunDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) + : undefined, + outputPath: + isSingleRun && hasAnswer + ? path.posix.join(singleRunDir, 'outputs', 'answer.md') + : undefined, + answerPath: + isSingleRun && hasAnswer + ? path.posix.join(singleRunDir, 'outputs', 'answer.md') + : undefined, + }) + : undefined; return { timestamp: result.timestamp, @@ -2171,6 +2379,16 @@ export function buildResultIndexArtifact( error: result.error, failure_stage: result.failureStage, failure_reason_code: result.failureReasonCode, + target_execution: toTargetExecutionWire(targetExecution), + target_execution_path: result.targetExecution + ? path.posix.join(singleRunDir, TARGET_EXECUTION_ARTIFACT_PATH) + : undefined, + stdout_path: result.targetExecution + ? path.posix.join(singleRunDir, TARGET_STDOUT_ARTIFACT_PATH) + : undefined, + stderr_path: result.targetExecution + ? path.posix.join(singleRunDir, TARGET_STDERR_ARTIFACT_PATH) + : undefined, workspace_path: result.workspacePath, result_dir: artifactSubdir, summary_path: path.posix.join(artifactSubdir, RUN_SUMMARY_FILENAME), @@ -2835,6 +3053,18 @@ export async function writePerTestArtifacts( isSingleRun && result.fileChanges ? path.join(singleRunDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) : undefined; + const singleTargetExecutionPath = + isSingleRun && result.targetExecution + ? path.join(singleRunDir, TARGET_EXECUTION_ARTIFACT_PATH) + : undefined; + const singleStdoutPath = + isSingleRun && result.targetExecution + ? path.join(singleRunDir, TARGET_STDOUT_ARTIFACT_PATH) + : undefined; + const singleStderrPath = + isSingleRun && result.targetExecution + ? path.join(singleRunDir, TARGET_STDERR_ARTIFACT_PATH) + : undefined; const extraIndexFields = await collectAdditionalIndexFields( result, @@ -2856,6 +3086,9 @@ export async function writePerTestArtifacts( transcriptPath: singleTranscriptPath, transcriptRawPath: singleTranscriptRawPath, fileChangesPath: singleFileChangesPath, + targetExecutionPath: singleTargetExecutionPath, + stdoutPath: singleStdoutPath, + stderrPath: singleStderrPath, extraIndexFields, runtimeSource: options?.runtimeSource, projectionIdentity, @@ -2949,6 +3182,18 @@ export async function writeArtifactsFromResults( isSingleRun && result.fileChanges ? path.join(singleRunDir, CANONICAL_FILE_CHANGES_ARTIFACT_PATH) : undefined; + const singleTargetExecutionPath = + isSingleRun && result.targetExecution + ? path.join(singleRunDir, TARGET_EXECUTION_ARTIFACT_PATH) + : undefined; + const singleStdoutPath = + isSingleRun && result.targetExecution + ? path.join(singleRunDir, TARGET_STDOUT_ARTIFACT_PATH) + : undefined; + const singleStderrPath = + isSingleRun && result.targetExecution + ? path.join(singleRunDir, TARGET_STDERR_ARTIFACT_PATH) + : undefined; return { result, testDir, @@ -2961,6 +3206,9 @@ export async function writeArtifactsFromResults( singleGradingPath, singleMetricsPath, singleFileChangesPath, + singleTargetExecutionPath, + singleStdoutPath, + singleStderrPath, identityId, }; }); @@ -3035,6 +3283,9 @@ export async function writeArtifactsFromResults( transcriptPath: plan.singleTranscriptPath, transcriptRawPath: plan.singleTranscriptRawPath, fileChangesPath: plan.singleFileChangesPath, + targetExecutionPath: plan.singleTargetExecutionPath, + stdoutPath: plan.singleStdoutPath, + stderrPath: plan.singleStderrPath, extraIndexFields, runtimeSource: options?.runtimeSource, projectionIdentity: plan.projectionIdentity, diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 3d8eddd81..4bf71e5c4 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -1,3 +1,4 @@ +import type { TargetExecutionEnvelope } from './providers/types.js'; import type { TokenUsage, ToolTrajectoryGraderConfig, Trace } from './trace.js'; /** A single assertion verdict with optional evidence. */ @@ -1305,6 +1306,8 @@ export interface EvaluationResult { * as raw, non-canonical evidence. */ readonly rawProviderLogPath?: string; + /** Structured target runtime outcome for process/protocol-backed providers. */ + readonly targetExecution?: TargetExecutionEnvelope; /** Path to the temporary workspace directory (included on failure for debugging) */ readonly workspacePath?: string; /** Input messages sent to the agent. Always Message[] for consistent shape with output. */ diff --git a/packages/core/test/evaluation/providers/cli.test.ts b/packages/core/test/evaluation/providers/cli.test.ts index 0d2958a84..af96b1b4d 100644 --- a/packages/core/test/evaluation/providers/cli.test.ts +++ b/packages/core/test/evaluation/providers/cli.test.ts @@ -63,6 +63,9 @@ describe('CliProvider', () => { expect(runner).toHaveBeenCalledTimes(1); expect(extractLastAssistantContent(response.output)).toContain('Test response from CLI'); expect(response.raw && (response.raw as Record).command).toBeDefined(); + expect(response.targetExecution?.status).toBe('success'); + expect(response.targetExecution?.targetId).toBe('cli-target'); + expect(response.targetExecution?.logs?.stdout?.text).toContain('agent-cli run'); const command = runner.mock.calls[0]?.[0] as string; expect(command).toContain('--file'); expect(command).toContain('Hello world'); @@ -107,7 +110,7 @@ describe('CliProvider', () => { expect(command).toContain('.prompt.txt'); }); - it('throws on non-zero exit codes with stderr context', async () => { + it('returns target execution envelopes for non-zero exit codes', async () => { const runner = mock( async (_command, _options): Promise => ({ stdout: '', @@ -119,7 +122,12 @@ describe('CliProvider', () => { const provider = new CliProvider('cli-target', baseConfig, runner); - await expect(provider.invoke(baseRequest)).rejects.toThrow(/exit code 2/i); + const response = await provider.invoke(baseRequest); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('nonzero_exit'); + expect(response.targetExecution?.exitCode).toBe(2); + expect((response.raw as { error?: string }).error).toMatch(/exit code 2/i); }); it('treats timed out commands as failures', async () => { @@ -135,7 +143,119 @@ describe('CliProvider', () => { const provider = new CliProvider('cli-target', baseConfig, runner); - await expect(provider.invoke(baseRequest)).rejects.toThrow(/timed out/i); + const response = await provider.invoke(baseRequest); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('timeout'); + expect(response.targetExecution?.message).toMatch(/timed out/i); + }); + + it('returns target execution envelopes for spawn failures', async () => { + const runner = mock( + async (_command, _options): Promise => ({ + stdout: '', + stderr: 'not found', + exitCode: null, + failed: true, + spawnErrorCode: 'ENOENT', + }), + ); + + const provider = new CliProvider('cli-target', baseConfig, runner); + const response = await provider.invoke(baseRequest); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('spawn_failure'); + expect(response.targetExecution?.logs?.stderr?.text).toBe('not found'); + }); + + it('returns target execution envelopes for signal crashes', async () => { + const runner = mock( + async (_command, _options): Promise => ({ + stdout: 'partial stdout', + stderr: 'partial stderr', + exitCode: null, + failed: true, + signal: 'SIGSEGV', + }), + ); + + const provider = new CliProvider('cli-target', baseConfig, runner); + const response = await provider.invoke(baseRequest); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('signal_crash'); + expect(response.targetExecution?.signal).toBe('SIGSEGV'); + expect(response.targetExecution?.logs?.stdout?.text).toBe('partial stdout'); + }); + + it('returns target execution envelopes for cancellation', async () => { + const controller = new AbortController(); + const runner = mock(async (_command, _options): Promise => { + controller.abort(); + return { + stdout: 'partial stdout before cancel', + stderr: '', + exitCode: null, + failed: true, + signal: 'SIGTERM', + }; + }); + + const provider = new CliProvider('cli-target', baseConfig, runner); + const response = await provider.invoke({ ...baseRequest, signal: controller.signal }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('cancelled'); + expect(response.targetExecution?.logs?.stdout?.text).toContain('partial stdout'); + }); + + it('returns malformed-output envelopes when the output file is missing', async () => { + const runner = mock( + async (_command, _options): Promise => ({ + stdout: '{"event":"partial"}\n', + stderr: '', + exitCode: 0, + failed: false, + }), + ); + + const provider = new CliProvider('cli-target', baseConfig, runner); + const response = await provider.invoke(baseRequest); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('malformed_output'); + expect(response.targetExecution?.logs?.stdout?.text).toContain('partial'); + }); + + it('returns target task failure envelopes for CLI-reported errors', async () => { + const runner = mock(async (command: string): Promise => { + const match = command.match(/agentv-case-1-\d+-\w+\.json/); + if (match) { + const outputFilePath = path.join(os.tmpdir(), match[0]); + await writeFile( + outputFilePath, + JSON.stringify({ text: 'partial answer', error: 'task failed' }), + 'utf-8', + ); + createdFiles.push(outputFilePath); + } + + return { + stdout: 'structured event before failure', + stderr: '', + exitCode: 0, + failed: false, + }; + }); + + const provider = new CliProvider('cli-target', baseConfig, runner); + const response = await provider.invoke(baseRequest); + + expect(extractLastAssistantContent(response.output)).toBe('partial answer'); + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('target_task_failure'); + expect(response.targetExecution?.message).toBe('task failed'); }); it('uses request.cwd as working directory override in invoke', async () => { @@ -295,6 +415,7 @@ describe('CliProvider', () => { const errorContent = extractLastAssistantContent(responses[1]?.output); expect(errorContent).toMatch(/Batch output missing id 'case-2'/); expect(responses[1]?.raw?.error).toBe("Batch output missing id 'case-2'"); + expect(responses[1]?.targetExecution?.errorKind).toBe('malformed_output'); }); it('parses output from single case JSON output', async () => { diff --git a/packages/core/test/evaluation/target-execution-artifacts.test.ts b/packages/core/test/evaluation/target-execution-artifacts.test.ts new file mode 100644 index 000000000..863223f0c --- /dev/null +++ b/packages/core/test/evaluation/target-execution-artifacts.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'bun:test'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { runIndexPath, writeArtifactsFromResults } from '../../src/evaluation/run-artifacts.js'; +import { buildTraceFromMessages } from '../../src/evaluation/trace.js'; +import type { EvaluationResult } from '../../src/evaluation/types.js'; + +describe('target execution artifacts', () => { + it('indexes target execution envelopes, stdout/stderr, and transcripts', async () => { + const outputDir = await mkdtemp(path.join(tmpdir(), 'agentv-target-execution-')); + try { + const result: EvaluationResult = { + timestamp: '2026-07-03T00:00:00.000Z', + testId: 'crash-case', + suite: 'runtime', + score: 0, + assertions: [{ text: 'target crashed', passed: false }], + target: 'fake-cli', + output: 'Error: target crashed', + input: [{ role: 'user', content: 'run task' }], + trace: buildTraceFromMessages({ + input: [{ role: 'user', content: 'run task' }], + output: [{ role: 'assistant', content: 'partial answer' }], + finalOutput: 'partial answer', + target: 'fake-cli', + testId: 'crash-case', + error: 'target crashed', + }), + error: 'target crashed', + executionStatus: 'execution_error', + failureStage: 'agent', + failureReasonCode: 'target_signal_crash', + targetExecution: { + schemaVersion: 'agentv.target_execution.v1', + status: 'error', + targetId: 'fake-cli', + providerId: 'cli:fake-cli', + providerKind: 'cli', + runtimeMode: 'host', + command: { + argv: ['fake-agent', 'run'], + commandLine: 'fake-agent run', + cwd: '/workspace', + }, + startedAt: '2026-07-03T00:00:00.000Z', + endedAt: '2026-07-03T00:00:01.000Z', + durationMs: 1000, + exitCode: null, + signal: 'SIGSEGV', + errorKind: 'signal_crash', + message: 'target crashed', + logs: { + stdout: { + text: '{"event":"assistant_delta","text":"partial"}\n', + truncated: false, + bytes: 45, + storedBytes: 45, + }, + stderr: { + text: 'segmentation fault\n', + truncated: false, + bytes: 19, + storedBytes: 19, + }, + }, + transcript: { + messages: [{ role: 'assistant', content: 'partial answer' }], + finalOutput: 'partial answer', + }, + }, + }; + + await writeArtifactsFromResults([result], outputDir, { + evalFile: 'runtime.eval.yaml', + runId: 'run-target-execution', + }); + + const indexContent = await readFile(runIndexPath(outputDir), 'utf8'); + const row = JSON.parse(indexContent.trim()) as Record; + expect(row.failure_reason_code).toBe('target_signal_crash'); + expect(row.target_execution_path).toBeTruthy(); + expect(row.stdout_path).toBeTruthy(); + expect(row.stderr_path).toBeTruthy(); + + const targetExecution = row.target_execution as Record; + expect(targetExecution.error_kind).toBe('signal_crash'); + expect(targetExecution.provider_kind).toBe('cli'); + expect((targetExecution.artifacts as Record).stdout_path).toBe( + row.stdout_path, + ); + expect(targetExecution.artifacts).toHaveProperty('transcript_path'); + + const stdoutPath = path.join(outputDir, row.stdout_path as string); + const stderrPath = path.join(outputDir, row.stderr_path as string); + const envelopePath = path.join(outputDir, row.target_execution_path as string); + + await expect(readFile(stdoutPath, 'utf8')).resolves.toContain('assistant_delta'); + await expect(readFile(stderrPath, 'utf8')).resolves.toContain('segmentation fault'); + const envelope = JSON.parse(await readFile(envelopePath, 'utf8')) as Record; + expect(envelope.error_kind).toBe('signal_crash'); + expect(envelope.artifacts).toHaveProperty('stderr_path'); + } finally { + await rm(outputDir, { recursive: true, force: true }); + } + }); +}); From ec9b68f156e6b5bbb29d4c65432004c51376d5ed Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 16:13:04 +0200 Subject: [PATCH 03/10] feat(providers): isolate coding agent sdk targets --- apps/cli/package.json | 15 +- .../docs/docs/next/targets/coding-agents.mdx | 16 +- .../docs/docs/next/targets/configuration.mdx | 7 +- bun.lock | 61 +-- packages/core/package.json | 10 +- .../core/src/evaluation/providers/codex.ts | 3 +- .../src/evaluation/providers/copilot-sdk.ts | 3 +- .../core/src/evaluation/providers/index.ts | 20 +- .../providers/sdk-child-protocol.ts | 173 ++++++++ .../providers/sdk-child-provider.ts | 372 ++++++++++++++++++ .../evaluation/providers/sdk-child-runner.ts | 156 ++++++++ .../core/src/evaluation/providers/targets.ts | 14 +- .../core/src/evaluation/providers/types.ts | 9 + packages/core/src/types/codex-sdk.d.ts | 3 + packages/core/src/types/copilot-sdk.d.ts | 4 + .../providers/claude-provider-aliases.test.ts | 6 +- .../providers/sdk-child-provider.test.ts | 128 ++++++ .../providers/sdk-provider-registry.test.ts | 34 ++ .../test/evaluation/providers/targets.test.ts | 29 ++ packages/core/tsup.config.ts | 14 +- 20 files changed, 993 insertions(+), 84 deletions(-) create mode 100644 packages/core/src/evaluation/providers/sdk-child-protocol.ts create mode 100644 packages/core/src/evaluation/providers/sdk-child-provider.ts create mode 100644 packages/core/src/evaluation/providers/sdk-child-runner.ts create mode 100644 packages/core/src/types/codex-sdk.d.ts create mode 100644 packages/core/src/types/copilot-sdk.d.ts create mode 100644 packages/core/test/evaluation/providers/sdk-child-provider.test.ts create mode 100644 packages/core/test/evaluation/providers/sdk-provider-registry.test.ts diff --git a/apps/cli/package.json b/apps/cli/package.json index d1d89eea2..01c18437a 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -28,12 +28,9 @@ "test:watch": "bun test --watch" }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.49", - "@github/copilot-sdk": "^1.0.3", "@hono/node-server": "^1.19.11", "@inquirer/prompts": "^8.2.1", "@earendil-works/pi-ai": "^0.74.0", - "@openai/codex-sdk": "^0.136.0", "cmd-ts": "^0.14.3", "dotenv": "^16.4.5", "fast-glob": "^3.3.3", @@ -44,9 +41,21 @@ "yaml": "^2.8.3" }, "peerDependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.2.88", + "@github/copilot-sdk": "^1.0.3", + "@openai/codex-sdk": "^0.136.0", "@earendil-works/pi-coding-agent": "^0.74.0" }, "peerDependenciesMeta": { + "@anthropic-ai/claude-agent-sdk": { + "optional": true + }, + "@github/copilot-sdk": { + "optional": true + }, + "@openai/codex-sdk": { + "optional": true + }, "@earendil-works/pi-coding-agent": { "optional": true } diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index a71c64b2e..e130212b8 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -7,6 +7,8 @@ sidebar: Coding agent targets evaluate AI coding assistants and CLI-based agents. These targets require a `grader_target` to run LLM-based graders. +AgentV's recommended coding-agent targets use process or protocol boundaries such as `claude-cli`, `codex-cli`, `copilot-cli`, and `pi-cli`. SDK-backed targets are opt-in provider kinds: `claude-sdk`, `codex-sdk`, `copilot-sdk`, and `pi-sdk`. When selected, AgentV runs the SDK adapter in an AgentV-owned child process and communicates with it through a structured protocol, so SDK crashes, malformed output, timeouts, and missing optional SDK packages are scoped to that target run instead of the AgentV orchestrator process. + ## Prompt format Agent providers receive a structured prompt document with two sections: a **preread block** listing files the agent must read, and the **user query** containing the eval input. @@ -78,7 +80,7 @@ targets: ```yaml targets: - label: codex_target - provider: codex + provider: codex-cli executable: codex-eng model: ${{ CODEX_MODEL }} reasoning_effort: ${{ CODEX_REASONING_EFFORT }} @@ -93,6 +95,8 @@ targets: | `cwd` | No | Working directory | | `grader_target` | Yes | LLM target for evaluation | +Use `provider: codex-sdk` only when you intentionally want the Codex SDK path. The SDK package is optional and loaded inside the isolated child runner. + ## Copilot CLI ```yaml @@ -130,12 +134,14 @@ targets: Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps these flat fields to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged. -## Pi Coding Agent +Use `provider: copilot-sdk` only when you intentionally want the Copilot SDK path. The SDK package is optional and loaded inside the isolated child runner. + +## Pi SDK ```yaml targets: - label: pi_target - provider: pi-coding-agent + provider: pi-sdk subprovider: openai-codex model: gpt-5.5 thinking: medium @@ -154,13 +160,13 @@ targets: | `timeout_seconds` | No | Per-case timeout | | `grader_target` | Yes | LLM target for evaluation | -For `provider: pi-coding-agent`, `base_url` is passed through the Pi SDK model +For `provider: pi-sdk`, `base_url` is passed through the Pi SDK model configuration. This works for OpenAI-compatible endpoints: ```yaml targets: - label: pi-sdk-openai - provider: pi-coding-agent + provider: pi-sdk subprovider: openai base_url: ${{ OPENAI_ENDPOINT }} api_key: ${{ OPENAI_API_KEY }} diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 18b5516eb..ff7fae1b4 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -86,11 +86,12 @@ already-exported secrets into `.env`. | `anthropic` | LLM | Anthropic Claude API | | `gemini` | LLM | Google Gemini | | `claude-cli` | Agent | Claude CLI subprocess | -| `claude-sdk` | Agent | Claude Agent SDK | +| `claude-sdk` | Agent | Claude Agent SDK in an isolated child runner | | `codex-cli` | Agent | Codex CLI subprocess | | `codex-app-server` | Agent | Codex app-server subprocess | -| `codex-sdk` | Agent | Codex SDK provider | -| `pi-coding-agent` | Agent | Pi Coding Agent | +| `codex-sdk` | Agent | Codex SDK in an isolated child runner | +| `copilot-sdk` | Agent | Copilot SDK in an isolated child runner | +| `pi-sdk` | Agent | Pi SDK in an isolated child runner | | `pi-cli` | Agent | Pi CLI subprocess | | `vscode` | Agent | VS Code with Copilot | | `vscode-insiders` | Agent | VS Code Insiders | diff --git a/bun.lock b/bun.lock index 1e891deb9..d0efaecab 100644 --- a/bun.lock +++ b/bun.lock @@ -24,12 +24,9 @@ "agentv": "./dist/cli.js", }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.49", "@earendil-works/pi-ai": "^0.74.0", - "@github/copilot-sdk": "^1.0.3", "@hono/node-server": "^1.19.11", "@inquirer/prompts": "^8.2.1", - "@openai/codex-sdk": "^0.136.0", "cmd-ts": "^0.14.3", "dotenv": "^16.4.5", "fast-glob": "^3.3.3", @@ -46,10 +43,16 @@ "execa": "^9.3.0", }, "peerDependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.2.88", "@earendil-works/pi-coding-agent": "^0.74.0", + "@github/copilot-sdk": "^1.0.3", + "@openai/codex-sdk": "^0.136.0", }, "optionalPeers": [ + "@anthropic-ai/claude-agent-sdk", "@earendil-works/pi-coding-agent", + "@github/copilot-sdk", + "@openai/codex-sdk", ], }, "apps/dashboard": { @@ -89,8 +92,6 @@ "dependencies": { "@agentclientprotocol/sdk": "^0.14.1", "@earendil-works/pi-ai": "^0.74.0", - "@github/copilot-sdk": "^1.0.3", - "@openai/codex-sdk": "^0.136.0", "fast-glob": "^3.3.3", "json5": "^2.2.3", "micromatch": "^4.0.8", @@ -106,17 +107,21 @@ "peerDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.88", "@earendil-works/pi-coding-agent": "^0.74.0", + "@github/copilot-sdk": "^1.0.3", + "@openai/codex-sdk": "^0.136.0", }, "optionalPeers": [ "@anthropic-ai/claude-agent-sdk", "@earendil-works/pi-coding-agent", + "@github/copilot-sdk", + "@openai/codex-sdk", ], }, "packages/sdk": { "name": "@agentv/sdk", "version": "5.0.0-next.1", "dependencies": { - "@agentv/core": "workspace:*", + "@agentv/core": "5.0.0-next.1", "yaml": "^2.8.3", "zod": "^3.23.8", }, @@ -133,8 +138,6 @@ "@agentv/web": ["@agentv/web@workspace:apps/web"], - "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.49", "", { "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-3avi409dwuGkPEETpWa0gyJvRMr3b6LxeuW5/sAPCOtLD9WxH9fYltbA5wZoazxTw5mlbXmjDp7JqO1rlmpaIQ=="], - "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.91.1", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw=="], "@astrojs/compiler": ["@astrojs/compiler@2.13.0", "", {}, "sha512-mqVORhUJViA28fwHYaWmsXSzLO9osbdZ5ImUfxBarqsYdMlPbqAqGJCxsNzvppp1BEzc1mJNjOVvQqeDN8Vspw=="], @@ -355,26 +358,6 @@ "@expressive-code/plugin-text-markers": ["@expressive-code/plugin-text-markers@0.41.6", "", { "dependencies": { "@expressive-code/core": "^0.41.6" } }, "sha512-PBFa1wGyYzRExMDzBmAWC6/kdfG1oLn4pLpBeTfIRrALPjcGA/59HP3e7q9J0Smk4pC7U+lWkA2LHR8FYV8U7Q=="], - "@github/copilot": ["@github/copilot@1.0.64-1", "", { "dependencies": { "detect-libc": "^2.1.2" }, "optionalDependencies": { "@github/copilot-darwin-arm64": "1.0.64-1", "@github/copilot-darwin-x64": "1.0.64-1", "@github/copilot-linux-arm64": "1.0.64-1", "@github/copilot-linux-x64": "1.0.64-1", "@github/copilot-linuxmusl-arm64": "1.0.64-1", "@github/copilot-linuxmusl-x64": "1.0.64-1", "@github/copilot-win32-arm64": "1.0.64-1", "@github/copilot-win32-x64": "1.0.64-1" }, "bin": { "copilot": "npm-loader.js" } }, "sha512-lojV4Cb7oT4VJnYPEKBRH8KI3W43Q4Lh0Pc+V6sej+xjPJkoqwm68sNKn73/p3wXPBSTVTzPeCm9WhIisgf1Jw=="], - - "@github/copilot-darwin-arm64": ["@github/copilot-darwin-arm64@1.0.64-1", "", { "os": "darwin", "cpu": "arm64", "bin": { "copilot-darwin-arm64": "copilot" } }, "sha512-MQHZT9LhmCiq+ogO1E8cPCWrurZ6x+r9lPJfYUSnOyMO+EHbREpiJwOOChxtLHgL2/tKJSZdId2pg3tDgUlcsw=="], - - "@github/copilot-darwin-x64": ["@github/copilot-darwin-x64@1.0.64-1", "", { "os": "darwin", "cpu": "x64", "bin": { "copilot-darwin-x64": "copilot" } }, "sha512-kOQY7CvI7He0eO3ObQAHePWdkNLWAOegCSzUqUmdcpa1SNVqbZ3GBMsQ7uAZQip2cQxnGZ7pS1v6tKQ0HJdkYw=="], - - "@github/copilot-linux-arm64": ["@github/copilot-linux-arm64@1.0.64-1", "", { "os": "linux", "cpu": "arm64", "bin": { "copilot-linux-arm64": "copilot" } }, "sha512-hIfuO7Q+pWs0SKfIRYqT+CjMaupudnhp4RMS6XoJ5s/e33rvpj2tkTkXYlHJo1PMDI823vvbqgpEdr+KeewMwg=="], - - "@github/copilot-linux-x64": ["@github/copilot-linux-x64@1.0.64-1", "", { "os": "linux", "cpu": "x64", "bin": { "copilot-linux-x64": "copilot" } }, "sha512-VHaE62pha0rDDvuNN3bd97gf0EZ+EJebstM1ejHsMYoPT1IOUkYEXlNfGGHY+GfUGYxAiy/+Uew4xw5mJyy/Sw=="], - - "@github/copilot-linuxmusl-arm64": ["@github/copilot-linuxmusl-arm64@1.0.64-1", "", { "os": "linux", "cpu": "arm64", "bin": { "copilot-linuxmusl-arm64": "copilot" } }, "sha512-L/YrZPotRujAP0QERq+DlkR1SLr7abbTSz/56JqKKOqEdjKZPdQW1bUlhL/w1CZg1gXlTNUsNVyKz/fUfrEBgw=="], - - "@github/copilot-linuxmusl-x64": ["@github/copilot-linuxmusl-x64@1.0.64-1", "", { "os": "linux", "cpu": "x64", "bin": { "copilot-linuxmusl-x64": "copilot" } }, "sha512-AGMjXqR128oyjiJhoI6Gd7JP5ddWkib+P4YH/JoHm05iNn23ZYl4tSc0XihHzeyMI1ix7Aacn8UINYB7lGOGOA=="], - - "@github/copilot-sdk": ["@github/copilot-sdk@1.0.3", "", { "dependencies": { "@github/copilot": "^1.0.64-1", "vscode-jsonrpc": "^8.2.1", "zod": "^4.3.6" } }, "sha512-ujnH2QVw3+xvjgo9cbpY0wik4fNxAmdMDSFnxGScDSvRuK2vUCL2xWW4V2ANc9pWwRHPBpEpMuNJMtmydmLCIQ=="], - - "@github/copilot-win32-arm64": ["@github/copilot-win32-arm64@1.0.64-1", "", { "os": "win32", "cpu": "arm64", "bin": { "copilot-win32-arm64": "copilot.exe" } }, "sha512-vvv+gnemi9WKaxF41zz7Xmq6a493n8Yjps5UFaOY6a3WR222kKXZXfOpeRvIYsDgnIPHGBHIj1TBOmnHQT4V4w=="], - - "@github/copilot-win32-x64": ["@github/copilot-win32-x64@1.0.64-1", "", { "os": "win32", "cpu": "x64", "bin": { "copilot-win32-x64": "copilot.exe" } }, "sha512-mcHvD0fjGDuqr/YXzy8mKuDmah1F+qjPujxoFuGmabmTJZ33cSIJ3nq7RRvxZNIdp8YJ57NkbcW30WvIcOeJ3w=="], - "@google/genai": ["@google/genai@1.51.0", "", { "dependencies": { "google-auth-library": "^10.3.0", "p-retry": "^4.6.2", "protobufjs": "^7.5.4", "ws": "^8.18.0" }, "peerDependencies": { "@modelcontextprotocol/sdk": "^1.25.2" }, "optionalPeers": ["@modelcontextprotocol/sdk"] }, "sha512-vTZZF3CSimN7cn2zsLpW2p5WF0eZa5Gz69ITMPCNHpPrDlAstOfGifSfi0p/s9Z9400f7xJRkgvkQNrcM7pJ6w=="], "@hono/node-server": ["@hono/node-server@1.19.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-dr8/3zEaB+p0D2n/IUrlPF1HZm586qgJNXK1a9fhg/PzdtkK7Ksd5l312tJX2yBuALqDYBlG20QEbayqPyxn+g=="], @@ -487,22 +470,6 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], - "@openai/codex": ["@openai/codex@0.136.0", "", { "optionalDependencies": { "@openai/codex-darwin-arm64": "npm:@openai/codex@0.136.0-darwin-arm64", "@openai/codex-darwin-x64": "npm:@openai/codex@0.136.0-darwin-x64", "@openai/codex-linux-arm64": "npm:@openai/codex@0.136.0-linux-arm64", "@openai/codex-linux-x64": "npm:@openai/codex@0.136.0-linux-x64", "@openai/codex-win32-arm64": "npm:@openai/codex@0.136.0-win32-arm64", "@openai/codex-win32-x64": "npm:@openai/codex@0.136.0-win32-x64" }, "bin": { "codex": "bin/codex.js" } }, "sha512-yu+diXznSOt8h296/CDOf2CNi55z1raA7aZ6sejjGy1QWPqn72YkBc2ByTzZG6G+1yiUqlm2G5Hid54hm3/kaA=="], - - "@openai/codex-darwin-arm64": ["@openai/codex@0.136.0-darwin-arm64", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9xnEohnUO9l6qtPSDbG0Y2UXX5mBZ9Lsn0VMgjtkREGfWL9TawNC1d7D1ERMSJtHTlVvieoaFyK9LHPLQCO9Vw=="], - - "@openai/codex-darwin-x64": ["@openai/codex@0.136.0-darwin-x64", "", { "os": "darwin", "cpu": "x64" }, "sha512-HD9YLalPg0cv+CiZSmRWOl/8sefuJlxJcAmxyzb8mSaAMchD3V+2yS6M6E1Z2I6NX1irp4Polt1fPsjGdlHYhQ=="], - - "@openai/codex-linux-arm64": ["@openai/codex@0.136.0-linux-arm64", "", { "os": "linux", "cpu": "arm64" }, "sha512-xDxTOk5ClEX/nzlQseqEoiubjIzVZfcWKn9nmKkHOLKknz+c7n6tbmw7eY4mwt29zTAAoFoecdRnbbsPZUHJ1g=="], - - "@openai/codex-linux-x64": ["@openai/codex@0.136.0-linux-x64", "", { "os": "linux", "cpu": "x64" }, "sha512-Q6yZHRtUWD+27B5yAiOYyPAtM0BzW0Mm23YGaXDmfNEO5BYgHuUT80VDofUpjSyo2+ShYoZQUFVQAsNPKqd+Sw=="], - - "@openai/codex-sdk": ["@openai/codex-sdk@0.136.0", "", { "dependencies": { "@openai/codex": "0.136.0" } }, "sha512-qrSirrVxrpVHR1YIQQ4WSouneaQrQrRAAlQWJYRflvMCcruFXorsmIPeIbDqFXOa8A9H9FHHyc+2U4tdbEGGiQ=="], - - "@openai/codex-win32-arm64": ["@openai/codex@0.136.0-win32-arm64", "", { "os": "win32", "cpu": "arm64" }, "sha512-mAHZXfygQxBg1JjZ9+bAZfBXe6fg8MabJhuDYhj5z0VF++orKSFsC1lMiv6PksQN36ikQefgVjc9EOlaE4lt7g=="], - - "@openai/codex-win32-x64": ["@openai/codex@0.136.0-win32-x64", "", { "os": "win32", "cpu": "x64" }, "sha512-zS6DAmvjdWeAB1CL9KTUMkwzTwfXtxHy8GAtePw2a93jIqawoG07fBxAXuyoHZ3QXQkwEgqBx1zEEh33gdIKAw=="], - "@oslojs/encoding": ["@oslojs/encoding@1.1.0", "", {}, "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ=="], "@pagefind/darwin-arm64": ["@pagefind/darwin-arm64@1.4.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2vMqkbv3lbx1Awea90gTaBsvpzgRs7MuSgKDxW0m9oV1GPZCZbZBJg/qL83GIUEN2BFlY46dtUZi54pwH+/pTQ=="], @@ -1813,8 +1780,6 @@ "vitefu": ["vitefu@1.1.1", "", { "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0-beta.0" }, "optionalPeers": ["vite"] }, "sha512-B/Fegf3i8zh0yFbpzZ21amWzHmuNlLlmJT6n7bu5e+pCHUKQIfXSYokrqOBGEMMe9UG2sostKQF9mml/vYaWJQ=="], - "vscode-jsonrpc": ["vscode-jsonrpc@8.2.1", "", {}, "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ=="], - "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], @@ -1857,8 +1822,6 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], - "@anthropic-ai/claude-agent-sdk/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], - "@astrojs/mdx/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "@aws-crypto/sha256-browser/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="], @@ -1885,8 +1848,6 @@ "@babel/traverse/@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="], - "@github/copilot-sdk/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], - "@mdx-js/mdx/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "@mistralai/mistralai/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], diff --git a/packages/core/package.json b/packages/core/package.json index 9b577cf59..638515a87 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -41,9 +41,7 @@ "files": ["dist", "README.md"], "dependencies": { "@agentclientprotocol/sdk": "^0.14.1", - "@github/copilot-sdk": "^1.0.3", "@earendil-works/pi-ai": "^0.74.0", - "@openai/codex-sdk": "^0.136.0", "fast-glob": "^3.3.3", "json5": "^2.2.3", "micromatch": "^4.0.8", @@ -53,12 +51,20 @@ }, "peerDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.88", + "@github/copilot-sdk": "^1.0.3", + "@openai/codex-sdk": "^0.136.0", "@earendil-works/pi-coding-agent": "^0.74.0" }, "peerDependenciesMeta": { "@anthropic-ai/claude-agent-sdk": { "optional": true }, + "@github/copilot-sdk": { + "optional": true + }, + "@openai/codex-sdk": { + "optional": true + }, "@earendil-works/pi-coding-agent": { "optional": true } diff --git a/packages/core/src/evaluation/providers/codex.ts b/packages/core/src/evaluation/providers/codex.ts index 71e3d19c5..ee8669843 100644 --- a/packages/core/src/evaluation/providers/codex.ts +++ b/packages/core/src/evaluation/providers/codex.ts @@ -22,7 +22,8 @@ import type { // biome-ignore lint/suspicious/noExplicitAny: dynamic import type let codexSdkModule: any = null; -async function loadCodexSdk(): Promise { +// biome-ignore lint/suspicious/noExplicitAny: optional SDK package is loaded only inside child runner +async function loadCodexSdk(): Promise { if (!codexSdkModule) { try { codexSdkModule = await import('@openai/codex-sdk'); diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index 90353f16a..6f6af7991 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -29,7 +29,8 @@ import type { // biome-ignore lint/suspicious/noExplicitAny: dynamic import type let copilotSdkModule: any = null; -async function loadCopilotSdk(): Promise { +// biome-ignore lint/suspicious/noExplicitAny: optional SDK package is loaded only inside child runner +async function loadCopilotSdk(): Promise { if (!copilotSdkModule) { try { copilotSdkModule = await import('@github/copilot-sdk'); diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 98ae28744..ab93a43b9 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -1,12 +1,9 @@ import { AgentvProvider } from './agentv-provider.js'; import { ClaudeCliProvider } from './claude-cli.js'; -import { ClaudeSdkProvider } from './claude-sdk.js'; -import { ClaudeProvider } from './claude.js'; import { CliProvider } from './cli.js'; -import { CodexProvider } from './codex.js'; +import { CodexCliProvider } from './codex-cli.js'; import { CopilotCliProvider } from './copilot-cli.js'; import { CopilotLogProvider } from './copilot-log.js'; -import { CopilotSdkProvider } from './copilot-sdk.js'; import { AnthropicProvider, AzureProvider, @@ -16,9 +13,9 @@ import { } from './llm-providers.js'; import { MockProvider } from './mock.js'; import { PiCliProvider } from './pi-cli.js'; -import { PiCodingAgentProvider } from './pi-coding-agent.js'; import { ProviderRegistry } from './provider-registry.js'; import { ReplayProvider } from './replay.js'; +import { SdkChildProvider } from './sdk-child-provider.js'; import type { ResolvedTarget } from './targets.js'; import { COMMON_TARGET_SETTINGS, @@ -110,17 +107,20 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { .register('anthropic', (t) => new AnthropicProvider(t.name, t.config as never)) .register('gemini', (t) => new GeminiProvider(t.name, t.config as never)) .register('cli', (t) => new CliProvider(t.name, t.config as never)) - .register('codex', (t) => new CodexProvider(t.name, t.config as never)) - .register('copilot-sdk', (t) => new CopilotSdkProvider(t.name, t.config as never)) + .register('codex', (t) => new CodexCliProvider(t.name, t.config as never)) + .register('codex-cli', (t) => new CodexCliProvider(t.name, t.config as never)) + .register('codex-sdk', (t) => new SdkChildProvider('codex-sdk', t.name, t.config)) + .register('copilot-sdk', (t) => new SdkChildProvider('copilot-sdk', t.name, t.config)) .register('copilot-cli', (t) => new CopilotCliProvider(t.name, t.config as never)) .register('copilot-log', (t) => new CopilotLogProvider(t.name, t.config as never)) - .register('pi-coding-agent', (t) => new PiCodingAgentProvider(t.name, t.config as never)) + .register('pi-sdk', (t) => new SdkChildProvider('pi-sdk', t.name, t.config)) + .register('pi-coding-agent', (t) => new SdkChildProvider('pi-sdk', t.name, t.config)) .register('pi-cli', (t) => new PiCliProvider(t.name, t.config as never)) // claude-cli is the new default subprocess provider; claude is an alias .register('claude-cli', (t) => new ClaudeCliProvider(t.name, t.config as never)) .register('claude', (t) => new ClaudeCliProvider(t.name, t.config as never)) - // claude-sdk is the explicit SDK provider (requires @anthropic-ai/claude-agent-sdk) - .register('claude-sdk', (t) => new ClaudeSdkProvider(t.name, t.config as never)) + // Explicit SDK providers are isolated behind an AgentV child runner. + .register('claude-sdk', (t) => new SdkChildProvider('claude-sdk', t.name, t.config)) .register('mock', (t) => new MockProvider(t.name, t.config as never)) .register('agentv', (t) => new AgentvProvider(t.name, t.config as never)) .register('replay', (t) => new ReplayProvider(t.name, t.config as never)) diff --git a/packages/core/src/evaluation/providers/sdk-child-protocol.ts b/packages/core/src/evaluation/providers/sdk-child-protocol.ts new file mode 100644 index 000000000..f98a679e9 --- /dev/null +++ b/packages/core/src/evaluation/providers/sdk-child-protocol.ts @@ -0,0 +1,173 @@ +import type { JsonObject } from '../types.js'; +import type { + ChatPrompt, + Message, + ProviderRequest, + ProviderResponse, + ProviderTokenUsage, +} from './types.js'; + +export const SDK_CHILD_PROTOCOL_VERSION = 1; + +export type SdkChildProviderKind = 'codex-sdk' | 'claude-sdk' | 'copilot-sdk' | 'pi-sdk'; + +export interface SdkChildRequestEnvelope { + readonly protocol_version: typeof SDK_CHILD_PROTOCOL_VERSION; + readonly provider_kind: SdkChildProviderKind; + readonly target_name: string; + readonly config: unknown; + readonly request: SdkChildProviderRequestWire; +} + +export interface SdkChildProviderRequestWire { + readonly question: string; + readonly system_prompt?: string; + readonly chat_prompt?: ChatPrompt; + readonly input_files?: readonly string[]; + readonly eval_case_id?: string; + readonly suite?: string; + readonly eval_file_path?: string; + readonly attempt?: number; + readonly max_output_tokens?: number; + readonly temperature?: number; + readonly metadata?: JsonObject; + readonly cwd?: string; + readonly workspace_file?: string; + readonly capture_file_changes?: boolean; + readonly braintrust_span_ids?: { readonly parent_span_id: string; readonly root_span_id: string }; + readonly images?: ProviderRequest['images']; +} + +export type SdkChildOutputEnvelope = + | { + readonly protocol_version: typeof SDK_CHILD_PROTOCOL_VERSION; + readonly type: 'event'; + readonly event: SdkChildEventWire; + } + | { + readonly protocol_version: typeof SDK_CHILD_PROTOCOL_VERSION; + readonly type: 'result'; + readonly response: SdkChildProviderResponseWire; + } + | { + readonly protocol_version: typeof SDK_CHILD_PROTOCOL_VERSION; + readonly type: 'error'; + readonly error: SdkChildErrorWire; + }; + +export interface SdkChildEventWire { + readonly kind: 'log' | 'provider_event' | 'lifecycle'; + readonly stream?: 'stdout' | 'stderr'; + readonly message?: string; + readonly provider_event_type?: string; + readonly data?: unknown; +} + +export interface SdkChildErrorWire { + readonly code: string; + readonly message: string; + readonly stack?: string; +} + +export interface SdkChildProviderResponseWire { + readonly raw?: unknown; + readonly usage?: JsonObject; + readonly output?: readonly Message[]; + readonly token_usage?: ProviderTokenUsage; + readonly cost_usd?: number; + readonly duration_ms?: number; + readonly start_time?: string; + readonly end_time?: string; + readonly steps?: ProviderResponse['steps']; + readonly file_changes?: string; +} + +export function providerRequestToWire(request: ProviderRequest): SdkChildProviderRequestWire { + if (request.tools && request.tools.length > 0) { + throw new Error('SDK child providers do not support in-process tool callbacks'); + } + + return { + question: request.question, + system_prompt: request.systemPrompt, + chat_prompt: request.chatPrompt, + input_files: request.inputFiles, + eval_case_id: request.evalCaseId, + suite: request.suite, + eval_file_path: request.evalFilePath, + attempt: request.attempt, + max_output_tokens: request.maxOutputTokens, + temperature: request.temperature, + metadata: request.metadata, + cwd: request.cwd, + workspace_file: request.workspaceFile, + capture_file_changes: request.captureFileChanges, + braintrust_span_ids: request.braintrustSpanIds + ? { + parent_span_id: request.braintrustSpanIds.parentSpanId, + root_span_id: request.braintrustSpanIds.rootSpanId, + } + : undefined, + images: request.images, + }; +} + +export function providerRequestFromWire(request: SdkChildProviderRequestWire): ProviderRequest { + return { + question: request.question, + systemPrompt: request.system_prompt, + chatPrompt: request.chat_prompt, + inputFiles: request.input_files, + evalCaseId: request.eval_case_id, + suite: request.suite, + evalFilePath: request.eval_file_path, + attempt: request.attempt, + maxOutputTokens: request.max_output_tokens, + temperature: request.temperature, + metadata: request.metadata, + cwd: request.cwd, + workspaceFile: request.workspace_file, + captureFileChanges: request.capture_file_changes, + braintrustSpanIds: request.braintrust_span_ids + ? { + parentSpanId: request.braintrust_span_ids.parent_span_id, + rootSpanId: request.braintrust_span_ids.root_span_id, + } + : undefined, + images: request.images, + }; +} + +export function providerResponseToWire(response: ProviderResponse): SdkChildProviderResponseWire { + return { + raw: response.raw, + usage: response.usage, + output: response.output, + token_usage: response.tokenUsage, + cost_usd: response.costUsd, + duration_ms: response.durationMs, + start_time: response.startTime, + end_time: response.endTime, + steps: response.steps, + file_changes: response.fileChanges, + }; +} + +export function providerResponseFromWire(response: SdkChildProviderResponseWire): ProviderResponse { + return { + raw: response.raw, + usage: response.usage, + output: response.output, + tokenUsage: response.token_usage, + costUsd: response.cost_usd, + durationMs: response.duration_ms, + startTime: response.start_time, + endTime: response.end_time, + steps: response.steps, + fileChanges: response.file_changes, + }; +} + +export function writeSdkChildEnvelope(envelope: SdkChildOutputEnvelope): void { + process.stdout.write(`${JSON.stringify(envelope)}\n`); +} diff --git a/packages/core/src/evaluation/providers/sdk-child-provider.ts b/packages/core/src/evaluation/providers/sdk-child-provider.ts new file mode 100644 index 000000000..d59336167 --- /dev/null +++ b/packages/core/src/evaluation/providers/sdk-child-provider.ts @@ -0,0 +1,372 @@ +import { spawn } from 'node:child_process'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +import { trackChild } from '../../runtime/child-tracker.js'; +import { + SDK_CHILD_PROTOCOL_VERSION, + type SdkChildEventWire, + type SdkChildOutputEnvelope, + type SdkChildProviderKind, + type SdkChildRequestEnvelope, + providerRequestToWire, + providerResponseFromWire, +} from './sdk-child-protocol.js'; +import type { Provider, ProviderRequest, ProviderResponse } from './types.js'; + +export class SdkChildRunnerError extends Error { + readonly reason: + | 'spawn' + | 'exit' + | 'signal' + | 'timeout' + | 'cancelled' + | 'malformed_output' + | 'protocol' + | 'child_error'; + readonly details?: unknown; + + constructor( + providerKind: SdkChildProviderKind, + reason: SdkChildRunnerError['reason'], + message: string, + details?: unknown, + ) { + super(`${providerKind} child runner ${reason}: ${message}`); + this.name = 'SdkChildRunnerError'; + this.reason = reason; + this.details = details; + } +} + +export interface SdkChildProviderOptions { + readonly runnerArgv?: readonly string[]; +} + +export class SdkChildProvider implements Provider { + readonly id: string; + readonly targetName: string; + readonly supportsBatch = false; + + readonly kind: SdkChildProviderKind; + + private readonly config: unknown; + private readonly runnerArgv?: readonly string[]; + + constructor( + kind: SdkChildProviderKind, + targetName: string, + config: unknown, + options: SdkChildProviderOptions = {}, + ) { + this.kind = kind; + this.id = `${kind}:${targetName}`; + this.targetName = targetName; + this.config = config; + this.runnerArgv = options.runnerArgv; + } + + async invoke(request: ProviderRequest): Promise { + if (request.signal?.aborted) { + throw new SdkChildRunnerError(this.kind, 'cancelled', 'request was aborted before execution'); + } + + const childRequest: SdkChildRequestEnvelope = { + protocol_version: SDK_CHILD_PROTOCOL_VERSION, + provider_kind: this.kind, + target_name: this.targetName, + config: this.config, + request: providerRequestToWire(request), + }; + + const argv = this.runnerArgv ?? resolveDefaultRunnerArgv(); + if (argv.length === 0) { + throw new SdkChildRunnerError(this.kind, 'spawn', 'runner argv is empty'); + } + + return await this.runChild(argv, childRequest, request); + } + + private async runChild( + argv: readonly string[], + childRequest: SdkChildRequestEnvelope, + request: ProviderRequest, + ): Promise { + const [command, ...args] = argv; + let child: ChildProcessWithoutNullStreams; + try { + child = spawn(command, args, { + stdio: ['pipe', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + env: process.env, + }); + } catch (error) { + throw new SdkChildRunnerError(this.kind, 'spawn', formatError(error), { argv }); + } + + trackChild(child); + + const events: SdkChildEventWire[] = []; + const stderrChunks: Buffer[] = []; + let finalResponse: ProviderResponse | undefined; + let childError: (SdkChildOutputEnvelope & { readonly type: 'error' }) | undefined; + let settled = false; + let timedOut = false; + let cancelled = false; + let malformedOutput: string | undefined; + let stdoutBuffer = ''; + + const cleanupAbortListener = this.attachAbortHandler(request, child, () => { + cancelled = true; + }); + + const timeout = + getTimeoutMs(this.config) !== undefined + ? setTimeout(() => { + timedOut = true; + killChildProcessGroup(child); + }, getTimeoutMs(this.config)) + : undefined; + timeout?.unref?.(); + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { + stdoutBuffer += chunk; + let newlineIndex = stdoutBuffer.indexOf('\n'); + while (newlineIndex >= 0) { + const line = stdoutBuffer.slice(0, newlineIndex); + stdoutBuffer = stdoutBuffer.slice(newlineIndex + 1); + try { + this.handleProtocolLine( + line, + events, + (response) => { + finalResponse = response; + }, + (errorEnvelope) => { + childError = errorEnvelope; + }, + ); + } catch (error) { + malformedOutput = formatError(error); + killChildProcessGroup(child); + return; + } + newlineIndex = stdoutBuffer.indexOf('\n'); + } + }); + + child.stderr.on('data', (chunk: Buffer) => { + stderrChunks.push(chunk); + }); + + const closeResult = await new Promise<{ + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + }>((resolve, reject) => { + child.once('error', (error) => { + if (!settled) { + settled = true; + reject(new SdkChildRunnerError(this.kind, 'spawn', formatError(error), { argv })); + } + }); + child.once('close', (code, signal) => { + if (!settled) { + settled = true; + resolve({ code, signal }); + } + }); + + child.stdin.end(`${JSON.stringify(childRequest)}\n`); + }).finally(() => { + cleanupAbortListener(); + if (timeout) { + clearTimeout(timeout); + } + }); + + if (stdoutBuffer.trim().length > 0 && !malformedOutput) { + try { + this.handleProtocolLine( + stdoutBuffer, + events, + (response) => { + finalResponse = response; + }, + (errorEnvelope) => { + childError = errorEnvelope; + }, + ); + } catch (error) { + malformedOutput = formatError(error); + } + } + + const stderr = Buffer.concat(stderrChunks).toString('utf8').replace(/\r\n/g, '\n'); + + if (timedOut) { + throw new SdkChildRunnerError( + this.kind, + 'timeout', + formatTimeout(getTimeoutMs(this.config)), + { + stderr, + events, + }, + ); + } + if (cancelled) { + throw new SdkChildRunnerError(this.kind, 'cancelled', 'request was cancelled', { + stderr, + events, + }); + } + if (malformedOutput) { + throw new SdkChildRunnerError(this.kind, 'malformed_output', malformedOutput, { + stderr, + events, + }); + } + if (childError) { + throw new SdkChildRunnerError(this.kind, 'child_error', childError.error.message, { + code: childError.error.code, + stack: childError.error.stack, + stderr, + events, + }); + } + if (closeResult.signal) { + throw new SdkChildRunnerError(this.kind, 'signal', String(closeResult.signal), { + stderr, + events, + }); + } + if (closeResult.code !== 0) { + throw new SdkChildRunnerError(this.kind, 'exit', `exit code ${closeResult.code}`, { + stderr, + events, + }); + } + if (!finalResponse) { + throw new SdkChildRunnerError(this.kind, 'protocol', 'missing final result envelope', { + stderr, + events, + }); + } + + return attachChildRunnerRaw(finalResponse, { events, stderr }); + } + + private handleProtocolLine( + line: string, + events: SdkChildEventWire[], + setResult: (response: ProviderResponse) => void, + setError: (errorEnvelope: SdkChildOutputEnvelope & { readonly type: 'error' }) => void, + ): void { + const trimmed = line.trim(); + if (!trimmed) { + return; + } + + const parsed = JSON.parse(trimmed) as SdkChildOutputEnvelope; + if (parsed.protocol_version !== SDK_CHILD_PROTOCOL_VERSION) { + throw new Error(`unexpected protocol_version ${String(parsed.protocol_version)}`); + } + + if (parsed.type === 'event') { + events.push(parsed.event); + return; + } + if (parsed.type === 'result') { + setResult(providerResponseFromWire(parsed.response)); + return; + } + if (parsed.type === 'error') { + setError(parsed); + return; + } + + throw new Error(`unknown protocol message type ${(parsed as { type?: unknown }).type}`); + } + + private attachAbortHandler( + request: ProviderRequest, + child: ChildProcessWithoutNullStreams, + onAbort: () => void, + ): () => void { + if (!request.signal) { + return () => {}; + } + const abortHandler = () => { + onAbort(); + killChildProcessGroup(child); + }; + request.signal.addEventListener('abort', abortHandler, { once: true }); + return () => request.signal?.removeEventListener('abort', abortHandler); + } +} + +function resolveDefaultRunnerArgv(): readonly string[] { + const jsRunnerPath = fileURLToPath(new URL('./sdk-child-runner.js', import.meta.url)); + if (existsSync(jsRunnerPath)) { + return [process.execPath, jsRunnerPath]; + } + const tsRunnerPath = fileURLToPath(new URL('./sdk-child-runner.ts', import.meta.url)); + return [process.execPath, tsRunnerPath]; +} + +function killChildProcessGroup(child: ChildProcessWithoutNullStreams): void { + if (child.pid === undefined) { + return; + } + try { + if (process.platform === 'win32') { + child.kill('SIGKILL'); + } else { + process.kill(-child.pid, 'SIGKILL'); + } + } catch { + try { + child.kill('SIGKILL'); + } catch {} + } +} + +function getTimeoutMs(config: unknown): number | undefined { + if (typeof config !== 'object' || config === null) { + return undefined; + } + const timeoutMs = (config as { timeoutMs?: unknown }).timeoutMs; + return typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs > 0 + ? timeoutMs + : undefined; +} + +function formatTimeout(timeoutMs: number | undefined): string { + return timeoutMs ? `timed out after ${timeoutMs}ms` : 'timed out'; +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function attachChildRunnerRaw( + response: ProviderResponse, + childRunner: { readonly events: readonly SdkChildEventWire[]; readonly stderr: string }, +): ProviderResponse { + const raw = + typeof response.raw === 'object' && response.raw !== null && !Array.isArray(response.raw) + ? response.raw + : response.raw === undefined + ? {} + : { provider_raw: response.raw }; + + return { + ...response, + raw: { + ...raw, + child_runner: childRunner, + }, + }; +} diff --git a/packages/core/src/evaluation/providers/sdk-child-runner.ts b/packages/core/src/evaluation/providers/sdk-child-runner.ts new file mode 100644 index 000000000..8ac59c900 --- /dev/null +++ b/packages/core/src/evaluation/providers/sdk-child-runner.ts @@ -0,0 +1,156 @@ +#!/usr/bin/env node +import { + SDK_CHILD_PROTOCOL_VERSION, + type SdkChildErrorWire, + type SdkChildProviderKind, + type SdkChildRequestEnvelope, + providerRequestFromWire, + providerResponseToWire, + writeSdkChildEnvelope, +} from './sdk-child-protocol.js'; +import type { Provider, ProviderResponse } from './types.js'; + +async function main(): Promise { + const input = await readStdin(); + const envelope = parseRequest(input); + + writeSdkChildEnvelope({ + protocol_version: SDK_CHILD_PROTOCOL_VERSION, + type: 'event', + event: { + kind: 'lifecycle', + message: `starting ${envelope.provider_kind} child runner`, + }, + }); + + const provider = await createChildProvider( + envelope.provider_kind, + envelope.target_name, + envelope.config, + ); + const restoreConsole = installConsoleProtocolBridge(); + let response: ProviderResponse; + try { + response = await provider.invoke(providerRequestFromWire(envelope.request)); + } finally { + restoreConsole(); + } + + writeSdkChildEnvelope({ + protocol_version: SDK_CHILD_PROTOCOL_VERSION, + type: 'result', + response: providerResponseToWire(response), + }); +} + +function parseRequest(input: string): SdkChildRequestEnvelope { + const parsed = JSON.parse(input) as SdkChildRequestEnvelope; + if (parsed.protocol_version !== SDK_CHILD_PROTOCOL_VERSION) { + throw new Error(`Unsupported SDK child protocol version: ${String(parsed.protocol_version)}`); + } + if (!isSdkChildProviderKind(parsed.provider_kind)) { + throw new Error(`Unsupported SDK child provider: ${String(parsed.provider_kind)}`); + } + if (typeof parsed.target_name !== 'string' || parsed.target_name.trim().length === 0) { + throw new Error('SDK child request target_name is required'); + } + return parsed; +} + +async function createChildProvider( + kind: SdkChildProviderKind, + targetName: string, + config: unknown, +): Promise { + switch (kind) { + case 'codex-sdk': { + const { CodexProvider } = await import('./codex.js'); + return new CodexProvider(targetName, config as never); + } + case 'claude-sdk': { + const { ClaudeSdkProvider } = await import('./claude-sdk.js'); + return new ClaudeSdkProvider(targetName, config as never); + } + case 'copilot-sdk': { + const { CopilotSdkProvider } = await import('./copilot-sdk.js'); + return new CopilotSdkProvider(targetName, config as never); + } + case 'pi-sdk': { + const { PiCodingAgentProvider } = await import('./pi-coding-agent.js'); + return new PiCodingAgentProvider(targetName, config as never); + } + } +} + +function isSdkChildProviderKind(value: unknown): value is SdkChildProviderKind { + return ( + value === 'codex-sdk' || value === 'claude-sdk' || value === 'copilot-sdk' || value === 'pi-sdk' + ); +} + +function installConsoleProtocolBridge(): () => void { + const original = { + log: console.log, + warn: console.warn, + error: console.error, + }; + + console.log = (...args: unknown[]) => { + writeLogEvent('stdout', args); + }; + console.warn = (...args: unknown[]) => { + writeLogEvent('stderr', args); + original.warn(...args); + }; + console.error = (...args: unknown[]) => { + writeLogEvent('stderr', args); + original.error(...args); + }; + + return () => { + console.log = original.log; + console.warn = original.warn; + console.error = original.error; + }; +} + +function writeLogEvent(stream: 'stdout' | 'stderr', args: readonly unknown[]): void { + writeSdkChildEnvelope({ + protocol_version: SDK_CHILD_PROTOCOL_VERSION, + type: 'event', + event: { + kind: 'log', + stream, + message: args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' '), + }, + }); +} + +async function readStdin(): Promise { + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const input = Buffer.concat(chunks).toString('utf8').trim(); + if (!input) { + throw new Error('SDK child runner expected a JSON request on stdin'); + } + return input; +} + +function errorToWire(error: unknown): SdkChildErrorWire { + return { + code: error instanceof Error ? error.name : 'Error', + message: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + }; +} + +main().catch((error) => { + writeSdkChildEnvelope({ + protocol_version: SDK_CHILD_PROTOCOL_VERSION, + type: 'error', + error: errorToWire(error), + }); + process.exitCode = 1; +}); diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 23a6c94ed..e6171c9fd 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -827,7 +827,10 @@ export type ResolvedTarget = | (ResolvedTargetBase & { readonly kind: 'azure'; readonly config: AzureResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'anthropic'; readonly config: AnthropicResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'gemini'; readonly config: GeminiResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'codex'; readonly config: CodexResolvedConfig }) + | (ResolvedTargetBase & { + readonly kind: 'codex' | 'codex-cli' | 'codex-sdk'; + readonly config: CodexResolvedConfig; + }) | (ResolvedTargetBase & { readonly kind: 'copilot-sdk'; readonly config: CopilotSdkResolvedConfig; @@ -841,7 +844,7 @@ export type ResolvedTarget = readonly config: CopilotLogResolvedConfig; }) | (ResolvedTargetBase & { - readonly kind: 'pi-coding-agent'; + readonly kind: 'pi-sdk' | 'pi-coding-agent'; readonly config: PiCodingAgentResolvedConfig; }) | (ResolvedTargetBase & { readonly kind: 'pi-cli'; readonly config: PiCliResolvedConfig }) @@ -1079,8 +1082,10 @@ export function resolveTargetDefinition( config: resolveGeminiConfig(parsed, env), }; case 'codex': + case 'codex-cli': + case 'codex-sdk': return { - kind: 'codex', + kind: provider as 'codex' | 'codex-cli' | 'codex-sdk', ...base, config: resolveCodexConfig(parsed, env, evalFilePath), }; @@ -1102,9 +1107,10 @@ export function resolveTargetDefinition( ...base, config: resolveCopilotLogConfig(parsed, env), }; + case 'pi-sdk': case 'pi-coding-agent': return { - kind: 'pi-coding-agent', + kind: provider as 'pi-sdk' | 'pi-coding-agent', ...base, config: resolvePiCodingAgentConfig(parsed, env, evalFilePath), }; diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 8186ff112..4cf70c0cc 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -19,9 +19,12 @@ export type ProviderKind = | 'anthropic' | 'gemini' | 'codex' + | 'codex-cli' + | 'codex-sdk' | 'copilot-sdk' | 'copilot-cli' | 'copilot-log' + | 'pi-sdk' | 'pi-coding-agent' | 'pi-cli' | 'claude' @@ -45,8 +48,11 @@ export type ProviderKind = */ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ 'codex', + 'codex-cli', + 'codex-sdk', 'copilot-sdk', 'copilot-cli', + 'pi-sdk', 'pi-coding-agent', 'pi-cli', 'claude', @@ -85,9 +91,12 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ 'anthropic', 'gemini', 'codex', + 'codex-cli', + 'codex-sdk', 'copilot-sdk', 'copilot-cli', 'copilot-log', + 'pi-sdk', 'pi-coding-agent', 'pi-cli', 'claude', diff --git a/packages/core/src/types/codex-sdk.d.ts b/packages/core/src/types/codex-sdk.d.ts new file mode 100644 index 000000000..ef3d367e6 --- /dev/null +++ b/packages/core/src/types/codex-sdk.d.ts @@ -0,0 +1,3 @@ +declare module '@openai/codex-sdk' { + export const Codex: unknown; +} diff --git a/packages/core/src/types/copilot-sdk.d.ts b/packages/core/src/types/copilot-sdk.d.ts new file mode 100644 index 000000000..2aa205e77 --- /dev/null +++ b/packages/core/src/types/copilot-sdk.d.ts @@ -0,0 +1,4 @@ +declare module '@github/copilot-sdk' { + export const CopilotClient: unknown; + export const RuntimeConnection: unknown; +} diff --git a/packages/core/test/evaluation/providers/claude-provider-aliases.test.ts b/packages/core/test/evaluation/providers/claude-provider-aliases.test.ts index 0ef9ee056..e0b545fdf 100644 --- a/packages/core/test/evaluation/providers/claude-provider-aliases.test.ts +++ b/packages/core/test/evaluation/providers/claude-provider-aliases.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from 'bun:test'; import { ClaudeCliProvider } from '../../../src/evaluation/providers/claude-cli.js'; -import { ClaudeSdkProvider } from '../../../src/evaluation/providers/claude-sdk.js'; import { ClaudeProvider } from '../../../src/evaluation/providers/claude.js'; import { createBuiltinProviderRegistry } from '../../../src/evaluation/providers/index.js'; +import { SdkChildProvider } from '../../../src/evaluation/providers/sdk-child-provider.js'; const mockClaudeConfig = { executable: 'claude', @@ -41,13 +41,13 @@ describe('Claude provider alias resolution', () => { expect(provider.kind).toBe('claude-cli'); }); - it('creates a ClaudeSdkProvider for claude-sdk kind', () => { + it('creates an isolated child provider for claude-sdk kind', () => { const provider = registry.create({ name: 'test-target', kind: 'claude-sdk', config: mockClaudeConfig, }); - expect(provider).toBeInstanceOf(ClaudeSdkProvider); + expect(provider).toBeInstanceOf(SdkChildProvider); expect(provider.kind).toBe('claude-sdk'); expect(provider.id).toBe('claude-sdk:test-target'); }); diff --git a/packages/core/test/evaluation/providers/sdk-child-provider.test.ts b/packages/core/test/evaluation/providers/sdk-child-provider.test.ts new file mode 100644 index 000000000..453c1069b --- /dev/null +++ b/packages/core/test/evaluation/providers/sdk-child-provider.test.ts @@ -0,0 +1,128 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { + SdkChildProvider, + SdkChildRunnerError, +} from '../../../src/evaluation/providers/sdk-child-provider.js'; +import { extractLastAssistantContent } from '../../../src/evaluation/providers/types.js'; + +describe('SdkChildProvider', () => { + let fixturesRoot: string; + let runnerPath: string; + + beforeEach(async () => { + fixturesRoot = await mkdtemp(path.join(tmpdir(), 'agentv-sdk-child-')); + runnerPath = path.join(fixturesRoot, 'fake-sdk-runner.js'); + await writeFakeRunner(runnerPath); + }); + + afterEach(async () => { + await rm(fixturesRoot, { recursive: true, force: true }); + }); + + it('returns the final child result and keeps structured child events', async () => { + const provider = fakeProvider('success'); + + const response = await provider.invoke({ question: 'hello' }); + + expect(extractLastAssistantContent(response.output)).toBe('child ok'); + const raw = response.raw as { + child_runner?: { events?: readonly { message?: string }[]; stderr?: string }; + }; + expect(raw.child_runner?.events?.some((event) => event.message === 'fake event')).toBe(true); + expect(raw.child_runner?.stderr).toContain('stderr log'); + }); + + it('maps fatal child exit before a result to a provider-scoped error', async () => { + const provider = fakeProvider('fatal'); + + await expect(provider.invoke({ question: 'hello' })).rejects.toThrow(SdkChildRunnerError); + await expect(provider.invoke({ question: 'hello' })).rejects.toThrow(/exit code 7/); + }); + + it('maps malformed child stdout to a protocol error without crashing the parent', async () => { + const provider = fakeProvider('malformed'); + + await expect(provider.invoke({ question: 'hello' })).rejects.toThrow(/malformed_output/); + + const survivor = fakeProvider('success'); + const response = await survivor.invoke({ question: 'still alive' }); + expect(extractLastAssistantContent(response.output)).toBe('child ok'); + }); + + it('kills the child process group on timeout', async () => { + const provider = fakeProvider('hang', { timeoutMs: 50 }); + + await expect(provider.invoke({ question: 'hello' })).rejects.toThrow(/timeout/); + }); + + it('kills the child process group on cancellation', async () => { + const provider = fakeProvider('hang', { timeoutMs: 5_000 }); + const controller = new AbortController(); + const invoke = provider.invoke({ question: 'hello', signal: controller.signal }); + + setTimeout(() => controller.abort(), 30); + + await expect(invoke).rejects.toThrow(/cancelled/); + }); + + function fakeProvider(mode: string, config: Record = {}): SdkChildProvider { + return new SdkChildProvider('codex-sdk', 'fake-target', config, { + runnerArgv: [process.execPath, runnerPath, mode], + }); + } +}); + +async function writeFakeRunner(filePath: string): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile( + filePath, + ` +const mode = process.argv[2]; +await new Promise((resolve) => { + let body = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + body += chunk; + }); + process.stdin.on('end', resolve); +}); + +function write(message) { + process.stdout.write(JSON.stringify({ protocol_version: 1, ...message }) + '\\n'); +} + +if (mode === 'success') { + console.error('stderr log'); + write({ type: 'event', event: { kind: 'provider_event', message: 'fake event' } }); + write({ + type: 'result', + response: { + output: [{ role: 'assistant', content: 'child ok' }], + token_usage: { input: 1, output: 2 }, + duration_ms: 3, + }, + }); + process.exit(0); +} + +if (mode === 'fatal') { + process.stderr.write('fatal before result'); + process.exit(7); +} + +if (mode === 'malformed') { + process.stdout.write('not-json\\n'); + setTimeout(() => process.exit(0), 1000); +} + +if (mode === 'hang') { + setInterval(() => {}, 1000); +} +`, + 'utf8', + ); +} diff --git a/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts b/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts new file mode 100644 index 000000000..ce6d66163 --- /dev/null +++ b/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +import { createBuiltinProviderRegistry } from '../../../src/evaluation/providers/index.js'; +import { SdkChildProvider } from '../../../src/evaluation/providers/sdk-child-provider.js'; + +describe('SDK provider registry isolation', () => { + it('registers explicit SDK providers through the child-runner wrapper', () => { + const registry = createBuiltinProviderRegistry(); + + for (const kind of ['codex-sdk', 'claude-sdk', 'copilot-sdk', 'pi-sdk'] as const) { + const provider = registry.create({ + name: `${kind}-target`, + kind, + config: kind === 'codex-sdk' ? { executable: 'codex' } : {}, + }); + expect(provider).toBeInstanceOf(SdkChildProvider); + expect(provider.kind).toBe(kind); + } + }); + + it('keeps direct SDK provider modules out of the built-in registry imports', () => { + const source = readFileSync( + fileURLToPath(new URL('../../../src/evaluation/providers/index.ts', import.meta.url)), + 'utf8', + ); + + expect(source).not.toContain("from './codex.js'"); + expect(source).not.toContain("from './claude-sdk.js'"); + expect(source).not.toContain("from './copilot-sdk.js'"); + expect(source).not.toContain("from './pi-coding-agent.js'"); + }); +}); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 18234c153..b30e90746 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -918,6 +918,20 @@ describe('resolveTargetDefinition', () => { }); }); + it('resolves codex-sdk as an explicit SDK provider kind', () => { + const target = resolveTargetDefinition({ + name: 'codex-sdk-target', + provider: 'codex-sdk', + model: 'gpt-5-codex', + }); + + expect(target.kind).toBe('codex-sdk'); + if (target.kind !== 'codex-sdk') { + throw new Error('expected codex-sdk target'); + } + expect(target.config.model).toBe('gpt-5-codex'); + }); + it('rejects unsupported codex reasoning_effort values', () => { expect(() => resolveTargetDefinition( @@ -1578,6 +1592,21 @@ describe('createProvider', () => { expect(resolved.config.thinking).toBe('medium'); }); + it('resolves pi-sdk as an explicit SDK provider kind', () => { + const resolved = resolveTargetDefinition( + { + name: 'pi-sdk-agent', + provider: 'pi-sdk', + model: 'gpt-5.5', + }, + {}, + ); + + expect(resolved.kind).toBe('pi-sdk'); + if (resolved.kind !== 'pi-sdk') throw new Error('expected pi-sdk'); + expect(resolved.config.model).toBe('gpt-5.5'); + }); + it('resolves pi-cli with azure subprovider and base_url', () => { const env = { AZURE_OPENAI_ENDPOINT: 'https://my-resource.openai.azure.com', diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 7b18bdf23..4fd7ceabd 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -1,7 +1,11 @@ import { defineConfig } from 'tsup'; export default defineConfig({ - entry: ['src/index.ts', 'src/evaluation/validation/index.ts'], + entry: [ + 'src/index.ts', + 'src/evaluation/validation/index.ts', + 'src/evaluation/providers/sdk-child-runner.ts', + ], format: ['esm', 'cjs'], shims: true, sourcemap: true, @@ -14,7 +18,13 @@ export default defineConfig({ }, target: 'node20', tsconfig: './tsconfig.build.json', - external: ['@earendil-works/pi-coding-agent', '@earendil-works/pi-ai'], + external: [ + '@anthropic-ai/claude-agent-sdk', + '@github/copilot-sdk', + '@openai/codex-sdk', + '@earendil-works/pi-coding-agent', + '@earendil-works/pi-ai', + ], outExtension({ format }) { return { js: format === 'cjs' ? '.cjs' : '.js', From b934a158dd8e57ac43c80dae187d1c6096bec3d3 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 17:22:56 +0200 Subject: [PATCH 04/10] feat(providers): add explicit codex runtime targets --- .../docs/next/evaluation/running-evals.mdx | 12 +- .../docs/docs/next/targets/coding-agents.mdx | 22 +- .../content/docs/docs/next/tools/import.mdx | 2 +- .../content/docs/docs/next/tools/prepare.mdx | 2 +- .../docs/v4.42.4/evaluation/running-evals.mdx | 14 +- .../docs/v4.42.4/targets/coding-agents.mdx | 13 +- .../docs/v4.42.4/targets/configuration.mdx | 8 +- .../docs/docs/v4.42.4/tools/import.mdx | 2 +- .../docs/docs/v4.42.4/tools/prepare.mdx | 2 +- .../multi-provider-skill-trigger.EVAL.yaml | 2 +- .../src/evaluation/providers/codex-cli.ts | 500 ++++++++++++++++-- .../core/src/evaluation/providers/codex.ts | 15 +- .../core/src/evaluation/providers/index.ts | 4 +- .../core/src/evaluation/providers/targets.ts | 193 ++++++- .../core/src/evaluation/providers/types.ts | 9 +- .../core/src/evaluation/transcript-summary.ts | 7 +- .../validation/targets-validator.ts | 8 +- packages/core/src/import/codex-parser.ts | 4 +- .../evaluation/graders/skill-trigger.test.ts | 2 +- .../core/test/evaluation/orchestrator.test.ts | 10 +- .../test/evaluation/providers/codex.test.ts | 173 +++++- .../providers/normalize-tool-call.test.ts | 9 +- .../providers/sdk-provider-registry.test.ts | 2 +- .../evaluation/providers/targets-file.test.ts | 6 +- .../test/evaluation/providers/targets.test.ts | 69 ++- .../validation/eval-file-schema.test.ts | 3 +- .../validation/eval-validator.test.ts | 3 +- .../validation/targets-validator.test.ts | 21 +- 28 files changed, 930 insertions(+), 187 deletions(-) diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index 05d1785dc..ecaa07778 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -611,12 +611,14 @@ Then add a replay target alias in `.agentv/targets.yaml`: ```yaml targets: - - label: live_coding_agent - provider: codex - model: gpt-5 - grader_target: grader_gpt_5_mini + - id: live_coding_agent + provider: codex-cli + runtime: host + config: + command: ["codex"] + model: gpt-5 - - label: replay_coding_agent + - id: replay_coding_agent provider: replay fixtures: ../fixtures/legal-review-target-output.jsonl source_target: live_coding_agent diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index e130212b8..b25dda3db 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -5,7 +5,7 @@ sidebar: order: 3 --- -Coding agent targets evaluate AI coding assistants and CLI-based agents. These targets require a `grader_target` to run LLM-based graders. +Coding agent targets evaluate AI coding assistants and CLI-based agents. Use `defaults.grader` or an evaluator-level grader override for LLM-based grading. AgentV's recommended coding-agent targets use process or protocol boundaries such as `claude-cli`, `codex-cli`, `copilot-cli`, and `pi-cli`. SDK-backed targets are opt-in provider kinds: `claude-sdk`, `codex-sdk`, `copilot-sdk`, and `pi-sdk`. When selected, AgentV runs the SDK adapter in an AgentV-owned child process and communicates with it through a structured protocol, so SDK crashes, malformed output, timeouts, and missing optional SDK packages are scoped to that target run instead of the AgentV orchestrator process. @@ -79,21 +79,21 @@ targets: ```yaml targets: - - label: codex_target + - id: codex-target provider: codex-cli - executable: codex-eng - model: ${{ CODEX_MODEL }} - reasoning_effort: ${{ CODEX_REASONING_EFFORT }} - grader_target: azure-base + runtime: host + config: + command: ["codex-eng"] + model: ${{ CODEX_MODEL }} + reasoning_effort: ${{ CODEX_REASONING_EFFORT }} ``` | Field | Required | Description | |-------|----------|-------------| -| `executable` | No | Codex binary or profile shim to run, such as `codex-eng` | -| `model` | No | Model to use | -| `reasoning_effort` | No | Codex SDK reasoning effort: `minimal`, `low`, `medium`, `high`, or `xhigh` | -| `cwd` | No | Working directory | -| `grader_target` | Yes | LLM target for evaluation | +| `config.command` | Yes | Codex binary or profile shim argv, such as `["codex-eng"]` | +| `config.model` | No | Model to use | +| `config.reasoning_effort` | No | Codex reasoning effort: `minimal`, `low`, `medium`, `high`, or `xhigh` | +| `config.cwd` | No | Working directory | Use `provider: codex-sdk` only when you intentionally want the Codex SDK path. The SDK package is optional and loaded inside the isolated child runner. diff --git a/apps/web/src/content/docs/docs/next/tools/import.mdx b/apps/web/src/content/docs/docs/next/tools/import.mdx index 6184971e0..c0a7f8bbc 100644 --- a/apps/web/src/content/docs/docs/next/tools/import.mdx +++ b/apps/web/src/content/docs/docs/next/tools/import.mdx @@ -231,7 +231,7 @@ uv run scripts/import-huggingface.py \ --output /tmp/swebench-eval/ # Run with a coding agent target -agentv eval /tmp/swebench-eval/*.EVAL.yaml --target codex +agentv eval /tmp/swebench-eval/*.EVAL.yaml --target codex-cli ``` The Docker workspace spins up the pre-built SWE-bench image, checks out the imported `commit`, runs the agent to apply a patch, then grades by running the test suite inside the container. diff --git a/apps/web/src/content/docs/docs/next/tools/prepare.mdx b/apps/web/src/content/docs/docs/next/tools/prepare.mdx index 2469e59ba..a42a63f25 100644 --- a/apps/web/src/content/docs/docs/next/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/next/tools/prepare.mdx @@ -10,7 +10,7 @@ sidebar: This is the manual-attempt workflow: ```bash -agentv prepare evals/foo.eval.yaml --test-id case-1 --target codex --out /tmp/agentv-case-1 +agentv prepare evals/foo.eval.yaml --test-id case-1 --target codex-cli --out /tmp/agentv-case-1 ``` The prepared directory contains: diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx index d3ecc6828..1fe98c7f0 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx @@ -481,12 +481,14 @@ Then add a replay target alias in `.agentv/targets.yaml`: ```yaml targets: - - name: live_coding_agent - provider: codex - model: gpt-5 - grader_target: grader_gpt_5_mini - - - name: replay_coding_agent + - id: live_coding_agent + provider: codex-cli + runtime: host + config: + command: ["codex"] + model: gpt-5 + + - id: replay_coding_agent provider: replay fixtures: ../fixtures/legal-review-target-output.jsonl source_target: live_coding_agent diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx index 621861291..b936d406b 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx @@ -80,12 +80,13 @@ targets: ```yaml targets: - - name: codex_target - provider: codex - executable: codex-eng - model: ${{ CODEX_MODEL }} - model_reasoning_effort: ${{ CODEX_REASONING_EFFORT }} - grader_target: azure-base + - id: codex_target + provider: codex-cli + runtime: host + config: + command: ["codex-eng"] + model: ${{ CODEX_MODEL }} + model_reasoning_effort: ${{ CODEX_REASONING_EFFORT }} ``` | Field | Required | Description | diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx index ba2fb4b1f..a8fb01afb 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx @@ -86,9 +86,11 @@ Agent targets that need LLM-based evaluation specify a `grader_target` (also acc ```yaml targets: - - name: codex_target - provider: codex - grader_target: azure-base # LLM used for grading + - id: codex_target + provider: codex-cli + runtime: host + config: + command: ["codex"] ``` ### Workspace Lifecycle Hooks diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx index fde465dbc..d9f4d8890 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/import.mdx @@ -239,7 +239,7 @@ uv run scripts/import-huggingface.py \ --output /tmp/swebench-eval/ # Run with a coding agent target -agentv eval /tmp/swebench-eval/*.EVAL.yaml --target codex +agentv eval /tmp/swebench-eval/*.EVAL.yaml --target codex-cli ``` The Docker workspace spins up the pre-built SWE-bench image, checks out `base_commit`, runs the agent to apply a patch, then grades by running the test suite inside the container. diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx index 94a83b7e4..9df429646 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx @@ -13,7 +13,7 @@ pagefind: false This is the manual-attempt workflow: ```bash -agentv prepare evals/foo.eval.yaml --test-id case-1 --target codex --out /tmp/agentv-case-1 +agentv prepare evals/foo.eval.yaml --test-id case-1 --target codex-cli --out /tmp/agentv-case-1 ``` The prepared directory contains: diff --git a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml index dbe313a5a..6ee9c1660 100644 --- a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml +++ b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml @@ -14,7 +14,7 @@ # # agentv eval this-file.EVAL.yaml --target claude --targets ../.agentv/targets.yaml # agentv eval this-file.EVAL.yaml --target copilot --targets ../.agentv/targets.yaml -# agentv eval this-file.EVAL.yaml --target codex --targets ../.agentv/targets.yaml +# agentv eval this-file.EVAL.yaml --target codex-cli --targets ../.agentv/targets.yaml # # The grader automatically resolves the correct tool names for each # provider. No provider-specific config needed in test cases. diff --git a/packages/core/src/evaluation/providers/codex-cli.ts b/packages/core/src/evaluation/providers/codex-cli.ts index 82b842b9e..283ebcea9 100644 --- a/packages/core/src/evaluation/providers/codex-cli.ts +++ b/packages/core/src/evaluation/providers/codex-cli.ts @@ -1,9 +1,10 @@ import { exec as execCallback, spawn } from 'node:child_process'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { constants, createWriteStream } from 'node:fs'; import type { WriteStream } from 'node:fs'; import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import path from 'node:path'; import { promisify } from 'node:util'; @@ -12,7 +13,16 @@ import { recordCodexLogEntry } from './codex-log-tracker.js'; import { resolveDefaultProviderLogDir } from './log-directory.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; import type { CodexResolvedConfig } from './targets.js'; -import type { Provider, ProviderRequest, ProviderResponse } from './types.js'; +import type { + Message, + Provider, + ProviderRequest, + ProviderResponse, + TargetExecutionEnvelope, + TargetExecutionErrorKind, + TargetExecutionLogCapture, +} from './types.js'; +import { extractLastAssistantContent } from './types.js'; const execAsync = promisify(execCallback); const WORKSPACE_PREFIX = 'agentv-codex-'; @@ -34,15 +44,17 @@ interface CodexRunOptions { interface CodexRunResult { readonly stdout: string; readonly stderr: string; - readonly exitCode: number; + readonly exitCode: number | null; + readonly signal?: NodeJS.Signals | null; readonly timedOut?: boolean; + readonly cancelled?: boolean; } type CodexRunner = (options: CodexRunOptions) => Promise; export class CodexCliProvider implements Provider { readonly id: string; - readonly kind = 'codex' as const; + readonly kind: 'codex-cli' | 'codex-app-server'; readonly targetName: string; readonly supportsBatch = false; @@ -55,8 +67,10 @@ export class CodexCliProvider implements Provider { targetName: string, config: CodexResolvedConfig, runner: CodexRunner = defaultCodexRunner, + kind: 'codex-cli' | 'codex-app-server' = 'codex-cli', ) { - this.id = `codex:${targetName}`; + this.kind = kind; + this.id = `${kind}:${targetName}`; this.targetName = targetName; this.config = config; this.runCodex = runner; @@ -64,12 +78,19 @@ export class CodexCliProvider implements Provider { async invoke(request: ProviderRequest): Promise { if (request.signal?.aborted) { - throw new Error('Codex provider request was aborted before execution'); + return this.buildUnsupportedRuntimeOrEarlyCancel('cancelled', 'Codex request was cancelled'); + } + + if (this.config.runtime.mode === 'sandbox') { + return this.buildUnsupportedRuntimeOrEarlyCancel( + 'sandbox_infra_failure', + 'runtime.mode: sandbox for Codex targets is not available in this AgentV build. Use runtime: host/profile or the sandbox runtime provider once configured.', + ); } await this.ensureEnvironmentReady(); - const inputFiles = normalizeInputFiles(request.inputFiles); + const inputFiles = normalizeInputFiles(request.inputFiles) ?? []; const workspaceRoot = await this.createWorkspace(); const logger = await this.createStreamLogger(request).catch(() => undefined); @@ -82,23 +103,109 @@ export class CodexCliProvider implements Provider { const args = this.buildCodexArgs(); const cwd = this.resolveCwd(workspaceRoot, request.cwd); - - const result = await this.executeCodex(args, cwd, promptContent, request.signal, logger); + const env = await this.buildProcessEnv(workspaceRoot); + const startedAt = Date.now(); + let result: CodexRunResult; + try { + result = await this.executeCodex( + args, + cwd, + this.buildStdinPayload(promptContent, request), + env, + request.signal, + logger, + ); + } catch (error) { + const message = formatError(error); + return this.buildErrorResponse({ + errorKind: 'spawn_failure', + message, + args, + cwd, + startedAt, + endedAt: Date.now(), + stdout: '', + stderr: message, + inputFiles, + promptFile, + workspaceRoot, + logFile: logger?.filePath, + }); + } if (result.timedOut) { - throw new Error( - `Codex CLI timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`, - ); + return this.buildErrorResponse({ + errorKind: 'timeout', + message: `Codex ${this.providerLabel()} timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`, + args, + cwd, + startedAt, + endedAt: Date.now(), + result, + inputFiles, + promptFile, + workspaceRoot, + logFile: logger?.filePath, + }); + } + + if (result.cancelled || request.signal?.aborted) { + return this.buildErrorResponse({ + errorKind: 'cancelled', + message: `Codex ${this.providerLabel()} request was cancelled`, + args, + cwd, + startedAt, + endedAt: Date.now(), + result, + inputFiles, + promptFile, + workspaceRoot, + logFile: logger?.filePath, + }); } if (result.exitCode !== 0) { const detail = pickDetail(result.stderr, result.stdout); - const prefix = `Codex CLI exited with code ${result.exitCode}`; - throw new Error(detail ? `${prefix}: ${detail}` : prefix); + const prefix = `Codex ${this.providerLabel()} exited with code ${result.exitCode}`; + return this.buildErrorResponse({ + errorKind: result.signal ? 'signal_crash' : 'nonzero_exit', + message: detail ? `${prefix}: ${detail}` : prefix, + args, + cwd, + startedAt, + endedAt: Date.now(), + result, + inputFiles, + promptFile, + workspaceRoot, + logFile: logger?.filePath, + }); } - const parsed = parseCodexJson(result.stdout); - const assistantText = extractAssistantText(parsed); + let parsed: unknown; + let messages: readonly Message[]; + try { + parsed = parseCodexJson(result.stdout); + messages = extractAssistantMessages(parsed); + } catch (error) { + const message = formatError(error); + return this.buildErrorResponse({ + errorKind: 'malformed_output', + message, + args, + cwd, + startedAt, + endedAt: Date.now(), + result, + inputFiles, + promptFile, + workspaceRoot, + logFile: logger?.filePath, + }); + } + const assistantText = extractLastAssistantContent(messages); + const durationMs = Date.now() - startedAt; return { raw: { @@ -107,13 +214,34 @@ export class CodexCliProvider implements Provider { stderr: result.stderr, exitCode: result.exitCode, args, - executable: this.resolvedExecutable ?? this.config.executable, + executable: this.resolvedExecutable ?? this.config.command?.[0], promptFile, workspace: workspaceRoot, inputFiles, logFile: logger?.filePath, }, - output: [{ role: 'assistant' as const, content: assistantText }], + output: messages, + durationMs, + targetExecution: { + ...this.buildEnvelopeBase({ + args, + cwd, + startedAt, + endedAt: Date.now(), + result, + }), + status: 'success', + transcript: { + messages, + finalOutput: assistantText, + }, + details: { + promptFile, + workspace: workspaceRoot, + inputFiles, + logFile: logger?.filePath, + }, + }, }; } finally { await logger?.close(); @@ -129,7 +257,7 @@ export class CodexCliProvider implements Provider { } private async validateEnvironment(): Promise { - this.resolvedExecutable = await locateExecutable(this.config.executable); + this.resolvedExecutable = await locateExecutable(this.commandExecutable()); } private resolveCwd(workspaceRoot: string, cwdOverride?: string): string { @@ -144,19 +272,28 @@ export class CodexCliProvider implements Provider { } private buildCodexArgs(): string[] { - // Global flags must come before 'exec' subcommand - const args = [ - '--ask-for-approval', - 'never', - 'exec', - '--json', - '--color', - 'never', - '--skip-git-repo-check', - ]; - if (this.config.args && this.config.args.length > 0) { - args.push(...this.config.args); + const [, ...configuredArgs] = this.config.command ?? []; + if (this.kind === 'codex-app-server') { + return configuredArgs; } + + const args = [...configuredArgs]; + if (this.config.model) { + args.push('--model', this.config.model); + } + if (this.config.modelReasoningEffort) { + args.push('--config', `model_reasoning_effort=${this.config.modelReasoningEffort}`); + } + if (this.config.modelVerbosity) { + args.push('--config', `model_verbosity=${this.config.modelVerbosity}`); + } + if (this.config.sandboxMode) { + args.push('--sandbox', this.config.sandboxMode); + } + if (this.config.approvalPolicy) { + args.push('--ask-for-approval', this.config.approvalPolicy); + } + args.push('exec', '--json', '--color', 'never', '--skip-git-repo-check'); args.push('-'); return args; } @@ -164,37 +301,208 @@ export class CodexCliProvider implements Provider { private async executeCodex( args: readonly string[], cwd: string, - promptContent: string, + stdinPayload: string, + env: NodeJS.ProcessEnv, signal: AbortSignal | undefined, logger: CodexStreamLogger | undefined, ): Promise { - try { - return await this.runCodex({ - executable: this.resolvedExecutable ?? this.config.executable, - args, - cwd, - prompt: promptContent, - timeoutMs: this.config.timeoutMs, - env: process.env, - signal, - onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : undefined, - onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : undefined, - }); - } catch (error) { - const err = error as NodeJS.ErrnoException; - if (err.code === 'ENOENT') { - throw new Error( - `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`, - ); - } - throw error; + return await this.runCodex({ + executable: this.resolvedExecutable ?? this.commandExecutable(), + args, + cwd, + prompt: stdinPayload, + timeoutMs: this.config.timeoutMs, + env, + signal, + onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : undefined, + onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : undefined, + }); + } + + private buildStdinPayload(promptContent: string, request: ProviderRequest): string { + if (this.kind !== 'codex-app-server') { + return promptContent; } + return `${JSON.stringify({ + type: 'agentv.invoke', + question: request.question, + prompt: promptContent, + eval_case_id: request.evalCaseId, + attempt: request.attempt, + })}\n`; } private async createWorkspace(): Promise { return await mkdtemp(path.join(tmpdir(), WORKSPACE_PREFIX)); } + private commandExecutable(): string { + const executable = this.config.command?.[0]; + if (!executable) { + throw new Error(`Codex ${this.providerLabel()} requires config.command`); + } + return executable; + } + + private providerLabel(): string { + return this.kind === 'codex-app-server' ? 'app-server' : 'CLI'; + } + + private async buildProcessEnv(workspaceRoot: string): Promise { + if (this.config.runtime.mode === 'host') { + return process.env; + } + + const runtime = this.config.runtime; + const env: NodeJS.ProcessEnv = {}; + const allowlist = runtime.envAllowlist ?? defaultProfileEnvAllowlist(); + for (const key of allowlist) { + const value = process.env[key]; + if (value !== undefined) { + env[key] = value; + } + } + + const profileHome = path.resolve(runtime.home ?? path.join(workspaceRoot, 'profile-home')); + const codexHome = path.resolve(runtime.codexHome ?? path.join(profileHome, '.codex')); + const tmpRoot = path.resolve(runtime.tmpDir ?? path.join(profileHome, '.tmp')); + await mkdir(codexHome, { recursive: true }); + await mkdir(tmpRoot, { recursive: true }); + + env.HOME = profileHome; + env.USERPROFILE = profileHome; + env.CODEX_HOME = codexHome; + env.TMPDIR = tmpRoot; + env.TMP = tmpRoot; + env.TEMP = tmpRoot; + for (const [key, value] of Object.entries(runtime.env ?? {})) { + env[key] = value; + } + return env; + } + + private buildUnsupportedRuntimeOrEarlyCancel( + errorKind: TargetExecutionErrorKind, + message: string, + ): ProviderResponse { + const now = Date.now(); + return { + output: [{ role: 'assistant', content: `Error: ${message}` }], + durationMs: 0, + targetExecution: { + ...this.buildEnvelopeBase({ + args: this.config.command?.slice(1) ?? [], + cwd: this.config.cwd, + startedAt: now, + endedAt: now, + result: { stdout: '', stderr: '', exitCode: null }, + }), + status: 'error', + errorKind, + message, + transcript: { + messages: [{ role: 'assistant', content: `Error: ${message}` }], + finalOutput: `Error: ${message}`, + }, + }, + raw: { error: message }, + }; + } + + private buildErrorResponse(params: { + readonly errorKind: TargetExecutionErrorKind; + readonly message: string; + readonly args: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly endedAt: number; + readonly result?: CodexRunResult; + readonly stdout?: string; + readonly stderr?: string; + readonly inputFiles: readonly string[]; + readonly promptFile: string; + readonly workspaceRoot: string; + readonly logFile?: string; + }): ProviderResponse { + const output = [{ role: 'assistant' as const, content: `Error: ${params.message}` }]; + return { + output, + durationMs: params.endedAt - params.startedAt, + targetExecution: { + ...this.buildEnvelopeBase({ + args: params.args, + cwd: params.cwd, + startedAt: params.startedAt, + endedAt: params.endedAt, + result: params.result ?? { + stdout: params.stdout ?? '', + stderr: params.stderr ?? '', + exitCode: null, + }, + }), + status: 'error', + errorKind: params.errorKind, + message: params.message, + transcript: { + messages: output, + finalOutput: `Error: ${params.message}`, + }, + details: { + promptFile: params.promptFile, + workspace: params.workspaceRoot, + inputFiles: params.inputFiles, + logFile: params.logFile, + }, + }, + raw: { + stderr: params.result?.stderr ?? params.stderr ?? '', + stdout: params.result?.stdout ?? params.stdout ?? '', + exitCode: params.result?.exitCode ?? null, + signal: params.result?.signal ?? null, + cwd: params.cwd, + args: params.args, + executable: this.resolvedExecutable ?? this.config.command?.[0], + error: params.message, + logFile: params.logFile, + }, + }; + } + + private buildEnvelopeBase(params: { + readonly args: readonly string[]; + readonly cwd?: string; + readonly startedAt: number; + readonly endedAt: number; + readonly result?: CodexRunResult; + }): Omit { + const executable = this.resolvedExecutable ?? this.config.command?.[0] ?? 'codex'; + const argv = [executable, ...params.args]; + const stdout = params.result?.stdout ?? ''; + const stderr = params.result?.stderr ?? ''; + return { + schemaVersion: 'agentv.target_execution.v1', + targetId: this.targetName, + providerId: this.id, + providerKind: this.kind, + runtimeMode: this.config.runtime.mode, + command: { + argv, + commandLine: argv.map(shellQuote).join(' '), + cwd: params.cwd, + }, + timeoutMs: this.config.timeoutMs, + startedAt: new Date(params.startedAt).toISOString(), + endedAt: new Date(params.endedAt).toISOString(), + durationMs: params.endedAt - params.startedAt, + exitCode: params.result?.exitCode, + signal: params.result?.signal ?? null, + logs: { + stdout: captureLog(stdout), + stderr: captureLog(stderr), + }, + }; + } + private async cleanupWorkspace(workspaceRoot: string): Promise { try { await rm(workspaceRoot, { recursive: true, force: true }); @@ -254,6 +562,16 @@ export class CodexCliProvider implements Provider { } } +export class CodexAppServerProvider extends CodexCliProvider { + constructor( + targetName: string, + config: CodexResolvedConfig, + runner: CodexRunner = defaultCodexRunner, + ) { + super(targetName, config, runner, 'codex-app-server'); + } +} + class CodexStreamLogger { readonly filePath: string; private readonly stream: WriteStream; @@ -640,6 +958,11 @@ function extractAssistantText(parsed: unknown): string { throw new Error('Codex CLI JSON response did not include an assistant message'); } +function extractAssistantMessages(parsed: unknown): readonly Message[] { + const text = extractAssistantText(parsed); + return [{ role: 'assistant', content: text }]; +} + function extractFromEventStream(events: readonly unknown[]): string | undefined { for (let index = events.length - 1; index >= 0; index -= 1) { const candidate = events[index]; @@ -749,12 +1072,54 @@ function formatTimeoutSuffix(timeoutMs: number | undefined): string { return ` after ${seconds}s`; } +const INLINE_LOG_LIMIT_BYTES = 128 * 1024; + +function captureLog(text: string): TargetExecutionLogCapture { + const bytes = Buffer.byteLength(text, 'utf8'); + if (bytes <= INLINE_LOG_LIMIT_BYTES) { + return { + text, + truncated: false, + bytes, + storedBytes: bytes, + }; + } + let stored = text; + while (Buffer.byteLength(stored, 'utf8') > INLINE_LOG_LIMIT_BYTES) { + stored = stored.slice(0, Math.max(0, stored.length - 1024)); + } + return { + text: stored, + truncated: true, + bytes, + storedBytes: Buffer.byteLength(stored, 'utf8'), + }; +} + +function defaultProfileEnvAllowlist(): readonly string[] { + return process.platform === 'win32' + ? ['PATH', 'Path', 'PATHEXT', 'SystemRoot', 'COMSPEC', 'LANG', 'LC_ALL', 'NO_COLOR'] + : ['PATH', 'LANG', 'LC_ALL', 'TERM', 'NO_COLOR', 'SHELL']; +} + +function shellQuote(value: string): string { + if (/^[A-Za-z0-9_/:=.,@%+-]+$/.test(value)) { + return value; + } + return `'${value.replaceAll("'", `'\\''`)}'`; +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + async function defaultCodexRunner(options: CodexRunOptions): Promise { return await new Promise((resolve, reject) => { const child = spawn(options.executable, options.args, { cwd: options.cwd, env: options.env, stdio: ['pipe', 'pipe', 'pipe'], + detached: process.platform !== 'win32', shell: shouldShellExecute(options.executable), }); trackChild(child); @@ -762,9 +1127,12 @@ async function defaultCodexRunner(options: CodexRunOptions): Promise { - child.kill('SIGTERM'); + cancelled = true; + terminateChild(child, 'SIGTERM'); + setTimeout(() => terminateChild(child, 'SIGKILL'), 2_000).unref?.(); }; if (options.signal) { @@ -779,7 +1147,8 @@ async function defaultCodexRunner(options: CodexRunOptions): Promise 0) { timeoutHandle = setTimeout(() => { timedOut = true; - child.kill('SIGTERM'); + terminateChild(child, 'SIGTERM'); + setTimeout(() => terminateChild(child, 'SIGKILL'), 2_000).unref?.(); }, options.timeoutMs); timeoutHandle.unref?.(); } @@ -812,18 +1181,37 @@ async function defaultCodexRunner(options: CodexRunOptions): Promise { + child.on('close', (code, signal) => { cleanup(); resolve({ stdout, stderr, - exitCode: typeof code === 'number' ? code : -1, + exitCode: typeof code === 'number' ? code : null, + signal, timedOut, + cancelled, }); }); }); } +function terminateChild(child: ChildProcessWithoutNullStreams, signal: NodeJS.Signals): void { + if (child.pid === undefined) { + return; + } + try { + if (process.platform === 'win32') { + child.kill(signal); + } else { + process.kill(-child.pid, signal); + } + } catch { + try { + child.kill(signal); + } catch {} + } +} + function shouldShellExecute(executable: string): boolean { if (process.platform !== 'win32') { return false; diff --git a/packages/core/src/evaluation/providers/codex.ts b/packages/core/src/evaluation/providers/codex.ts index ee8669843..ce7e0b147 100644 --- a/packages/core/src/evaluation/providers/codex.ts +++ b/packages/core/src/evaluation/providers/codex.ts @@ -45,14 +45,14 @@ async function loadCodexSdk(): Promise { */ export class CodexProvider implements Provider { readonly id: string; - readonly kind = 'codex' as const; + readonly kind = 'codex-sdk' as const; readonly targetName: string; readonly supportsBatch = false; private readonly config: CodexResolvedConfig; constructor(targetName: string, config: CodexResolvedConfig) { - this.id = `codex:${targetName}`; + this.id = `codex-sdk:${targetName}`; this.targetName = targetName; this.config = config; } @@ -72,8 +72,9 @@ export class CodexProvider implements Provider { // Build Codex SDK options // biome-ignore lint/suspicious/noExplicitAny: SDK constructor options are dynamic const codexOptions: any = {}; - if (this.config.executable) { - codexOptions.codexPathOverride = this.config.executable; + const codexCommand = this.config.command?.[0]; + if (codexCommand) { + codexOptions.codexPathOverride = codexCommand; } if (this.config.apiKey) { codexOptions.apiKey = this.config.apiKey; @@ -280,7 +281,7 @@ export class CodexProvider implements Provider { if (itemType === 'command_execution') { completedToolCalls.push( - normalizeToolCall('codex', { + normalizeToolCall('codex-sdk', { tool: 'command_execution', input: { command: item.command }, output: item.aggregated_output, @@ -291,7 +292,7 @@ export class CodexProvider implements Provider { if (itemType === 'file_change') { completedToolCalls.push( - normalizeToolCall('codex', { + normalizeToolCall('codex-sdk', { tool: 'file_change', input: item.changes, id: item.id, @@ -301,7 +302,7 @@ export class CodexProvider implements Provider { if (itemType === 'mcp_tool_call') { completedToolCalls.push( - normalizeToolCall('codex', { + normalizeToolCall('codex-sdk', { tool: `mcp:${item.server}/${item.tool}`, input: item.arguments, output: item.result ?? item.error, diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index ab93a43b9..83800a1e5 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -1,7 +1,7 @@ import { AgentvProvider } from './agentv-provider.js'; import { ClaudeCliProvider } from './claude-cli.js'; import { CliProvider } from './cli.js'; -import { CodexCliProvider } from './codex-cli.js'; +import { CodexAppServerProvider, CodexCliProvider } from './codex-cli.js'; import { CopilotCliProvider } from './copilot-cli.js'; import { CopilotLogProvider } from './copilot-log.js'; import { @@ -107,8 +107,8 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { .register('anthropic', (t) => new AnthropicProvider(t.name, t.config as never)) .register('gemini', (t) => new GeminiProvider(t.name, t.config as never)) .register('cli', (t) => new CliProvider(t.name, t.config as never)) - .register('codex', (t) => new CodexCliProvider(t.name, t.config as never)) .register('codex-cli', (t) => new CodexCliProvider(t.name, t.config as never)) + .register('codex-app-server', (t) => new CodexAppServerProvider(t.name, t.config as never)) .register('codex-sdk', (t) => new SdkChildProvider('codex-sdk', t.name, t.config)) .register('copilot-sdk', (t) => new SdkChildProvider('copilot-sdk', t.name, t.config)) .register('copilot-cli', (t) => new CopilotCliProvider(t.name, t.config as never)) diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index e6171c9fd..f18eb4dc4 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -429,8 +429,8 @@ export interface CodexResolvedConfig { readonly apiFormat?: ApiFormat; readonly sandboxMode?: CodexSandboxMode; readonly approvalPolicy?: CodexApprovalPolicy; - readonly executable: string; - readonly args?: readonly string[]; + readonly command?: readonly string[]; + readonly runtime: CodingAgentRuntimeConfig; readonly cwd?: string; readonly timeoutMs?: number; readonly logDir?: string; @@ -439,6 +439,17 @@ export interface CodexResolvedConfig { readonly systemPrompt?: string; } +export type CodingAgentRuntimeMode = 'host' | 'profile' | 'sandbox'; + +export interface CodingAgentRuntimeConfig { + readonly mode: CodingAgentRuntimeMode; + readonly home?: string; + readonly codexHome?: string; + readonly tmpDir?: string; + readonly env?: Readonly>; + readonly envAllowlist?: readonly string[]; +} + export interface CopilotCliResolvedConfig { readonly executable: string; readonly model?: string; @@ -828,7 +839,7 @@ export type ResolvedTarget = | (ResolvedTargetBase & { readonly kind: 'anthropic'; readonly config: AnthropicResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'gemini'; readonly config: GeminiResolvedConfig }) | (ResolvedTargetBase & { - readonly kind: 'codex' | 'codex-cli' | 'codex-sdk'; + readonly kind: 'codex-cli' | 'codex-app-server' | 'codex-sdk'; readonly config: CodexResolvedConfig; }) | (ResolvedTargetBase & { @@ -1082,12 +1093,16 @@ export function resolveTargetDefinition( config: resolveGeminiConfig(parsed, env), }; case 'codex': + throw new Error( + `Target "${parsed.name}" uses ambiguous provider 'codex'. Choose 'codex-cli', 'codex-app-server', or 'codex-sdk'.`, + ); case 'codex-cli': + case 'codex-app-server': case 'codex-sdk': return { - kind: provider as 'codex' | 'codex-cli' | 'codex-sdk', + kind: provider as 'codex-cli' | 'codex-app-server' | 'codex-sdk', ...base, - config: resolveCodexConfig(parsed, env, evalFilePath), + config: resolveCodexConfig(parsed, env, provider, evalFilePath), }; case 'copilot-sdk': return { @@ -1372,6 +1387,7 @@ function resolveGeminiConfig( function resolveCodexConfig( target: z.infer, env: EnvLookup, + provider: string, _evalFilePath?: string, ): CodexResolvedConfig { const modelSource = target.model; @@ -1382,12 +1398,18 @@ function resolveCodexConfig( const apiFormatSource = target.api_format; const sandboxModeSource = target.sandbox_mode; const approvalPolicySource = target.approval_policy; - const executableSource = target.executable ?? target.command ?? target.binary; - const argsSource = target.args ?? target.arguments; const cwdSource = target.cwd; const timeoutSource = target.timeout_seconds; const logDirSource = target.log_dir ?? target.log_directory; const systemPromptSource = target.system_prompt; + const runtime = resolveCodingAgentRuntime(target.runtime, env, target.name); + + if (provider === 'codex') { + throw new Error( + `Target "${target.name}" uses ambiguous provider 'codex'. Choose 'codex-cli', 'codex-app-server', or 'codex-sdk'.`, + ); + } + assertNoCodexProcessFieldAliases(target, provider); const streamLogResult = resolveStreamLog({ name: target.name, stream_log: target.stream_log }); @@ -1439,13 +1461,10 @@ function resolveCodexConfig( }), ); - const executable = - resolveOptionalString(executableSource, env, `${target.name} codex executable`, { - allowLiteral: true, - optionalEnv: true, - }) ?? 'codex'; - - const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`); + const command = + provider === 'codex-sdk' + ? resolveOptionalCommandArgv(target.command, env, `${target.name} codex command`) + : resolveRequiredCommandArgv(target.command, env, `${target.name} codex command`); const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, { allowLiteral: true, @@ -1472,8 +1491,8 @@ function resolveCodexConfig( apiFormat, sandboxMode, approvalPolicy, - executable, - args, + command, + runtime, cwd, timeoutMs, logDir, @@ -1482,6 +1501,148 @@ function resolveCodexConfig( }; } +function assertNoCodexProcessFieldAliases( + target: z.infer, + provider: string, +): void { + if (provider === 'codex-sdk') { + return; + } + for (const field of ['executable', 'binary', 'args', 'arguments'] as const) { + if (target[field] !== undefined) { + throw new Error( + `Target "${target.name}" (${provider}) uses removed field '${field}'. Use config.command as a non-empty argv array instead.`, + ); + } + } +} + +function resolveRequiredCommandArgv( + value: unknown, + env: EnvLookup, + label: string, +): readonly string[] { + const command = resolveOptionalCommandArgv(value, env, label); + if (!command || command.length === 0) { + throw new Error(`${label} must be a non-empty argv array`); + } + return command; +} + +function resolveOptionalCommandArgv( + value: unknown, + env: EnvLookup, + label: string, +): readonly string[] | undefined { + if (value === undefined || value === null) { + return undefined; + } + if (!Array.isArray(value)) { + throw new Error(`${label} must be a non-empty argv array`); + } + const resolved = value.map((entry, index) => + resolveString(entry, env, `${label}[${index}]`, true), + ); + if (resolved.length === 0 || resolved.some((entry) => entry.trim().length === 0)) { + throw new Error(`${label} must be a non-empty argv array`); + } + return resolved; +} + +function resolveCodingAgentRuntime( + value: unknown, + env: EnvLookup, + targetName: string, +): CodingAgentRuntimeConfig { + if (value === undefined || value === null) { + return { mode: 'host' }; + } + if (typeof value === 'string') { + return { mode: normalizeCodingAgentRuntimeMode(value, targetName) }; + } + if (!isRecord(value)) { + throw new Error( + `Target "${targetName}" runtime must be 'host' or an object with mode: host|profile|sandbox.`, + ); + } + + const mode = normalizeCodingAgentRuntimeMode(value.mode, targetName); + const runtimeEnv = resolveRuntimeEnv(value.env, env, targetName); + const envAllowlist = resolveRuntimeEnvAllowlist(value.env_allowlist ?? value.envAllowlist); + return { + mode, + home: resolveOptionalString(value.home, env, `${targetName} runtime home`, { + allowLiteral: true, + optionalEnv: true, + }), + codexHome: resolveOptionalString( + value.codex_home ?? value.codexHome, + env, + `${targetName} runtime CODEX_HOME`, + { + allowLiteral: true, + optionalEnv: true, + }, + ), + tmpDir: resolveOptionalString( + value.tmp_dir ?? value.tmpDir, + env, + `${targetName} runtime tmp dir`, + { + allowLiteral: true, + optionalEnv: true, + }, + ), + ...(runtimeEnv ? { env: runtimeEnv } : {}), + ...(envAllowlist ? { envAllowlist } : {}), + }; +} + +function normalizeCodingAgentRuntimeMode( + value: unknown, + targetName: string, +): CodingAgentRuntimeMode { + if (typeof value !== 'string') { + throw new Error(`Target "${targetName}" runtime.mode must be one of: host, profile, sandbox.`); + } + const normalized = value.trim().toLowerCase(); + if (normalized === 'host' || normalized === 'profile' || normalized === 'sandbox') { + return normalized; + } + throw new Error(`Target "${targetName}" runtime.mode must be one of: host, profile, sandbox.`); +} + +function resolveRuntimeEnv( + value: unknown, + env: EnvLookup, + targetName: string, +): Readonly> | undefined { + if (value === undefined || value === null) { + return undefined; + } + if (!isRecord(value)) { + throw new Error(`Target "${targetName}" runtime.env must be an object of string values.`); + } + const resolved: Record = {}; + for (const [key, raw] of Object.entries(value)) { + if (typeof raw !== 'string') { + throw new Error(`Target "${targetName}" runtime.env.${key} must be a string.`); + } + resolved[key] = resolveString(raw, env, `${targetName} runtime.env.${key}`, true); + } + return resolved; +} + +function resolveRuntimeEnvAllowlist(value: unknown): readonly string[] | undefined { + if (value === undefined || value === null) { + return undefined; + } + if (!Array.isArray(value) || !value.every((entry) => typeof entry === 'string')) { + throw new Error('runtime.env_allowlist must be an array of strings.'); + } + return value.map((entry) => entry.trim()).filter((entry) => entry.length > 0); +} + function normalizeCodexModelReasoningEffort( value: string | undefined, ): CodexModelReasoningEffort | undefined { diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 4cf70c0cc..f71e7dacb 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -18,8 +18,8 @@ export type ProviderKind = | 'azure' | 'anthropic' | 'gemini' - | 'codex' | 'codex-cli' + | 'codex-app-server' | 'codex-sdk' | 'copilot-sdk' | 'copilot-cli' @@ -47,8 +47,8 @@ export type ProviderKind = * (e.g., skill-trigger) to run without a grader_target or LLM API key. */ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ - 'codex', 'codex-cli', + 'codex-app-server', 'codex-sdk', 'copilot-sdk', 'copilot-cli', @@ -90,8 +90,8 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ 'azure', 'anthropic', 'gemini', - 'codex', 'codex-cli', + 'codex-app-server', 'codex-sdk', 'copilot-sdk', 'copilot-cli', @@ -454,6 +454,7 @@ export interface TargetDefinition { readonly api_key?: string | unknown | undefined; readonly deployment?: string | unknown | undefined; readonly model?: string | unknown | undefined; + readonly runtime?: unknown | undefined; readonly version?: string | unknown | undefined; readonly api_version?: string | unknown | undefined; readonly api_format?: string | unknown | undefined; @@ -467,7 +468,7 @@ export interface TargetDefinition { readonly max_output_tokens?: number | unknown | undefined; // Codex fields readonly executable?: string | unknown | undefined; - readonly command?: string | unknown | undefined; + readonly command?: string | readonly string[] | unknown | undefined; readonly binary?: string | unknown | undefined; readonly args?: unknown | undefined; readonly arguments?: unknown | undefined; diff --git a/packages/core/src/evaluation/transcript-summary.ts b/packages/core/src/evaluation/transcript-summary.ts index 582996c1a..81e6f2bfa 100644 --- a/packages/core/src/evaluation/transcript-summary.ts +++ b/packages/core/src/evaluation/transcript-summary.ts @@ -40,9 +40,10 @@ export interface TranscriptSummaryWire { } const PROVIDER_ALIASES: Readonly> = { - codex: 'codex', - 'codex-cli': 'codex', - 'codex-sdk': 'codex', + codex: 'codex-cli', + 'codex-cli': 'codex-cli', + 'codex-app-server': 'codex-app-server', + 'codex-sdk': 'codex-sdk', copilot: 'copilot-sdk', 'copilot-cli': 'copilot-cli', 'copilot-sdk': 'copilot-sdk', diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 2ae2eca2d..3d33dacc2 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -110,11 +110,7 @@ const CODEX_SETTINGS = new Set([ 'model_verbosity', 'sandbox_mode', 'approval_policy', - 'executable', 'command', - 'binary', - 'args', - 'arguments', 'cwd', 'timeout_seconds', 'log_dir', @@ -226,7 +222,9 @@ function getKnownSettings(provider: string): Set | null { return ANTHROPIC_SETTINGS; case 'gemini': return GEMINI_SETTINGS; - case 'codex': + case 'codex-cli': + case 'codex-app-server': + case 'codex-sdk': return CODEX_SETTINGS; case 'copilot-sdk': return COPILOT_SDK_SETTINGS; diff --git a/packages/core/src/import/codex-parser.ts b/packages/core/src/import/codex-parser.ts index 6d4527db5..ec2177d9a 100644 --- a/packages/core/src/import/codex-parser.ts +++ b/packages/core/src/import/codex-parser.ts @@ -125,7 +125,7 @@ export function parseCodexSession(jsonl: string): TranscriptEntry { input = payload.arguments; } - const toolCall: ToolCall = normalizeToolCall('codex', { + const toolCall: ToolCall = normalizeToolCall('codex-cli', { tool: toolName, input, id: callId, @@ -156,7 +156,7 @@ export function parseCodexSession(jsonl: string): TranscriptEntry { input = payload.arguments; } - const toolCall: ToolCall = normalizeToolCall('codex', { + const toolCall: ToolCall = normalizeToolCall('codex-cli', { tool: toolName, input, id: callId, diff --git a/packages/core/test/evaluation/graders/skill-trigger.test.ts b/packages/core/test/evaluation/graders/skill-trigger.test.ts index ba7df77ab..9519a8cac 100644 --- a/packages/core/test/evaluation/graders/skill-trigger.test.ts +++ b/packages/core/test/evaluation/graders/skill-trigger.test.ts @@ -143,7 +143,7 @@ describe('SkillTriggerGrader', () => { }); it('should work with any provider kind (provider-agnostic)', () => { - for (const kind of ['claude-cli', 'copilot-cli', 'codex', 'pi-cli', 'openai']) { + for (const kind of ['claude-cli', 'copilot-cli', 'codex-cli', 'pi-cli', 'openai']) { const evaluator = new SkillTriggerGrader(makeConfig()); const context = makeContext({ provider: { kind, targetName: 'test' }, diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index 760306e80..fed9c1fb9 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -1147,7 +1147,7 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, it('populates agent_provider_request for agent providers', async () => { class AgentProvider implements Provider { readonly id = 'agent'; - readonly kind = 'codex'; // Agent provider kind + readonly kind = 'codex-cli'; // Agent provider kind readonly targetName = 'agent'; async invoke() { return { output: [{ role: 'assistant', content: 'ok' }] }; @@ -1162,8 +1162,8 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, provider, target: { ...baseTarget, - kind: 'codex', - config: { executable: 'echo' }, + kind: 'codex-cli', + config: { command: ['echo'], runtime: { mode: 'host' } }, }, evaluators: evaluatorRegistry, }); @@ -1177,8 +1177,8 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, provider, target: { ...baseTarget, - kind: 'codex', - config: { executable: 'echo' }, + kind: 'codex-cli', + config: { command: ['echo'], runtime: { mode: 'host' } }, }, evaluators: evaluatorRegistry, verbose: true, diff --git a/packages/core/test/evaluation/providers/codex.test.ts b/packages/core/test/evaluation/providers/codex.test.ts index e01440831..9a7ba904c 100644 --- a/packages/core/test/evaluation/providers/codex.test.ts +++ b/packages/core/test/evaluation/providers/codex.test.ts @@ -3,7 +3,10 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { CodexCliProvider } from '../../../src/evaluation/providers/codex-cli.js'; +import { + CodexAppServerProvider, + CodexCliProvider, +} from '../../../src/evaluation/providers/codex-cli.js'; import { type CodexLogEntry, consumeCodexLogEntries, @@ -45,8 +48,9 @@ describe('CodexCliProvider', () => { const provider = new CodexCliProvider( 'codex-target', { - executable: process.execPath, - args: ['--profile', 'default', '--model', 'test'], + command: [process.execPath, '--profile', 'default'], + model: 'test', + runtime: { mode: 'host' }, timeoutMs: 1000, logDir: fixturesRoot, }, @@ -71,15 +75,17 @@ describe('CodexCliProvider', () => { expect(extractLastAssistantContent(response.output)).toBe('done'); expect(runner).toHaveBeenCalledTimes(1); const invocation = runner.mock.calls[0][0]; + expect(invocation.executable).toBe(process.execPath); expect(invocation.args.slice(0, 7)).toEqual([ - '--ask-for-approval', - 'never', + '--profile', + 'default', + '--model', + 'test', 'exec', '--json', '--color', - 'never', - '--skip-git-repo-check', ]); + expect(invocation.args.slice(7, 9)).toEqual(['never', '--skip-git-repo-check']); expect(invocation.args).toContain('--profile'); expect(invocation.args).toContain('default'); expect(invocation.args).toContain('--model'); @@ -98,7 +104,7 @@ describe('CodexCliProvider', () => { expect(inputFilePaths).toContain(attachmentFile); }); - it('fails when Codex CLI emits invalid JSON', async () => { + it('returns a target error envelope when Codex CLI emits invalid JSON', async () => { const runner = mock(async () => ({ stdout: 'not json', stderr: '', @@ -107,7 +113,8 @@ describe('CodexCliProvider', () => { const provider = new CodexCliProvider( 'codex-target', { - executable: process.execPath, + command: [process.execPath], + runtime: { mode: 'host' }, logDir: fixturesRoot, }, runner, @@ -117,7 +124,12 @@ describe('CodexCliProvider', () => { question: 'Hello', }; - await expect(provider.invoke(request)).rejects.toThrow(/invalid JSON|assistant message/i); + const response = await provider.invoke(request); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('malformed_output'); + expect(response.targetExecution?.logs?.stdout?.text).toBe('not json'); + expect(extractLastAssistantContent(response.output)).toContain('Error:'); }); it('parses JSONL output from codex exec', async () => { @@ -138,7 +150,8 @@ describe('CodexCliProvider', () => { const provider = new CodexCliProvider( 'codex-target', { - executable: process.execPath, + command: [process.execPath], + runtime: { mode: 'host' }, logDir: fixturesRoot, }, runner, @@ -174,7 +187,8 @@ describe('CodexCliProvider', () => { const provider = new CodexCliProvider( 'codex-target', { - executable: process.execPath, + command: [process.execPath], + runtime: { mode: 'host' }, logDir: fixturesRoot, }, runner, @@ -219,7 +233,8 @@ describe('CodexCliProvider', () => { const provider = new CodexCliProvider( 'codex-target', { - executable: process.execPath, + command: [process.execPath], + runtime: { mode: 'host' }, logDir: fixturesRoot, streamLog: 'raw', }, @@ -233,4 +248,136 @@ describe('CodexCliProvider', () => { expect(logContent).toContain('"tool": "search"'); expect(logContent).toContain('"q": "hello"'); }); + + it('builds an isolated profile environment without copying host HOME or CODEX_HOME', async () => { + const profileHome = path.join(fixturesRoot, 'profile-home'); + const codexHome = path.join(fixturesRoot, 'codex-home'); + const tmp = path.join(fixturesRoot, 'tmp'); + const runner = mock(async () => ({ + stdout: JSON.stringify({ messages: [{ role: 'assistant', content: 'profile ok' }] }), + stderr: '', + exitCode: 0, + })); + const provider = new CodexCliProvider( + 'codex-profile', + { + command: [process.execPath], + runtime: { + mode: 'profile', + home: profileHome, + codexHome, + tmpDir: tmp, + env: { AGENTV_PROFILE_MARKER: 'yes' }, + }, + }, + runner, + ); + + await provider.invoke({ question: 'profile env' }); + + const invocation = runner.mock.calls[0][0]; + expect(invocation.env.HOME).toBe(profileHome); + expect(invocation.env.CODEX_HOME).toBe(codexHome); + expect(invocation.env.TMPDIR).toBe(tmp); + expect(invocation.env.AGENTV_PROFILE_MARKER).toBe('yes'); + expect(invocation.env.OPENAI_API_KEY).toBeUndefined(); + }); + + it('returns a nonzero-exit target envelope with captured stderr', async () => { + const runner = mock(async () => ({ + stdout: '', + stderr: 'boom', + exitCode: 3, + })); + const provider = new CodexCliProvider( + 'codex-crash', + { + command: [process.execPath], + runtime: { mode: 'host' }, + }, + runner, + ); + + const response = await provider.invoke({ question: 'fail' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('nonzero_exit'); + expect(response.targetExecution?.exitCode).toBe(3); + expect(response.targetExecution?.logs?.stderr?.text).toBe('boom'); + }); + + it('returns a timeout target envelope', async () => { + const runner = mock(async () => ({ + stdout: 'partial', + stderr: '', + exitCode: null, + timedOut: true, + })); + const provider = new CodexCliProvider( + 'codex-timeout', + { + command: [process.execPath], + runtime: { mode: 'host' }, + timeoutMs: 10, + }, + runner, + ); + + const response = await provider.invoke({ question: 'hang' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('timeout'); + expect(response.targetExecution?.logs?.stdout?.text).toBe('partial'); + }); + + it('returns unsupported target error for sandbox runtime until sandbox runtime is available', async () => { + const runner = mock(async () => { + throw new Error('should not run'); + }); + const provider = new CodexCliProvider( + 'codex-sandbox', + { + command: [process.execPath], + runtime: { mode: 'sandbox' }, + }, + runner, + ); + + const response = await provider.invoke({ question: 'sandbox' }); + + expect(runner).not.toHaveBeenCalled(); + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('sandbox_infra_failure'); + }); + + it('runs codex-app-server through the configured argv and JSON request payload', async () => { + const runner = mock( + async (opts: { prompt: string; executable: string; args: readonly string[] }) => ({ + stdout: JSON.stringify({ output: [{ text: 'app server answer' }] }), + stderr: '', + exitCode: 0, + }), + ); + const provider = new CodexAppServerProvider( + 'codex-app', + { + command: [process.execPath, 'app-server', '--profile', 'eng'], + runtime: { mode: 'host' }, + }, + runner, + ); + + const response = await provider.invoke({ question: 'hello', evalCaseId: 'case-app' }); + + const invocation = runner.mock.calls[0][0]; + expect(invocation.executable).toBe(process.execPath); + expect(invocation.args).toEqual(['app-server', '--profile', 'eng']); + expect(JSON.parse(invocation.prompt)).toMatchObject({ + type: 'agentv.invoke', + question: 'hello', + eval_case_id: 'case-app', + }); + expect(response.targetExecution?.providerKind).toBe('codex-app-server'); + expect(extractLastAssistantContent(response.output)).toBe('app server answer'); + }); }); diff --git a/packages/core/test/evaluation/providers/normalize-tool-call.test.ts b/packages/core/test/evaluation/providers/normalize-tool-call.test.ts index 7ec7322eb..6fa5a6e38 100644 --- a/packages/core/test/evaluation/providers/normalize-tool-call.test.ts +++ b/packages/core/test/evaluation/providers/normalize-tool-call.test.ts @@ -116,17 +116,20 @@ describe('normalizeToolCall', () => { // ------------------------------------------------------------------------- describe('codex', () => { it('command_execution → Bash', () => { - const result = normalizeToolCall('codex', tc('command_execution', { command: 'cat file' })); + const result = normalizeToolCall( + 'codex-cli', + tc('command_execution', { command: 'cat file' }), + ); expect(result.tool).toBe('Bash'); }); it('file_change → Edit', () => { - const result = normalizeToolCall('codex', tc('file_change', { changes: [] })); + const result = normalizeToolCall('codex-cli', tc('file_change', { changes: [] })); expect(result.tool).toBe('Edit'); }); it('"mcp:server/skill-name" prefix → Skill with extracted name', () => { - const result = normalizeToolCall('codex', tc('mcp:my-server/my-skill', {})); + const result = normalizeToolCall('codex-cli', tc('mcp:my-server/my-skill', {})); expect(result.tool).toBe('Skill'); expect((result.input as Record).skill).toBe('my-server/my-skill'); }); diff --git a/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts b/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts index ce6d66163..14102eab6 100644 --- a/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts +++ b/packages/core/test/evaluation/providers/sdk-provider-registry.test.ts @@ -13,7 +13,7 @@ describe('SDK provider registry isolation', () => { const provider = registry.create({ name: `${kind}-target`, kind, - config: kind === 'codex-sdk' ? { executable: 'codex' } : {}, + config: kind === 'codex-sdk' ? { command: ['codex'] } : {}, }); expect(provider).toBeInstanceOf(SdkChildProvider); expect(provider.kind).toBe(kind); diff --git a/packages/core/test/evaluation/providers/targets-file.test.ts b/packages/core/test/evaluation/providers/targets-file.test.ts index 502d67236..07d56bdb1 100644 --- a/packages/core/test/evaluation/providers/targets-file.test.ts +++ b/packages/core/test/evaluation/providers/targets-file.test.ts @@ -27,8 +27,9 @@ describe('readTargetDefinitions', () => { const filePath = await writeTargetsYaml(`targets: - label: candidate-agent id: openai:gpt-5-codex - provider: codex + provider: codex-cli config: + command: ["codex"] model: gpt-5-codex reasoning_effort: low grader_target: grader @@ -41,7 +42,8 @@ describe('readTargetDefinitions', () => { id: 'openai:gpt-5-codex', name: 'candidate-agent', label: 'candidate-agent', - provider: 'codex', + provider: 'codex-cli', + command: ['codex'], model: 'gpt-5-codex', reasoning_effort: 'low', grader_target: 'grader', diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index b30e90746..a9f4a5fcc 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -834,7 +834,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/unsupported placeholder/i); }); - it('resolves codex args using ${{ }} syntax', () => { + it('resolves codex-cli command argv using ${{ }} syntax', () => { const env = { CODEX_PROFILE: 'default', CODEX_MODEL: 'gpt-4', @@ -842,26 +842,39 @@ describe('resolveTargetDefinition', () => { const target = resolveTargetDefinition( { - name: 'codex', - provider: 'codex', - args: ['--profile', '${{ CODEX_PROFILE }}', '--model', '${{ CODEX_MODEL }}'], + name: 'codex-cli', + provider: 'codex-cli', + command: [ + 'codex-personal', + '--profile', + '${{ CODEX_PROFILE }}', + '--model', + '${{ CODEX_MODEL }}', + ], }, env, ); - expect(target.kind).toBe('codex'); - if (target.kind !== 'codex') { - throw new Error('expected codex target'); + expect(target.kind).toBe('codex-cli'); + if (target.kind !== 'codex-cli') { + throw new Error('expected codex-cli target'); } - expect(target.config.args).toEqual(['--profile', 'default', '--model', 'gpt-4']); + expect(target.config.command).toEqual([ + 'codex-personal', + '--profile', + 'default', + '--model', + 'gpt-4', + ]); }); - it('resolves codex reasoning_effort from env', () => { + it('resolves codex-cli reasoning_effort from env', () => { const target = resolveTargetDefinition( { - name: 'codex', - provider: 'codex', + name: 'codex-cli', + provider: 'codex-cli', + command: ['codex'], model: '${{ CODEX_MODEL }}', reasoning_effort: '${{ CODEX_REASONING_EFFORT }}', }, @@ -871,20 +884,21 @@ describe('resolveTargetDefinition', () => { }, ); - expect(target.kind).toBe('codex'); - if (target.kind !== 'codex') { - throw new Error('expected codex target'); + expect(target.kind).toBe('codex-cli'); + if (target.kind !== 'codex-cli') { + throw new Error('expected codex-cli target'); } expect(target.config.model).toBe('gpt-5.5'); expect(target.config.modelReasoningEffort).toBe('low'); }); - it('resolves codex OpenAI-compatible endpoint settings', () => { + it('resolves codex-cli OpenAI-compatible endpoint settings', () => { const target = resolveTargetDefinition( { name: 'codex-local-openai', - provider: 'codex', + provider: 'codex-cli', + command: ['codex-eng'], model: '${{ CODEX_MODEL }}', reasoning_effort: 'medium', model_verbosity: 'medium', @@ -901,12 +915,13 @@ describe('resolveTargetDefinition', () => { }, ); - expect(target.kind).toBe('codex'); - if (target.kind !== 'codex') { - throw new Error('expected codex target'); + expect(target.kind).toBe('codex-cli'); + if (target.kind !== 'codex-cli') { + throw new Error('expected codex-cli target'); } expect(target.config).toMatchObject({ + command: ['codex-eng'], model: 'gpt-5.3-codex-spark', modelReasoningEffort: 'medium', modelVerbosity: 'medium', @@ -937,7 +952,8 @@ describe('resolveTargetDefinition', () => { resolveTargetDefinition( { name: 'codex', - provider: 'codex', + provider: 'codex-cli', + command: ['codex'], reasoning_effort: 'tiny', }, {}, @@ -945,6 +961,19 @@ describe('resolveTargetDefinition', () => { ).toThrow(/reasoning_effort must be one of: minimal, low, medium, high, xhigh/); }); + it('rejects bare codex provider alias', () => { + expect(() => + resolveTargetDefinition( + { + name: 'codex', + provider: 'codex', + command: ['codex'], + }, + {}, + ), + ).toThrow(/ambiguous provider 'codex'/); + }); + it('does not canonicalize removed provider aliases to built-ins', () => { const target = resolveTargetDefinition( { diff --git a/packages/core/test/evaluation/validation/eval-file-schema.test.ts b/packages/core/test/evaluation/validation/eval-file-schema.test.ts index b4a3b3c85..162f0c20d 100644 --- a/packages/core/test/evaluation/validation/eval-file-schema.test.ts +++ b/packages/core/test/evaluation/validation/eval-file-schema.test.ts @@ -256,8 +256,9 @@ describe('EvalFileSchema input shorthand', () => { targets: [ { id: 'local-agent', - provider: 'codex', + provider: 'codex-cli', config: { + command: ['codex'], model: 'gpt-5.4-mini', }, }, diff --git a/packages/core/test/evaluation/validation/eval-validator.test.ts b/packages/core/test/evaluation/validation/eval-validator.test.ts index 227bf7a73..e3f2a275c 100644 --- a/packages/core/test/evaluation/validation/eval-validator.test.ts +++ b/packages/core/test/evaluation/validation/eval-validator.test.ts @@ -306,7 +306,8 @@ prompts: - raw: "Review {{ vars.diff }}" targets: - label: local-agent - provider: codex + provider: codex-cli + command: ["codex"] default_test: vars: tone: concise diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index c6edd5369..20d3e5319 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -47,8 +47,9 @@ describe('validateTargetsFile', () => { `targets: - label: candidate-agent id: openai:gpt-5-codex - provider: codex + provider: codex-cli config: + command: ["codex"] model: \${{ CODEX_MODEL }} reasoning_effort: low base_url: \${{ OPENAI_BASE_URL }} @@ -167,8 +168,6 @@ targets: provider: google - label: google-gemini-alias provider: google-gemini - - label: codex-cli-alias - provider: codex-cli - label: copilot-alias provider: copilot - label: copilot-sdk-alias @@ -190,7 +189,6 @@ targets: 'azure-openai', 'google', 'google-gemini', - 'codex-cli', 'copilot', 'copilot_sdk', 'pi', @@ -215,7 +213,8 @@ targets: filePath, `targets: - label: codex-target - provider: codex + provider: codex-cli + command: ["codex"] timeoutSeconds: 30 logDir: ./logs systemPrompt: Be precise. @@ -280,7 +279,8 @@ targets: filePath, `targets: - label: codex-target - provider: codex + provider: codex-cli + command: ["codex"] model: \${{ CODEX_MODEL }} reasoning_effort: \${{ CODEX_REASONING_EFFORT }} `, @@ -326,7 +326,8 @@ targets: filePath, `targets: - label: codex-local-openai - provider: codex + provider: codex-cli + command: ["codex"] model: \${{ CODEX_MODEL }} reasoning_effort: medium model_verbosity: medium @@ -396,7 +397,8 @@ targets: - label: grader use_target: \${{ GRADER_TARGET }} - label: codex-agent - provider: codex + provider: codex-cli + command: ["codex"] grader_target: grader `, ); @@ -437,7 +439,8 @@ targets: filePath, `targets: - label: codex-agent - provider: codex + provider: codex-cli + command: ["codex"] model: gpt-5 judge_target: grader - label: grader From ab9a5637bbcace2efec9631f60b33e3e8fee065f Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 17:15:32 +0200 Subject: [PATCH 05/10] feat(runtime): add sandbox target runner --- .../docs/docs/next/targets/coding-agents.mdx | 45 +++ .../docs/docs/next/targets/configuration.mdx | 56 ++++ .../src/evaluation/loaders/config-graph.ts | 13 +- packages/core/src/evaluation/providers/cli.ts | 61 +++- .../core/src/evaluation/providers/index.ts | 149 +++++++++- .../evaluation/providers/sandbox-runner.ts | 273 ++++++++++++++++++ .../core/src/evaluation/providers/targets.ts | 32 +- .../core/src/evaluation/providers/types.ts | 1 + .../evaluation/loaders/config-loader.test.ts | 59 ++++ .../test/evaluation/providers/cli.test.ts | 133 +++++++++ .../providers/sandbox-runtime.test.ts | 30 ++ .../test/evaluation/providers/targets.test.ts | 20 ++ 12 files changed, 852 insertions(+), 20 deletions(-) create mode 100644 packages/core/src/evaluation/providers/sandbox-runner.ts create mode 100644 packages/core/test/evaluation/providers/sandbox-runtime.test.ts diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index b25dda3db..bea4bc221 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -9,6 +9,51 @@ Coding agent targets evaluate AI coding assistants and CLI-based agents. Use `de AgentV's recommended coding-agent targets use process or protocol boundaries such as `claude-cli`, `codex-cli`, `copilot-cli`, and `pi-cli`. SDK-backed targets are opt-in provider kinds: `claude-sdk`, `codex-sdk`, `copilot-sdk`, and `pi-sdk`. When selected, AgentV runs the SDK adapter in an AgentV-owned child process and communicates with it through a structured protocol, so SDK crashes, malformed output, timeouts, and missing optional SDK packages are scoped to that target run instead of the AgentV orchestrator process. +## Runtime isolation + +Coding-agent targets can use three runtime placements: + +- `runtime: host` runs the installed CLI or child runner on the current machine + with the configured local environment. Use this to evaluate the same agent + profile you use manually. +- `runtime.mode: profile` keeps host process execution but points the agent at + an explicit home/config/profile directory. +- `runtime.mode: sandbox` runs through a separate substrate such as Docker. The + first built-in sandbox path supports `provider: cli` with explicit + `runtime.image`, `runtime.mounts`, `runtime.env`, and `runtime.secrets`. + Provider-specific adapters such as `codex-cli`, `claude-cli`, `copilot-cli`, + and `pi-cli` currently return a structured unsupported target error in + sandbox mode until their transcript handling is sandbox-aware. + +Sandbox mode never reuses host credentials implicitly. For reproducible CI, +prefer API keys or explicit secrets injected through `runtime.secrets`. +Subscription OAuth can be evaluated only by intentionally mounting or seeding +the profile directory the agent needs, which trades reproducibility for fidelity +to a local authenticated setup. + +```yaml +targets: + - id: codex-ci-sandbox + provider: cli + runtime: + mode: sandbox + engine: docker + image: ghcr.io/acme/codex-agent:sha256 + workdir: /workspace + mounts: + - source: ./workspace + target: /workspace + access: rw + - source: ./.agentv/results + target: /results + access: rw + secrets: + OPENAI_API_KEY: ${{ OPENAI_API_KEY }} + config: + command: "codex exec --json --output-file {OUTPUT_FILE} {PROMPT_FILE}" + timeout_seconds: 300 +``` + ## Prompt format Agent providers receive a structured prompt document with two sections: a **preread block** listing files the agent must read, and the **user query** containing the eval input. diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index ff7fae1b4..5801da6e9 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -48,6 +48,62 @@ the shorthand for the current machine, or object form when you need settings belong under `config`. Process-backed providers use `config.command` as a non-empty argv array. +## Runtime Modes + +Use `runtime: host` when you want AgentV to run the target exactly as it is +installed on the current machine. This is the best fit for local research, +subscription-auth workflows, and evaluating the same CLI profile an engineer +uses manually. + +Use `runtime.mode: profile` when the target still runs as a host process but +should use an isolated home/config directory, such as a dedicated `CODEX_HOME` +or `HOME`. + +Use `runtime.mode: sandbox` when the target should run inside a separate +execution substrate. The built-in sandbox runner currently supports Docker for +`provider: cli`; provider-specific coding-agent adapters such as `codex-cli`, +`claude-cli`, `copilot-cli`, and `pi-cli` return a structured unsupported target +error until their transcript parsers are wired through sandbox-aware runners. + +```yaml +targets: + - id: codex-sandbox + provider: cli + runtime: + mode: sandbox + engine: docker + image: ghcr.io/acme/codex-agent:sha256 + workdir: /workspace + network: none + mounts: + - source: ./workspace + target: /workspace + access: rw + - source: ./.agentv/results + target: /results + access: rw + env: + AGENTV_RESULT_DIR: /results + secrets: + OPENAI_API_KEY: ${{ OPENAI_API_KEY }} + config: + command: "codex exec --json --output-file {OUTPUT_FILE} {PROMPT_FILE}" + timeout_seconds: 300 +``` + +Sandbox mode does not inherit host credentials by default. Mount only the +workspace, results, cache, or credential paths the target needs, and pass only +the environment variables and secrets listed under `runtime.env` and +`runtime.secrets`. Install the target CLI by using an image that already +contains it or by adding explicit setup under `runtime.setup`; locate the CLI +with `config.command`. + +For CI, API-key or explicitly injected secret auth is the most reproducible +path. Subscription OAuth can work in a sandbox only when you intentionally mount +or seed the relevant profile directory into the sandbox. That makes the run less +portable than API-key CI and should be reserved for workflows where matching a +local subscription profile is the point of the evaluation. + Any supported top-level field can be moved to a file reference: ```yaml diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index d533356b0..6eec6f2fc 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -222,7 +222,9 @@ function parseTarget(value: unknown, location: string): NormalizedTargetConfig { } const config = readOptionalObject(value.config, `${location}.config`) ?? {}; - validateCommand(config.command, `${location}.config.command`); + validateCommand(config.command, `${location}.config.command`, { + allowString: provider === 'cli', + }); return { id, @@ -356,10 +358,17 @@ function readOptionalObject(value: unknown, location: string): Record 0) { + return; + } if ( !Array.isArray(value) || value.length === 0 || diff --git a/packages/core/src/evaluation/providers/cli.ts b/packages/core/src/evaluation/providers/cli.ts index 408f8077f..a5393a68d 100644 --- a/packages/core/src/evaluation/providers/cli.ts +++ b/packages/core/src/evaluation/providers/cli.ts @@ -8,6 +8,11 @@ import { z } from 'zod'; import type { Content } from '../content.js'; import { isContentArray } from '../content.js'; import { readTextFile } from '../file-utils.js'; +import { + type SandboxCommandRunOptions, + type TargetRuntimeConfig, + runDockerSandboxCommand, +} from './sandbox-runner.js'; import type { CliResolvedConfig } from './targets.js'; import type { Message, @@ -170,6 +175,8 @@ export interface CommandRunResult { readonly timedOut?: boolean; readonly signal?: NodeJS.Signals | null; readonly spawnErrorCode?: string; + readonly sandboxInfraFailure?: boolean; + readonly sandboxDetails?: Record; } export type CommandRunner = ( @@ -177,6 +184,11 @@ export type CommandRunner = ( options: CommandRunOptions, ) => Promise; +export type SandboxCommandRunner = ( + command: string, + options: SandboxCommandRunOptions, +) => Promise; + async function defaultCommandRunner( command: string, options: CommandRunOptions, @@ -321,6 +333,7 @@ function commandEnvelopeBase(params: { timeoutMs?: number; startedAt: number; endedAt: number; + runtimeMode?: string; result?: CommandRunResult; }): Omit { const argv = @@ -332,7 +345,7 @@ function commandEnvelopeBase(params: { targetId: params.targetName, providerId: params.providerId, providerKind: params.providerKind, - runtimeMode: 'host', + runtimeMode: params.runtimeMode ?? 'host', command: { argv, commandLine: params.command, @@ -361,6 +374,9 @@ function classifyCommandFailure( if (result.timedOut) { return 'timeout'; } + if (result.sandboxInfraFailure) { + return 'sandbox_infra_failure'; + } if (result.spawnErrorCode) { return 'spawn_failure'; } @@ -377,6 +393,9 @@ function commandFailureMessage(result: CommandRunResult, errorKind: TargetExecut if (errorKind === 'timeout') { return 'CLI provider timed out'; } + if (errorKind === 'sandbox_infra_failure') { + return result.stderr.trim() || result.stdout.trim() || 'Sandbox runtime failed'; + } if (errorKind === 'spawn_failure') { return ( result.stderr.trim() || @@ -400,6 +419,8 @@ export class CliProvider implements Provider { private readonly config: CliResolvedConfig; private readonly runCommand: CommandRunner; + private readonly runSandboxCommand: SandboxCommandRunner; + private readonly runtime?: TargetRuntimeConfig; private readonly verbose: boolean; private readonly keepTempFiles: boolean; private healthcheckPromise?: Promise; @@ -408,11 +429,15 @@ export class CliProvider implements Provider { targetName: string, config: CliResolvedConfig, runner: CommandRunner = defaultCommandRunner, + runtime?: TargetRuntimeConfig, + sandboxRunner: SandboxCommandRunner = runDockerSandboxCommand, ) { this.targetName = targetName; this.id = `cli:${targetName}`; this.config = config; this.runCommand = runner; + this.runSandboxCommand = sandboxRunner; + this.runtime = runtime; this.verbose = config.verbose ?? false; this.keepTempFiles = config.keepTempFiles ?? false; } @@ -444,7 +469,7 @@ export class CliProvider implements Provider { // Measure wall-clock time as fallback for duration try { const startTime = Date.now(); - const result = await this.runCommand(renderedCommand, { + const result = await this.runCommandForRuntime(renderedCommand, { cwd: effectiveCwd, env: process.env, timeoutMs: this.config.timeoutMs, @@ -472,6 +497,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: startTime, endedAt: Date.now(), + runtimeMode: this.runtimeMode(), result, }), status: 'error', @@ -484,6 +510,7 @@ export class CliProvider implements Provider { details: { outputFile: outputFilePath, spawnErrorCode: result.spawnErrorCode, + sandbox: result.sandboxDetails, }, }, raw: { @@ -518,6 +545,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: startTime, endedAt: Date.now(), + runtimeMode: this.runtimeMode(), result, }), status: 'error', @@ -559,6 +587,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: startTime, endedAt: Date.now(), + runtimeMode: this.runtimeMode(), result, }), status: targetTaskFailure ? 'error' : 'success', @@ -638,7 +667,7 @@ export class CliProvider implements Provider { // Measure wall-clock time for batch (used as fallback if records don't provide duration) try { const startTime = Date.now(); - const result = await this.runCommand(renderedCommand, { + const result = await this.runCommandForRuntime(renderedCommand, { cwd: effectiveCwd, env: process.env, timeoutMs: this.config.timeoutMs, @@ -738,6 +767,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: startTime, endedAt: Date.now(), + runtimeMode: this.runtimeMode(), result, }), status: 'success', @@ -779,6 +809,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: startTime, endedAt: Date.now(), + runtimeMode: this.runtimeMode(), result, }), status: 'error', @@ -817,6 +848,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: startTime, endedAt: Date.now(), + runtimeMode: this.runtimeMode(), result, }), status: parsed.error ? 'error' : 'success', @@ -1022,6 +1054,7 @@ export class CliProvider implements Provider { timeoutMs: this.config.timeoutMs, startedAt: params.startedAt, endedAt: params.startedAt + params.durationMs, + runtimeMode: this.runtimeMode(), result: params.result, }), status: 'error', @@ -1035,6 +1068,7 @@ export class CliProvider implements Provider { outputFile: params.outputFilePath, recordId: params.request.evalCaseId, spawnErrorCode: params.result.spawnErrorCode, + sandbox: params.result.sandboxDetails, }, }, raw: { @@ -1133,7 +1167,7 @@ export class CliProvider implements Provider { } try { - const result = await this.runCommand(renderedCommand, { + const result = await this.runCommandForRuntime(renderedCommand, { cwd: hcCwd ?? this.config.cwd, env: process.env, timeoutMs, @@ -1152,6 +1186,25 @@ export class CliProvider implements Provider { await cleanupTempFile(promptFilePath, this.keepTempFiles); } } + + private runtimeMode(): string { + return this.runtime?.mode ?? 'host'; + } + + private async runCommandForRuntime( + command: string, + options: CommandRunOptions, + ): Promise { + if (this.runtime?.mode !== 'sandbox') { + return this.runCommand(command, options); + } + return this.runSandboxCommand(command, { + cwd: options.cwd, + timeoutMs: options.timeoutMs, + signal: options.signal, + runtime: this.runtime, + }); + } } async function buildTemplateValues( diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 83800a1e5..28a712196 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -22,7 +22,14 @@ import { resolveDelegatedTargetDefinition, resolveTargetDefinition, } from './targets.js'; -import type { EnvLookup, Provider, TargetDefinition } from './types.js'; +import type { + EnvLookup, + Provider, + ProviderKind, + ProviderRequest, + ProviderResponse, + TargetDefinition, +} from './types.js'; import { VSCodeProvider } from './vscode-provider.js'; export type { @@ -94,6 +101,78 @@ export { discoverProviders } from './provider-discovery.js'; export { discoverCopilotSessions, type CopilotSession } from './copilot-session-discovery.js'; export { ReplayProvider } from './replay.js'; +class UnsupportedSandboxProvider implements Provider { + readonly id: string; + readonly kind: ProviderKind; + readonly targetName: string; + + constructor( + private readonly providerKind: ProviderKind, + targetName: string, + private readonly message: string, + ) { + this.kind = providerKind; + this.targetName = targetName; + this.id = `${providerKind}:${targetName}`; + } + + async invoke(_request: ProviderRequest): Promise { + const now = Date.now(); + const content = `Error: ${this.message}`; + return { + output: [{ role: 'assistant', content }], + durationMs: 0, + raw: { + error: this.message, + unsupported_provider: this.providerKind, + runtime_mode: 'sandbox', + }, + targetExecution: { + schemaVersion: 'agentv.target_execution.v1', + status: 'error', + targetId: this.targetName, + providerId: this.id, + providerKind: this.providerKind, + runtimeMode: 'sandbox', + startedAt: new Date(now).toISOString(), + endedAt: new Date(now).toISOString(), + durationMs: 0, + errorKind: 'sandbox_infra_failure', + message: this.message, + logs: { + stdout: { text: '', truncated: false, bytes: 0, storedBytes: 0 }, + stderr: { + text: this.message, + truncated: false, + bytes: Buffer.byteLength(this.message, 'utf8'), + storedBytes: Buffer.byteLength(this.message, 'utf8'), + }, + }, + transcript: { + messages: [{ role: 'assistant', content }], + finalOutput: content, + }, + details: { + unsupported_provider: this.providerKind, + supported_sandbox_provider: 'cli', + }, + }, + }; + } +} + +function usesSandboxRuntime(target: ResolvedTarget): boolean { + return target.runtime?.mode === 'sandbox'; +} + +function unsupportedSandboxProvider(target: ResolvedTarget): Provider { + return new UnsupportedSandboxProvider( + target.kind as ProviderKind, + target.name, + `runtime.mode: sandbox is not implemented for provider '${target.kind}' yet. Use provider: cli with an explicit sandbox runtime and config.command, or switch this target to runtime: host/profile until this provider has a sandbox-aware runner.`, + ); +} + /** * Create and return the default provider registry with all built-in providers. */ @@ -106,21 +185,65 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { .register('azure', (t) => new AzureProvider(t.name, t.config as never)) .register('anthropic', (t) => new AnthropicProvider(t.name, t.config as never)) .register('gemini', (t) => new GeminiProvider(t.name, t.config as never)) - .register('cli', (t) => new CliProvider(t.name, t.config as never)) - .register('codex-cli', (t) => new CodexCliProvider(t.name, t.config as never)) - .register('codex-app-server', (t) => new CodexAppServerProvider(t.name, t.config as never)) - .register('codex-sdk', (t) => new SdkChildProvider('codex-sdk', t.name, t.config)) - .register('copilot-sdk', (t) => new SdkChildProvider('copilot-sdk', t.name, t.config)) - .register('copilot-cli', (t) => new CopilotCliProvider(t.name, t.config as never)) + .register('cli', (t) => new CliProvider(t.name, t.config as never, undefined, t.runtime)) + .register('codex-cli', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new CodexCliProvider(t.name, t.config as never), + ) + .register('codex-app-server', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new CodexAppServerProvider(t.name, t.config as never), + ) + .register('codex-sdk', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new SdkChildProvider('codex-sdk', t.name, t.config), + ) + .register('copilot-sdk', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new SdkChildProvider('copilot-sdk', t.name, t.config), + ) + .register('copilot-cli', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new CopilotCliProvider(t.name, t.config as never), + ) .register('copilot-log', (t) => new CopilotLogProvider(t.name, t.config as never)) - .register('pi-sdk', (t) => new SdkChildProvider('pi-sdk', t.name, t.config)) - .register('pi-coding-agent', (t) => new SdkChildProvider('pi-sdk', t.name, t.config)) - .register('pi-cli', (t) => new PiCliProvider(t.name, t.config as never)) + .register('pi-sdk', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new SdkChildProvider('pi-sdk', t.name, t.config), + ) + .register('pi-coding-agent', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new SdkChildProvider('pi-sdk', t.name, t.config), + ) + .register('pi-cli', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new PiCliProvider(t.name, t.config as never), + ) // claude-cli is the new default subprocess provider; claude is an alias - .register('claude-cli', (t) => new ClaudeCliProvider(t.name, t.config as never)) - .register('claude', (t) => new ClaudeCliProvider(t.name, t.config as never)) + .register('claude-cli', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new ClaudeCliProvider(t.name, t.config as never), + ) + .register('claude', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new ClaudeCliProvider(t.name, t.config as never), + ) // Explicit SDK providers are isolated behind an AgentV child runner. - .register('claude-sdk', (t) => new SdkChildProvider('claude-sdk', t.name, t.config)) + .register('claude-sdk', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new SdkChildProvider('claude-sdk', t.name, t.config), + ) .register('mock', (t) => new MockProvider(t.name, t.config as never)) .register('agentv', (t) => new AgentvProvider(t.name, t.config as never)) .register('replay', (t) => new ReplayProvider(t.name, t.config as never)) diff --git a/packages/core/src/evaluation/providers/sandbox-runner.ts b/packages/core/src/evaluation/providers/sandbox-runner.ts new file mode 100644 index 000000000..7a79730d7 --- /dev/null +++ b/packages/core/src/evaluation/providers/sandbox-runner.ts @@ -0,0 +1,273 @@ +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; + +export type TargetRuntimeMode = 'host' | 'profile' | 'sandbox'; + +export interface TargetRuntimeConfig { + readonly mode: TargetRuntimeMode; + readonly [key: string]: unknown; +} + +export interface SandboxMountConfig { + readonly source: string; + readonly target: string; + readonly access?: 'ro' | 'rw' | 'read_only' | 'read_write'; +} + +export interface SandboxCommandRunOptions { + readonly cwd?: string; + readonly timeoutMs?: number; + readonly signal?: AbortSignal; + readonly runtime: TargetRuntimeConfig; +} + +export interface SandboxCommandRunResult { + readonly stdout: string; + readonly stderr: string; + readonly exitCode: number | null; + readonly failed: boolean; + readonly timedOut?: boolean; + readonly signal?: NodeJS.Signals | null; + readonly spawnErrorCode?: string; + readonly sandboxInfraFailure?: boolean; + readonly sandboxDetails?: Record; +} + +const DOCKER_INFRA_EXIT_CODES = new Set([125]); + +function asString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function asStringArray(value: unknown): readonly string[] { + if (typeof value === 'string' && value.trim().length > 0) { + return [value.trim()]; + } + if (!Array.isArray(value)) { + return []; + } + return value.filter((entry): entry is string => typeof entry === 'string' && entry.length > 0); +} + +function asStringRecord(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return {}; + } + const record: Record = {}; + for (const [key, rawValue] of Object.entries(value)) { + if (typeof rawValue === 'string') { + record[key] = rawValue; + } + } + return record; +} + +function asMounts(value: unknown): readonly SandboxMountConfig[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((entry) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + return []; + } + const source = asString((entry as Record).source); + const target = asString((entry as Record).target); + if (!source || !target) { + return []; + } + const access = asString((entry as Record).access); + return [ + { + source, + target, + ...(access === 'ro' || access === 'rw' || access === 'read_only' || access === 'read_write' + ? { access } + : {}), + }, + ]; + }); +} + +function mountAccessSuffix(access: SandboxMountConfig['access']): string { + return access === 'ro' || access === 'read_only' ? 'ro' : 'rw'; +} + +function dockerEnv(runtime: TargetRuntimeConfig): Record { + return { + ...asStringRecord(runtime.env), + ...asStringRecord(runtime.secrets), + }; +} + +function dockerImage(runtime: TargetRuntimeConfig): string | undefined { + return asString(runtime.image) ?? asString(runtime.container_image); +} + +function dockerWorkdir(runtime: TargetRuntimeConfig): string | undefined { + return asString(runtime.workdir) ?? asString(runtime.workspace); +} + +function dockerNetwork(runtime: TargetRuntimeConfig): string { + const network = asString(runtime.network); + return network ?? 'none'; +} + +function dockerSetupCommands(runtime: TargetRuntimeConfig): readonly string[] { + return asStringArray(runtime.setup); +} + +export async function runDockerSandboxCommand( + command: string, + options: SandboxCommandRunOptions, +): Promise { + const engine = asString(options.runtime.engine) ?? 'docker'; + if (engine !== 'docker') { + return { + stdout: '', + stderr: `Unsupported sandbox engine '${engine}'. The built-in sandbox runner currently supports engine: docker.`, + exitCode: null, + failed: true, + sandboxInfraFailure: true, + sandboxDetails: { engine }, + }; + } + + const image = dockerImage(options.runtime); + if (!image) { + return { + stdout: '', + stderr: 'Sandbox runtime requires runtime.image for engine: docker.', + exitCode: null, + failed: true, + sandboxInfraFailure: true, + sandboxDetails: { engine }, + }; + } + + const containerName = `agentv-sandbox-${randomUUID()}`; + const argv = [ + 'run', + '--rm', + '--name', + containerName, + '--network', + dockerNetwork(options.runtime), + ]; + + const workdir = dockerWorkdir(options.runtime); + if (workdir) { + argv.push('--workdir', workdir); + } + + for (const [key, value] of Object.entries(dockerEnv(options.runtime))) { + argv.push('--env', `${key}=${value}`); + } + + for (const mount of asMounts(options.runtime.mounts)) { + argv.push('--volume', `${mount.source}:${mount.target}:${mountAccessSuffix(mount.access)}`); + } + + const commandLine = [...dockerSetupCommands(options.runtime), command].join(' && '); + argv.push(image, '/bin/sh', '-lc', commandLine); + + return new Promise((resolve) => { + const child = spawn('docker', argv, { + cwd: options.cwd, + env: process.env, + windowsHide: true, + }); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + let cancelled = false; + let settled = false; + + const append = (current: string, chunk: Buffer) => `${current}${chunk.toString('utf8')}`; + + const cleanupContainer = () => { + const cleanup = spawn('docker', ['rm', '-f', containerName], { + stdio: 'ignore', + windowsHide: true, + }); + cleanup.unref?.(); + }; + + const terminate = () => { + child.kill('SIGTERM'); + cleanupContainer(); + setTimeout(() => { + child.kill('SIGKILL'); + cleanupContainer(); + }, 2_000).unref?.(); + }; + + const timeout = options.timeoutMs + ? setTimeout(() => { + timedOut = true; + terminate(); + }, options.timeoutMs) + : undefined; + timeout?.unref?.(); + + const abort = () => { + cancelled = true; + terminate(); + }; + if (options.signal) { + if (options.signal.aborted) { + abort(); + } else { + options.signal.addEventListener('abort', abort, { once: true }); + } + } + + child.stdout?.on('data', (chunk: Buffer) => { + stdout = append(stdout, chunk); + }); + child.stderr?.on('data', (chunk: Buffer) => { + stderr = append(stderr, chunk); + }); + + child.on('error', (error: NodeJS.ErrnoException) => { + if (settled) { + return; + } + settled = true; + if (timeout) clearTimeout(timeout); + if (options.signal) options.signal.removeEventListener('abort', abort); + resolve({ + stdout, + stderr: stderr || error.message, + exitCode: null, + failed: true, + timedOut, + signal: null, + spawnErrorCode: error.code, + sandboxInfraFailure: true, + sandboxDetails: { engine, image, containerName, argv: ['docker', ...argv] }, + }); + }); + + child.on('close', (code, signal) => { + if (settled) { + return; + } + settled = true; + if (timeout) clearTimeout(timeout); + if (options.signal) options.signal.removeEventListener('abort', abort); + const sandboxInfraFailure = + code !== null && DOCKER_INFRA_EXIT_CODES.has(code) && !timedOut && !cancelled; + resolve({ + stdout, + stderr, + exitCode: code, + failed: code !== 0 || signal !== null || timedOut || cancelled, + timedOut, + signal, + sandboxInfraFailure, + sandboxDetails: { engine, image, containerName, argv: ['docker', ...argv] }, + }); + }); + }); +} diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index f18eb4dc4..8e6dbd0d4 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import { z } from 'zod'; import { renderEnvTemplateString } from '../interpolation.js'; +import type { TargetRuntimeConfig, TargetRuntimeMode } from './sandbox-runner.js'; import type { EnvLookup, TargetDefinition } from './types.js'; // --------------------------------------------------------------------------- @@ -695,7 +696,7 @@ export function normalizeTargetDefinition( typeof rawLabel === 'string' && rawLabel.trim().length > 0 ? rawLabel.trim() : undefined; const legacyName = typeof rawName === 'string' && rawName.trim().length > 0 ? rawName.trim() : undefined; - const name = label ?? legacyName ?? options.defaultName; + const name = label ?? legacyName ?? id ?? options.defaultName; if (!name || name.trim().length === 0) { throw new Error("Target definition is missing a valid 'label' field"); } @@ -813,6 +814,7 @@ export type CliHealthcheck = Readonly; interface ResolvedTargetBase { readonly name: string; readonly label?: string; + readonly runtime?: TargetRuntimeConfig; readonly graderTarget?: string; readonly workers?: number; readonly providerBatching?: boolean; @@ -879,6 +881,7 @@ export type ResolvedTarget = */ export const COMMON_TARGET_SETTINGS = [ 'use_target', + 'runtime', 'batch_requests', 'subagent_mode_allowed', 'fallback_targets', @@ -894,6 +897,7 @@ const BASE_TARGET_SCHEMA = z label: z.string().optional(), provider: z.string().optional(), config: z.record(z.unknown()).optional(), + runtime: z.unknown().optional(), use_target: z.string().optional(), grader_target: z.string().optional(), workers: z.number().int().min(1).optional(), @@ -907,6 +911,7 @@ const BASE_TARGET_SCHEMA = z // (`2024-12-01-preview`) is no longer reachable from the Azure provider here. const DEFAULT_AZURE_API_VERSION = 'v1'; const DEFAULT_OPENAI_BASE_URL = 'https://api.openai.com/v1'; +const TARGET_RUNTIME_MODE_VALUES = new Set(['host', 'profile', 'sandbox']); function normalizeAzureApiVersion(value: string | undefined): string { if (!value) { @@ -961,6 +966,30 @@ function resolveRetryConfig(target: z.infer): RetryCo }; } +function resolveTargetRuntime( + rawRuntime: unknown, + targetName: string, +): TargetRuntimeConfig | undefined { + if (rawRuntime === undefined || rawRuntime === null) { + return undefined; + } + if (typeof rawRuntime === 'string') { + const mode = rawRuntime.trim(); + if (TARGET_RUNTIME_MODE_VALUES.has(mode as TargetRuntimeMode)) { + return { mode: mode as TargetRuntimeMode }; + } + } + if (isRecord(rawRuntime)) { + const mode = typeof rawRuntime.mode === 'string' ? rawRuntime.mode.trim() : ''; + if (TARGET_RUNTIME_MODE_VALUES.has(mode as TargetRuntimeMode)) { + return { ...rawRuntime, mode: mode as TargetRuntimeMode }; + } + } + throw new Error( + `Invalid runtime for target "${targetName}": use 'host' or an object with mode: host|profile|sandbox.`, + ); +} + export function resolveDelegatedTargetDefinition( name: string, definitions: ReadonlyMap, @@ -1054,6 +1083,7 @@ export function resolveTargetDefinition( const base = { name: parsed.name, label: parsed.label, + runtime: resolveTargetRuntime(parsed.runtime, parsed.name), graderTarget: parsed.grader_target, workers: parsed.workers, providerBatching, diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index f71e7dacb..834b7c5ec 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -435,6 +435,7 @@ export interface TargetDefinition { readonly label?: string | undefined; /** Promptfoo-shaped provider options bag. Provider settings are flattened at the boundary. */ readonly config?: unknown | undefined; + readonly runtime?: unknown | undefined; readonly prompts?: unknown | undefined; readonly transform?: unknown | undefined; readonly delay?: number | unknown | undefined; diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index b79025772..435d45ca0 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -175,6 +175,65 @@ describe('loadConfig', () => { } }); + it('accepts sandbox runtime settings under runtime without top-level install fields', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-sandbox-')); + try { + const configPath = path.join(tempDir, 'sandbox.eval.yaml'); + writeFileSync( + configPath, + [ + 'targets:', + ' - id: agent-sandbox', + ' provider: cli', + ' runtime:', + ' mode: sandbox', + ' engine: docker', + ' image: ghcr.io/example/agent-cli:sha256', + ' workdir: /workspace', + ' setup:', + ' - agent-cli --version', + ' mounts:', + ' - source: ./workspace', + ' target: /workspace', + ' access: rw', + ' - source: ./.agentv/results', + ' target: /results', + ' access: rw', + ' env:', + ' AGENTV_RESULT_DIR: /results', + ' secrets:', + ' OPENAI_API_KEY: ${{ OPENAI_API_KEY }}', + ' config:', + ' command: "agent-cli run {PROMPT_FILE} {OUTPUT_FILE}"', + '', + ].join('\n'), + ); + + const graph = await loadComposableConfigGraph(configPath); + + expect(graph.targets?.[0]).toEqual({ + id: 'agent-sandbox', + provider: 'cli', + runtime: { + mode: 'sandbox', + engine: 'docker', + image: 'ghcr.io/example/agent-cli:sha256', + workdir: '/workspace', + setup: ['agent-cli --version'], + mounts: [ + { source: './workspace', target: '/workspace', access: 'rw' }, + { source: './.agentv/results', target: '/results', access: 'rw' }, + ], + env: { AGENTV_RESULT_DIR: '/results' }, + secrets: { OPENAI_API_KEY: '${{ OPENAI_API_KEY }}' }, + }, + config: { command: 'agent-cli run {PROMPT_FILE} {OUTPUT_FILE}' }, + }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('rejects wrapped referenced field files', async () => { const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-graph-wrapped-')); try { diff --git a/packages/core/test/evaluation/providers/cli.test.ts b/packages/core/test/evaluation/providers/cli.test.ts index af96b1b4d..88c40803c 100644 --- a/packages/core/test/evaluation/providers/cli.test.ts +++ b/packages/core/test/evaluation/providers/cli.test.ts @@ -258,6 +258,139 @@ describe('CliProvider', () => { expect(response.targetExecution?.message).toBe('task failed'); }); + it('runs sandbox runtime commands without inheriting host env', async () => { + process.env.AGENTV_SANDBOX_SHOULD_NOT_LEAK = 'host-secret'; + let capturedRuntime: Record | undefined; + const sandboxRunner = mock(async (command: string, options): Promise => { + capturedRuntime = options.runtime as Record; + const match = command.match(/agentv-case-1-\d+-\w+\.json/); + if (match) { + const outputFilePath = path.join(os.tmpdir(), match[0]); + await writeFile(outputFilePath, JSON.stringify({ text: 'sandbox response' }), 'utf-8'); + createdFiles.push(outputFilePath); + } + return { + stdout: 'sandbox stdout', + stderr: 'sandbox stderr', + exitCode: 0, + failed: false, + }; + }); + + const provider = new CliProvider( + 'cli-target', + baseConfig, + undefined, + { + mode: 'sandbox', + image: 'agentv-target:sha256', + env: { SAFE_ENV: '1' }, + secrets: { OPENAI_API_KEY: 'explicit-key' }, + mounts: [{ source: '/host/workspace', target: '/workspace', access: 'rw' }], + }, + sandboxRunner, + ); + + const response = await provider.invoke(baseRequest); + + expect(sandboxRunner).toHaveBeenCalledTimes(1); + expect(capturedRuntime?.env).toEqual({ SAFE_ENV: '1' }); + expect(capturedRuntime?.secrets).toEqual({ OPENAI_API_KEY: 'explicit-key' }); + expect(JSON.stringify(capturedRuntime)).not.toContain('host-secret'); + expect(response.targetExecution?.runtimeMode).toBe('sandbox'); + expect(response.targetExecution?.status).toBe('success'); + expect(response.targetExecution?.logs?.stdout?.text).toBe('sandbox stdout'); + expect(response.targetExecution?.logs?.stderr?.text).toBe('sandbox stderr'); + expect(extractLastAssistantContent(response.output)).toBe('sandbox response'); + process.env.AGENTV_SANDBOX_SHOULD_NOT_LEAK = undefined; + }); + + it('distinguishes sandbox infra failure from target nonzero exit', async () => { + const infraRunner = mock( + async (_command, _options): Promise => ({ + stdout: '', + stderr: 'docker daemon unavailable', + exitCode: 125, + failed: true, + sandboxInfraFailure: true, + }), + ); + const targetFailureRunner = mock( + async (_command, _options): Promise => ({ + stdout: 'target stdout', + stderr: 'target failed', + exitCode: 2, + failed: true, + }), + ); + const runtime = { mode: 'sandbox' as const, image: 'agentv-target:sha256' }; + + const infraResponse = await new CliProvider( + 'cli-target', + baseConfig, + undefined, + runtime, + infraRunner, + ).invoke(baseRequest); + const targetResponse = await new CliProvider( + 'cli-target', + baseConfig, + undefined, + runtime, + targetFailureRunner, + ).invoke(baseRequest); + + expect(infraResponse.targetExecution?.runtimeMode).toBe('sandbox'); + expect(infraResponse.targetExecution?.errorKind).toBe('sandbox_infra_failure'); + expect(targetResponse.targetExecution?.runtimeMode).toBe('sandbox'); + expect(targetResponse.targetExecution?.errorKind).toBe('nonzero_exit'); + expect(targetResponse.targetExecution?.logs?.stdout?.text).toBe('target stdout'); + }); + + it('returns timeout and cancellation envelopes for sandbox work', async () => { + const timeoutRunner = mock( + async (_command, _options): Promise => ({ + stdout: 'partial transcript', + stderr: '', + exitCode: null, + failed: true, + timedOut: true, + }), + ); + const controller = new AbortController(); + const cancelRunner = mock(async (_command, _options): Promise => { + controller.abort(); + return { + stdout: 'cancel partial', + stderr: '', + exitCode: null, + failed: true, + signal: 'SIGTERM', + }; + }); + const runtime = { mode: 'sandbox' as const, image: 'agentv-target:sha256' }; + + const timeoutResponse = await new CliProvider( + 'cli-target', + baseConfig, + undefined, + runtime, + timeoutRunner, + ).invoke(baseRequest); + const cancelResponse = await new CliProvider( + 'cli-target', + baseConfig, + undefined, + runtime, + cancelRunner, + ).invoke({ ...baseRequest, signal: controller.signal }); + + expect(timeoutResponse.targetExecution?.errorKind).toBe('timeout'); + expect(timeoutResponse.targetExecution?.logs?.stdout?.text).toBe('partial transcript'); + expect(cancelResponse.targetExecution?.errorKind).toBe('cancelled'); + expect(cancelResponse.targetExecution?.logs?.stdout?.text).toBe('cancel partial'); + }); + it('uses request.cwd as working directory override in invoke', async () => { let capturedCwd: string | undefined; const runner = mock(async (command: string, options): Promise => { diff --git a/packages/core/test/evaluation/providers/sandbox-runtime.test.ts b/packages/core/test/evaluation/providers/sandbox-runtime.test.ts new file mode 100644 index 000000000..d8f0b34c6 --- /dev/null +++ b/packages/core/test/evaluation/providers/sandbox-runtime.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'bun:test'; + +import { createProvider } from '../../../src/evaluation/providers/index.js'; +import type { ResolvedTarget } from '../../../src/evaluation/providers/targets.js'; + +describe('sandbox target runtime', () => { + it('returns deliberate unsupported envelopes for sandbox coding-agent adapters', async () => { + const target: ResolvedTarget = { + name: 'codex-sandbox', + kind: 'codex-cli', + runtime: { mode: 'sandbox', image: 'agentv-codex:sha256' }, + config: { + executable: 'codex', + }, + }; + + const provider = createProvider(target); + const response = await provider.invoke({ question: 'Fix the test' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.runtimeMode).toBe('sandbox'); + expect(response.targetExecution?.errorKind).toBe('sandbox_infra_failure'); + expect(response.targetExecution?.providerKind).toBe('codex-cli'); + expect(response.targetExecution?.message).toContain("provider 'codex-cli'"); + expect(response.targetExecution?.details).toMatchObject({ + unsupported_provider: 'codex-cli', + supported_sandbox_provider: 'cli', + }); + }); +}); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index a9f4a5fcc..7d78b2941 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -106,6 +106,26 @@ describe('resolveTargetDefinition', () => { expect(target.config.response).toBe('ok'); }); + it('uses clean target id as identity when label and legacy name are absent', () => { + const target = resolveTargetDefinition( + { + id: 'sandbox-cli', + provider: 'cli', + runtime: { mode: 'sandbox', image: 'agentv-target:sha256' }, + config: { + command: 'agent run {PROMPT_FILE} {OUTPUT_FILE}', + }, + } as never, + {}, + ); + + expect(target.name).toBe('sandbox-cli'); + expect(target.label).toBe('sandbox-cli'); + expect(target.kind).toBe('cli'); + expect(target.runtime).toEqual({ mode: 'sandbox', image: 'agentv-target:sha256' }); + expect(target.config.command).toBe('agent run {PROMPT_FILE} {OUTPUT_FILE}'); + }); + it('treats provider as backend kind while id remains a provider locator', () => { const target = resolveTargetDefinition( { From 610e107a9dd13d114e29bcaa963c7b83e8f24d3f Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 17:16:01 +0200 Subject: [PATCH 06/10] feat(providers): add pi rpc runtime boundary --- .../docs/docs/next/targets/coding-agents.mdx | 71 ++- .../docs/docs/next/targets/configuration.mdx | 1 + .../core/src/evaluation/providers/index.ts | 7 + .../providers/normalize-tool-call.ts | 2 + .../core/src/evaluation/providers/pi-cli.ts | 362 +++++++------- .../src/evaluation/providers/pi-process.ts | 339 +++++++++++++ .../core/src/evaluation/providers/pi-rpc.ts | 448 ++++++++++++++++++ .../core/src/evaluation/providers/targets.ts | 221 ++++++++- .../core/src/evaluation/providers/types.ts | 3 + .../core/src/evaluation/transcript-summary.ts | 1 + .../validation/targets-validator.ts | 32 ++ .../evaluation/providers/pi-runtime.test.ts | 220 +++++++++ 12 files changed, 1476 insertions(+), 231 deletions(-) create mode 100644 packages/core/src/evaluation/providers/pi-process.ts create mode 100644 packages/core/src/evaluation/providers/pi-rpc.ts create mode 100644 packages/core/test/evaluation/providers/pi-runtime.test.ts diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index bea4bc221..d764c9336 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -7,7 +7,7 @@ sidebar: Coding agent targets evaluate AI coding assistants and CLI-based agents. Use `defaults.grader` or an evaluator-level grader override for LLM-based grading. -AgentV's recommended coding-agent targets use process or protocol boundaries such as `claude-cli`, `codex-cli`, `copilot-cli`, and `pi-cli`. SDK-backed targets are opt-in provider kinds: `claude-sdk`, `codex-sdk`, `copilot-sdk`, and `pi-sdk`. When selected, AgentV runs the SDK adapter in an AgentV-owned child process and communicates with it through a structured protocol, so SDK crashes, malformed output, timeouts, and missing optional SDK packages are scoped to that target run instead of the AgentV orchestrator process. +AgentV's recommended coding-agent targets use process or protocol boundaries such as `claude-cli`, `codex-cli`, `copilot-cli`, `pi-cli`, and `pi-rpc`. SDK-backed targets are opt-in provider kinds: `claude-sdk`, `codex-sdk`, `copilot-sdk`, and `pi-sdk`. When selected, AgentV runs the SDK adapter in an AgentV-owned child process and communicates with it through a structured protocol, so SDK crashes, malformed output, timeouts, and missing optional SDK packages are scoped to that target run instead of the AgentV orchestrator process. ## Runtime isolation @@ -185,12 +185,13 @@ Use `provider: copilot-sdk` only when you intentionally want the Copilot SDK pat ```yaml targets: - - label: pi_target + - id: pi_target provider: pi-sdk - subprovider: openai-codex - model: gpt-5.5 - thinking: medium - grader_target: azure-base + runtime: host + config: + subprovider: openai-codex + model: gpt-5.5 + thinking: medium ``` | Field | Required | Description | @@ -210,21 +211,47 @@ configuration. This works for OpenAI-compatible endpoints: ```yaml targets: - - label: pi-sdk-openai + - id: pi-sdk-openai provider: pi-sdk - subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} - grader_target: azure-base + runtime: host + config: + subprovider: openai + base_url: ${{ OPENAI_ENDPOINT }} + api_key: ${{ OPENAI_API_KEY }} + model: ${{ OPENAI_MODEL }} ``` -Use `provider: pi-cli` instead when you want AgentV to spawn the `pi` binary directly. It accepts the same Pi fields above plus: +Use `provider: pi-rpc` when you want AgentV to launch Pi in Kata-style RPC mode and communicate over process stdio. AgentV starts the configured command with `--mode rpc` unless the command already includes a mode flag. The Pi runtime must already be installed or reachable through `config.command`; AgentV does not install workers or provision a fleet for this target. + +```yaml +targets: + - id: pi-rpc-local + provider: pi-rpc + runtime: host + config: + command: ["pi"] + model: gpt-5-codex +``` + +Use `provider: pi-cli` when you want AgentV to spawn the `pi` binary for simple JSONL subprocess execution: + +```yaml +targets: + - id: pi-cli-local + provider: pi-cli + runtime: + mode: profile + home: .agentv/profiles/pi-local + config: + command: ["pi"] + model: gpt-5-codex +``` + +`pi-cli` and `pi-rpc` accept the same Pi fields above plus: | Field | Required | Description | |-------|----------|-------------| -| `executable` | No | Pi binary or shim to run. Defaults to `pi`. | -| `args` | No | Extra arguments appended before the prompt. | +| `command` | No | Non-empty argv array for the Pi binary, shim, or remote command. Defaults to `["pi"]`. | Pi CLI has one important difference from the SDK path: the built-in `openai` provider does not currently expose a CLI base-url option. With `provider: pi-cli` @@ -236,13 +263,15 @@ is compatible with Azure OpenAI Responses: ```yaml targets: - - label: pi-cli-gateway + - id: pi-cli-gateway provider: pi-cli - subprovider: azure - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} - grader_target: azure-base + runtime: host + config: + command: ["pi"] + subprovider: azure + base_url: ${{ OPENAI_ENDPOINT }} + api_key: ${{ OPENAI_API_KEY }} + model: ${{ OPENAI_MODEL }} ``` ## VS Code diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 5801da6e9..6683fcb52 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -149,6 +149,7 @@ already-exported secrets into `.env`. | `copilot-sdk` | Agent | Copilot SDK in an isolated child runner | | `pi-sdk` | Agent | Pi SDK in an isolated child runner | | `pi-cli` | Agent | Pi CLI subprocess | +| `pi-rpc` | Agent | Pi RPC subprocess over stdio | | `vscode` | Agent | VS Code with Copilot | | `vscode-insiders` | Agent | VS Code Insiders | | `cli` | Agent | Any CLI command — see [CLI Provider](/docs/targets/cli-provider) | diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 28a712196..8e1a053fb 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -13,6 +13,7 @@ import { } from './llm-providers.js'; import { MockProvider } from './mock.js'; import { PiCliProvider } from './pi-cli.js'; +import { PiRpcProvider } from './pi-rpc.js'; import { ProviderRegistry } from './provider-registry.js'; import { ReplayProvider } from './replay.js'; import { SdkChildProvider } from './sdk-child-provider.js'; @@ -64,6 +65,7 @@ export type { OpenRouterResolvedConfig, PiCliResolvedConfig, PiCodingAgentResolvedConfig, + PiRpcResolvedConfig, ReplayResolvedConfig, ReplayResolvedSource, ResolvedTarget, @@ -227,6 +229,11 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { ? unsupportedSandboxProvider(t) : new PiCliProvider(t.name, t.config as never), ) + .register('pi-rpc', (t) => + usesSandboxRuntime(t) + ? unsupportedSandboxProvider(t) + : new PiRpcProvider(t.name, t.config as never), + ) // claude-cli is the new default subprocess provider; claude is an alias .register('claude-cli', (t) => usesSandboxRuntime(t) diff --git a/packages/core/src/evaluation/providers/normalize-tool-call.ts b/packages/core/src/evaluation/providers/normalize-tool-call.ts index 53b171376..80558f7a2 100644 --- a/packages/core/src/evaluation/providers/normalize-tool-call.ts +++ b/packages/core/src/evaluation/providers/normalize-tool-call.ts @@ -130,6 +130,8 @@ const TOOL_NAME_MAP = new Map([ ['pi-coding-agent::bash', 'Bash'], ['pi-cli::read', 'Read'], ['pi-cli::bash', 'Bash'], + ['pi-rpc::read', 'Read'], + ['pi-rpc::bash', 'Bash'], ]); // --------------------------------------------------------------------------- diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts index 258afc77c..b5d2955e3 100644 --- a/packages/core/src/evaluation/providers/pi-cli.ts +++ b/packages/core/src/evaluation/providers/pi-cli.ts @@ -2,24 +2,32 @@ * Pi CLI provider — shells out to the `pi` binary as a subprocess. * * Use this when you have the Pi CLI installed globally or want to point to - * a specific binary via the `executable` config field (defaults to `pi` on PATH). + * a specific binary via `config.command` (defaults to `["pi"]` on PATH). * Output is captured as JSONL from stdout and parsed into AgentV messages. * - * For the SDK-based approach (no subprocess), use the `pi-coding-agent` provider instead. + * For the SDK-based approach, use the explicit `pi-sdk` provider. */ -import { execSync, spawn } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { accessSync, createWriteStream, readFileSync } from 'node:fs'; +import { createWriteStream } from 'node:fs'; import type { WriteStream } from 'node:fs'; import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { trackChild } from '../../runtime/child-tracker.js'; import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { recordPiLogEntry } from './pi-log-tracker.js'; +import { + type PiProcessRunResult, + type PiProcessRunner, + buildPiRuntimeEnv, + buildPiTargetExecution, + classifyPiProcessFailure, + defaultPiProcessRunner, + piProcessFailureMessage, + splitPiCommand, +} from './pi-process.js'; import { extractAzureResourceName, resolveCliProvider, @@ -40,26 +48,6 @@ import type { const WORKSPACE_PREFIX = 'agentv-pi-'; const PROMPT_FILENAME = 'prompt.md'; -interface PiRunOptions { - readonly executable: string; - readonly args: readonly string[]; - readonly cwd: string; - readonly timeoutMs?: number; - readonly env: NodeJS.ProcessEnv; - readonly signal?: AbortSignal; - readonly onStdoutChunk?: (chunk: string) => void; - readonly onStderrChunk?: (chunk: string) => void; -} - -interface PiRunResult { - readonly stdout: string; - readonly stderr: string; - readonly exitCode: number; - readonly timedOut?: boolean; -} - -type PiRunner = (options: PiRunOptions) => Promise; - export class PiCliProvider implements Provider { readonly id: string; readonly kind = 'pi-cli' as const; @@ -67,9 +55,13 @@ export class PiCliProvider implements Provider { readonly supportsBatch = false; private readonly config: PiCliResolvedConfig; - private readonly runPi: PiRunner; + private readonly runPi: PiProcessRunner; - constructor(targetName: string, config: PiCliResolvedConfig, runner: PiRunner = defaultPiRunner) { + constructor( + targetName: string, + config: PiCliResolvedConfig, + runner: PiProcessRunner = defaultPiProcessRunner, + ) { this.id = `pi-cli:${targetName}`; this.targetName = targetName; this.config = config; @@ -98,22 +90,32 @@ export class PiCliProvider implements Provider { await writeFile(promptFile, request.question, 'utf8'); const args = this.buildPiArgs(request.question, inputFiles); + const command = splitPiCommand(this.config.command, args); const result = await this.executePi(args, cwd, request.signal, logger); - if (result.timedOut) { - throw new Error( - `Pi CLI timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`, - ); + if (result.timedOut || result.exitCode !== 0 || result.signal || result.spawnErrorCode) { + return this.buildProcessErrorResponse({ + result, + command, + cwd, + startedAt: startMs, + signalAborted: request.signal?.aborted, + }); } - if (result.exitCode !== 0) { - const detail = pickDetail(result.stderr, result.stdout); - const prefix = `Pi CLI exited with code ${result.exitCode}`; - throw new Error(detail ? `${prefix}: ${detail}` : prefix); + let parsed: unknown[]; + try { + parsed = parsePiJsonl(result.stdout); + } catch (error) { + return this.buildMalformedOutputResponse({ + result, + command, + cwd, + startedAt: startMs, + message: error instanceof Error ? error.message : String(error), + }); } - - const parsed = parsePiJsonl(result.stdout); const output = extractMessages(parsed); const tokenUsage = extractTokenUsage(parsed); @@ -145,6 +147,7 @@ export class PiCliProvider implements Provider { stderr: result.stderr, exitCode: result.exitCode, args, + command, executable: this.config.executable, promptFile, workspace: workspaceRoot ?? cwd, @@ -156,6 +159,22 @@ export class PiCliProvider implements Provider { durationMs, startTime, endTime, + targetExecution: buildPiTargetExecution({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + runtimeMode: this.config.runtime.mode, + command, + cwd, + timeoutMs: this.config.timeoutMs, + startedAt: startMs, + endedAt: Date.now(), + result, + status: 'success', + output, + finalOutput: extractPiTextContent(output.at(-1)?.content) ?? '', + details: { promptFile, inputFiles, logFile: logger?.filePath }, + }), }; } finally { await logger?.close(); @@ -203,10 +222,6 @@ export class PiCliProvider implements Provider { if (this.config.thinking) { args.push('--thinking', this.config.thinking); } - if (this.config.args && this.config.args.length > 0) { - args.push(...this.config.args); - } - if (inputFiles && inputFiles.length > 0) { for (const file of inputFiles) { args.push(`@${file}`); @@ -226,31 +241,23 @@ export class PiCliProvider implements Provider { cwd: string, signal: AbortSignal | undefined, logger: PiStreamLogger | undefined, - ): Promise { - try { - return await this.runPi({ - executable: this.config.executable, - args, - cwd, - timeoutMs: this.config.timeoutMs, - env: this.buildEnv(), - signal, - onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : undefined, - onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : undefined, - }); - } catch (error) { - const err = error as NodeJS.ErrnoException; - if (err.code === 'ENOENT') { - throw new Error( - `Pi CLI executable '${this.config.executable}' was not found. Update the target executable or add it to PATH.`, - ); - } - throw error; - } + ): Promise { + return await this.runPi({ + command: splitPiCommand(this.config.command, args), + cwd, + timeoutMs: this.config.timeoutMs, + env: this.buildEnv(), + signal, + onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : undefined, + onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : undefined, + }); } private buildEnv(): NodeJS.ProcessEnv { - const env = { ...process.env }; + const env = buildPiRuntimeEnv({ + runtime: this.config.runtime, + targetName: this.targetName, + }); const provider = this.config.subprovider?.toLowerCase() ?? 'google'; @@ -314,6 +321,101 @@ export class PiCliProvider implements Provider { return env; } + private buildProcessErrorResponse(params: { + readonly result: PiProcessRunResult; + readonly command: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly signalAborted?: boolean; + }): ProviderResponse { + const errorKind = classifyPiProcessFailure(params.result, params.signalAborted); + const message = piProcessFailureMessage({ + providerLabel: 'Pi CLI', + result: params.result, + errorKind, + timeoutMs: this.config.timeoutMs, + }); + const output = [{ role: 'assistant' as const, content: `Error: ${message}` }]; + const endedAt = Date.now(); + + return { + raw: { + stdout: params.result.stdout, + stderr: params.result.stderr, + exitCode: params.result.exitCode, + signal: params.result.signal, + command: params.command, + cwd: params.cwd, + error: message, + }, + output, + durationMs: endedAt - params.startedAt, + startTime: new Date(params.startedAt).toISOString(), + endTime: new Date(endedAt).toISOString(), + targetExecution: buildPiTargetExecution({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + runtimeMode: this.config.runtime.mode, + command: params.command, + cwd: params.cwd, + timeoutMs: this.config.timeoutMs, + startedAt: params.startedAt, + endedAt, + result: params.result, + status: 'error', + errorKind, + message, + output, + finalOutput: `Error: ${message}`, + details: { spawnErrorCode: params.result.spawnErrorCode }, + }), + }; + } + + private buildMalformedOutputResponse(params: { + readonly result: PiProcessRunResult; + readonly command: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly message: string; + }): ProviderResponse { + const message = `Pi CLI malformed output: ${params.message}`; + const output = [{ role: 'assistant' as const, content: `Error: ${message}` }]; + const endedAt = Date.now(); + return { + raw: { + stdout: params.result.stdout, + stderr: params.result.stderr, + exitCode: params.result.exitCode, + command: params.command, + cwd: params.cwd, + error: message, + }, + output, + durationMs: endedAt - params.startedAt, + startTime: new Date(params.startedAt).toISOString(), + endTime: new Date(endedAt).toISOString(), + targetExecution: buildPiTargetExecution({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + runtimeMode: this.config.runtime.mode, + command: params.command, + cwd: params.cwd, + timeoutMs: this.config.timeoutMs, + startedAt: params.startedAt, + endedAt, + result: params.result, + status: 'error', + errorKind: 'malformed_output', + message, + output, + finalOutput: `Error: ${message}`, + }), + }; + } + private async createWorkspace(): Promise { return await mkdtemp(path.join(tmpdir(), WORKSPACE_PREFIX)); } @@ -977,140 +1079,6 @@ function escapeAtSymbols(prompt: string): string { return prompt.replace(/@\[([^\]]+)\]:/g, '[[$1]]:'); } -function pickDetail(stderr: string, stdout: string): string | undefined { - const errorText = stderr.trim(); - if (errorText.length > 0) return errorText; - const stdoutText = stdout.trim(); - return stdoutText.length > 0 ? stdoutText : undefined; -} - -function formatTimeoutSuffix(timeoutMs: number | undefined): string { - if (!timeoutMs || timeoutMs <= 0) return ''; - return ` after ${Math.ceil(timeoutMs / 1000)}s`; -} - -/** - * On Windows, npm/bun global installs create `.cmd` and `.sh` wrappers. - * Bun's spawn can't capture stdout from sh-script wrappers (the forked - * node process writes to a different stdout). Resolve to the underlying - * node script so we can spawn `node script.js` directly. - */ -function resolveWindowsCmd(executable: string): [string, string[]] { - if (process.platform !== 'win32') return [executable, []]; - - // If already pointing at node/bun or a .js file, no resolution needed - const lower = executable.toLowerCase(); - if (lower.endsWith('.js') || lower.endsWith('.exe')) return [executable, []]; - - // Find the executable's full path using `where` - let fullPath: string; - try { - fullPath = execSync(`where ${executable}`, { encoding: 'utf-8' }) - .trim() - .split(/\r?\n/)[0] - .trim(); - } catch { - return [executable, []]; - } - - // Try .cmd wrapper first (has the script path embedded) - const cmdPath = fullPath.endsWith('.cmd') ? fullPath : `${fullPath}.cmd`; - try { - const content = readFileSync(cmdPath, 'utf-8'); - // npm .cmd wrappers end with: "%_prog%" "%dp0%\path\to\script.js" %* - const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/); - if (match) { - const dp0 = path.dirname(path.resolve(cmdPath)); - const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path.sep}`); - try { - accessSync(scriptPath); - return ['node', [scriptPath]]; - } catch { - // Script not found at resolved path, fall through - } - } - } catch { - // No .cmd wrapper, fall through - } - - return [executable, []]; -} - -async function defaultPiRunner(options: PiRunOptions): Promise { - return await new Promise((resolve, reject) => { - const parts = options.executable.split(/\s+/); - const [resolvedExe, prefixArgs] = resolveWindowsCmd(parts[0]); - const executableArgs = [...prefixArgs, ...parts.slice(1)]; - const allArgs = [...executableArgs, ...options.args]; - - const child = spawn(resolvedExe, allArgs, { - cwd: options.cwd, - env: options.env, - stdio: ['pipe', 'pipe', 'pipe'], - }); - trackChild(child); - - let stdout = ''; - let stderr = ''; - let timedOut = false; - - const onAbort = (): void => { - child.kill('SIGTERM'); - }; - - if (options.signal) { - if (options.signal.aborted) { - onAbort(); - } else { - options.signal.addEventListener('abort', onAbort, { once: true }); - } - } - - let timeoutHandle: NodeJS.Timeout | undefined; - if (options.timeoutMs && options.timeoutMs > 0) { - timeoutHandle = setTimeout(() => { - timedOut = true; - child.kill('SIGTERM'); - }, options.timeoutMs); - timeoutHandle.unref?.(); - } - - child.stdout.setEncoding('utf8'); - child.stdout.on('data', (chunk) => { - stdout += chunk; - options.onStdoutChunk?.(chunk); - }); - - child.stderr.setEncoding('utf8'); - child.stderr.on('data', (chunk) => { - stderr += chunk; - options.onStderrChunk?.(chunk); - }); - - child.stdin.end(); - - const cleanup = (): void => { - if (timeoutHandle) clearTimeout(timeoutHandle); - if (options.signal) options.signal.removeEventListener('abort', onAbort); - }; - - child.on('error', (error) => { - cleanup(); - reject(error); - }); - - child.on('close', (code) => { - cleanup(); - resolve({ - stdout, - stderr, - exitCode: typeof code === 'number' ? code : -1, - timedOut, - }); - }); - }); -} - /** @internal Exported for testing only. */ export const _internal = { extractMessages, diff --git a/packages/core/src/evaluation/providers/pi-process.ts b/packages/core/src/evaluation/providers/pi-process.ts new file mode 100644 index 000000000..01e99f9c1 --- /dev/null +++ b/packages/core/src/evaluation/providers/pi-process.ts @@ -0,0 +1,339 @@ +import { execSync, spawn } from 'node:child_process'; +import { accessSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { trackChild } from '../../runtime/child-tracker.js'; +import type { PiRuntimeResolvedConfig } from './targets.js'; +import type { Message, TargetExecutionEnvelope, TargetExecutionErrorKind } from './types.js'; + +const INLINE_LOG_LIMIT_BYTES = 64 * 1024; + +export interface PiProcessRunOptions { + readonly command: readonly string[]; + readonly cwd: string; + readonly timeoutMs?: number; + readonly env: NodeJS.ProcessEnv; + readonly signal?: AbortSignal; + readonly stdin?: string; + readonly onStdoutChunk?: (chunk: string) => void; + readonly onStderrChunk?: (chunk: string) => void; +} + +export interface PiProcessRunResult { + readonly stdout: string; + readonly stderr: string; + readonly exitCode: number | null; + readonly signal?: string | null; + readonly timedOut?: boolean; + readonly spawnErrorCode?: string; +} + +export type PiProcessRunner = (options: PiProcessRunOptions) => Promise; + +export function splitPiCommand( + command: readonly string[], + extraArgs: readonly string[] = [], +): readonly string[] { + if (command.length === 0) { + throw new Error('Pi provider command argv must not be empty'); + } + return [...command, ...extraArgs]; +} + +export function buildPiRuntimeEnv(params: { + readonly runtime: PiRuntimeResolvedConfig; + readonly targetName: string; + readonly env?: NodeJS.ProcessEnv; +}): NodeJS.ProcessEnv { + const env = { ...(params.env ?? process.env) }; + Object.assign(env, params.runtime.env); + + if (params.runtime.mode === 'profile') { + const home = params.runtime.home + ? path.resolve(params.runtime.home) + : path.resolve('.agentv', 'profiles', params.targetName); + env.HOME = home; + env.XDG_CONFIG_HOME = path.join(home, '.config'); + env.XDG_CACHE_HOME = path.join(home, '.cache'); + env.XDG_DATA_HOME = path.join(home, '.local', 'share'); + env.AGENTV_RUNTIME_PROFILE_HOME = home; + } + + return env; +} + +export function buildPiTargetExecution(params: { + readonly targetName: string; + readonly providerId: string; + readonly providerKind: string; + readonly runtimeMode: string; + readonly command: readonly string[]; + readonly cwd?: string; + readonly timeoutMs?: number; + readonly startedAt: number; + readonly endedAt: number; + readonly result?: PiProcessRunResult; + readonly status: 'success' | 'error'; + readonly errorKind?: TargetExecutionErrorKind; + readonly message?: string; + readonly output?: readonly Message[]; + readonly finalOutput?: string; + readonly details?: Record; +}): TargetExecutionEnvelope { + return { + schemaVersion: 'agentv.target_execution.v1', + status: params.status, + targetId: params.targetName, + providerId: params.providerId, + providerKind: params.providerKind, + runtimeMode: params.runtimeMode, + command: { + argv: params.command, + cwd: params.cwd, + }, + timeoutMs: params.timeoutMs, + startedAt: new Date(params.startedAt).toISOString(), + endedAt: new Date(params.endedAt).toISOString(), + durationMs: params.endedAt - params.startedAt, + exitCode: params.result?.exitCode, + signal: params.result?.signal ?? null, + errorKind: params.errorKind, + message: params.message, + logs: { + stdout: captureLog(params.result?.stdout ?? ''), + stderr: captureLog(params.result?.stderr ?? ''), + }, + transcript: + params.output || params.finalOutput !== undefined + ? { + messages: params.output, + finalOutput: params.finalOutput, + } + : undefined, + details: params.details, + }; +} + +export function classifyPiProcessFailure( + result: PiProcessRunResult, + signalAborted: boolean | undefined, +): TargetExecutionErrorKind { + if (signalAborted) { + return 'cancelled'; + } + if (result.timedOut) { + return 'timeout'; + } + if (result.spawnErrorCode) { + return 'spawn_failure'; + } + if (result.signal && result.exitCode === null) { + return 'signal_crash'; + } + return 'nonzero_exit'; +} + +export function piProcessFailureMessage(params: { + readonly providerLabel: string; + readonly result: PiProcessRunResult; + readonly errorKind: TargetExecutionErrorKind; + readonly timeoutMs?: number; +}): string { + if (params.errorKind === 'cancelled') { + return `${params.providerLabel} request was aborted`; + } + if (params.errorKind === 'timeout') { + return `${params.providerLabel} timed out${formatTimeoutSuffix(params.timeoutMs)}`; + } + if (params.errorKind === 'spawn_failure') { + return ( + params.result.stderr.trim() || + params.result.stdout.trim() || + `${params.providerLabel} failed to spawn (${params.result.spawnErrorCode})` + ); + } + if (params.errorKind === 'signal_crash') { + return `${params.providerLabel} terminated by signal ${params.result.signal ?? 'unknown'}`; + } + const codeText = params.result.exitCode !== null ? params.result.exitCode : 'unknown'; + const detail = params.result.stderr.trim() || params.result.stdout.trim(); + return detail + ? `${params.providerLabel} exited with code ${codeText}: ${detail}` + : `${params.providerLabel} exited with code ${codeText}`; +} + +export function formatTimeoutSuffix(timeoutMs: number | undefined): string { + if (!timeoutMs || timeoutMs <= 0) return ''; + return ` after ${Math.ceil(timeoutMs / 1000)}s`; +} + +export async function defaultPiProcessRunner( + options: PiProcessRunOptions, +): Promise { + return await new Promise((resolve) => { + const [command, ...args] = options.command; + const [resolvedExe, prefixArgs] = resolveWindowsCmd(command); + const allArgs = [...prefixArgs, ...args]; + + const child = spawn(resolvedExe, allArgs, { + cwd: options.cwd, + env: options.env, + stdio: ['pipe', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + }); + trackChild(child); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + let spawnErrorCode: string | undefined; + let settled = false; + + const onAbort = (): void => { + killChildProcessGroup(child); + }; + + if (options.signal) { + if (options.signal.aborted) { + onAbort(); + } else { + options.signal.addEventListener('abort', onAbort, { once: true }); + } + } + + let timeoutHandle: NodeJS.Timeout | undefined; + if (options.timeoutMs && options.timeoutMs > 0) { + timeoutHandle = setTimeout(() => { + timedOut = true; + killChildProcessGroup(child); + }, options.timeoutMs); + timeoutHandle.unref?.(); + } + + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + options.onStdoutChunk?.(chunk); + }); + + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk) => { + stderr += chunk; + options.onStderrChunk?.(chunk); + }); + + const cleanup = (): void => { + if (timeoutHandle) clearTimeout(timeoutHandle); + if (options.signal) options.signal.removeEventListener('abort', onAbort); + }; + + child.on('error', (error: NodeJS.ErrnoException) => { + if (settled) { + return; + } + settled = true; + cleanup(); + spawnErrorCode = error.code; + stderr += error.message; + resolve({ + stdout, + stderr, + exitCode: null, + signal: null, + timedOut, + spawnErrorCode, + }); + }); + + child.on('close', (code, signal) => { + if (settled) { + return; + } + settled = true; + cleanup(); + resolve({ + stdout, + stderr, + exitCode: typeof code === 'number' ? code : null, + signal: signal ? String(signal) : null, + timedOut, + spawnErrorCode, + }); + }); + + child.stdin.end(options.stdin ?? ''); + }); +} + +function captureLog(text: string) { + const bytes = Buffer.byteLength(text, 'utf8'); + if (bytes <= INLINE_LOG_LIMIT_BYTES) { + return { + text, + truncated: false, + bytes, + storedBytes: bytes, + }; + } + + let stored = text; + while (Buffer.byteLength(stored, 'utf8') > INLINE_LOG_LIMIT_BYTES) { + stored = stored.slice(0, Math.max(0, stored.length - 1024)); + } + const storedBytes = Buffer.byteLength(stored, 'utf8'); + return { + text: stored, + truncated: true, + bytes, + storedBytes, + }; +} + +function killChildProcessGroup(child: ReturnType): void { + if (child.pid && process.platform !== 'win32') { + try { + process.kill(-child.pid, 'SIGTERM'); + return; + } catch { + // Fall back to killing the direct child. + } + } + child.kill('SIGTERM'); +} + +function resolveWindowsCmd(executable: string): [string, string[]] { + if (process.platform !== 'win32') return [executable, []]; + + const lower = executable.toLowerCase(); + if (lower.endsWith('.js') || lower.endsWith('.exe')) return [executable, []]; + + let fullPath: string; + try { + fullPath = execSync(`where ${executable}`, { encoding: 'utf-8' }) + .trim() + .split(/\r?\n/)[0] + .trim(); + } catch { + return [executable, []]; + } + + const cmdPath = fullPath.endsWith('.cmd') ? fullPath : `${fullPath}.cmd`; + try { + const content = readFileSync(cmdPath, 'utf-8'); + const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/); + if (match) { + const dp0 = path.dirname(path.resolve(cmdPath)); + const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path.sep}`); + try { + accessSync(scriptPath); + return ['node', [scriptPath]]; + } catch { + // Fall through to original executable. + } + } + } catch { + // No .cmd wrapper. + } + + return [executable, []]; +} diff --git a/packages/core/src/evaluation/providers/pi-rpc.ts b/packages/core/src/evaluation/providers/pi-rpc.ts new file mode 100644 index 000000000..d9fd6f5bd --- /dev/null +++ b/packages/core/src/evaluation/providers/pi-rpc.ts @@ -0,0 +1,448 @@ +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; + +import { + type PiProcessRunResult, + type PiProcessRunner, + buildPiRuntimeEnv, + buildPiTargetExecution, + classifyPiProcessFailure, + defaultPiProcessRunner, + piProcessFailureMessage, +} from './pi-process.js'; +import { extractPiTextContent, toFiniteNumber } from './pi-utils.js'; +import { normalizeInputFiles } from './preread.js'; +import type { PiRpcResolvedConfig } from './targets.js'; +import type { + Message, + Provider, + ProviderRequest, + ProviderResponse, + ProviderTokenUsage, +} from './types.js'; +import { extractLastAssistantContent } from './types.js'; + +export class PiRpcProvider implements Provider { + readonly id: string; + readonly kind = 'pi-rpc' as const; + readonly targetName: string; + readonly supportsBatch = false; + + private readonly config: PiRpcResolvedConfig; + private readonly runPi: PiProcessRunner; + + constructor( + targetName: string, + config: PiRpcResolvedConfig, + runner: PiProcessRunner = defaultPiProcessRunner, + ) { + this.id = `pi-rpc:${targetName}`; + this.targetName = targetName; + this.config = config; + this.runPi = runner; + } + + async invoke(request: ProviderRequest): Promise { + if (request.signal?.aborted) { + throw new Error('Pi RPC request was aborted before execution'); + } + + const startedAt = Date.now(); + const cwd = this.resolveCwd(request.cwd); + const command = ensureRpcMode(this.config.command); + const inputFiles = normalizeInputFiles(request.inputFiles); + const rpcRequest = buildRpcRequest({ + request, + config: this.config, + inputFiles, + }); + + const result = await this.runPi({ + command, + cwd, + timeoutMs: this.config.timeoutMs, + env: buildPiRuntimeEnv({ runtime: this.config.runtime, targetName: this.targetName }), + signal: request.signal, + stdin: `${JSON.stringify(rpcRequest)}\n`, + }); + + if (result.timedOut || result.exitCode !== 0 || result.signal || result.spawnErrorCode) { + return this.buildProcessErrorResponse({ + result, + command, + cwd, + startedAt, + signalAborted: request.signal?.aborted, + }); + } + + let parsed: ParsedRpcOutput; + try { + parsed = parseRpcOutput(result.stdout, rpcRequest.id); + } catch (error) { + return this.buildProtocolErrorResponse({ + result, + command, + cwd, + startedAt, + message: error instanceof Error ? error.message : String(error), + }); + } + + if (parsed.error) { + return this.buildRpcTaskFailureResponse({ + result, + command, + cwd, + startedAt, + message: parsed.error, + events: parsed.events, + }); + } + + const output = messagesFromRpcResult(parsed.result); + const tokenUsage = tokenUsageFromRpcResult(parsed.result); + const endedAt = Date.now(); + const finalOutput = extractLastAssistantContent(output); + + return { + raw: { + response: parsed.result, + events: parsed.events, + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + command, + cwd, + inputFiles, + }, + output, + tokenUsage, + durationMs: endedAt - startedAt, + startTime: new Date(startedAt).toISOString(), + endTime: new Date(endedAt).toISOString(), + targetExecution: buildPiTargetExecution({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + runtimeMode: this.config.runtime.mode, + command, + cwd, + timeoutMs: this.config.timeoutMs, + startedAt, + endedAt, + result, + status: 'success', + output, + finalOutput, + details: { events: parsed.events, inputFiles }, + }), + }; + } + + private resolveCwd(cwdOverride?: string): string { + if (cwdOverride) { + return path.resolve(cwdOverride); + } + if (this.config.cwd) { + return path.resolve(this.config.cwd); + } + return process.cwd(); + } + + private buildProcessErrorResponse(params: { + readonly result: PiProcessRunResult; + readonly command: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly signalAborted?: boolean; + }): ProviderResponse { + const errorKind = classifyPiProcessFailure(params.result, params.signalAborted); + const message = piProcessFailureMessage({ + providerLabel: 'Pi RPC', + result: params.result, + errorKind, + timeoutMs: this.config.timeoutMs, + }); + return this.errorResponse({ + ...params, + errorKind, + message, + details: { spawnErrorCode: params.result.spawnErrorCode }, + }); + } + + private buildProtocolErrorResponse(params: { + readonly result: PiProcessRunResult; + readonly command: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly message: string; + }): ProviderResponse { + return this.errorResponse({ + ...params, + errorKind: 'malformed_output', + message: `Pi RPC malformed protocol output: ${params.message}`, + }); + } + + private buildRpcTaskFailureResponse(params: { + readonly result: PiProcessRunResult; + readonly command: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly message: string; + readonly events: readonly unknown[]; + }): ProviderResponse { + return this.errorResponse({ + ...params, + errorKind: 'target_task_failure', + message: params.message, + details: { events: params.events }, + }); + } + + private errorResponse(params: { + readonly result: PiProcessRunResult; + readonly command: readonly string[]; + readonly cwd: string; + readonly startedAt: number; + readonly errorKind: + | 'target_task_failure' + | 'malformed_output' + | ReturnType; + readonly message: string; + readonly details?: Record; + }): ProviderResponse { + const endedAt = Date.now(); + const output = [{ role: 'assistant' as const, content: `Error: ${params.message}` }]; + return { + raw: { + stdout: params.result.stdout, + stderr: params.result.stderr, + exitCode: params.result.exitCode, + signal: params.result.signal, + command: params.command, + cwd: params.cwd, + error: params.message, + }, + output, + durationMs: endedAt - params.startedAt, + startTime: new Date(params.startedAt).toISOString(), + endTime: new Date(endedAt).toISOString(), + targetExecution: buildPiTargetExecution({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + runtimeMode: this.config.runtime.mode, + command: params.command, + cwd: params.cwd, + timeoutMs: this.config.timeoutMs, + startedAt: params.startedAt, + endedAt, + result: params.result, + status: 'error', + errorKind: params.errorKind, + message: params.message, + output, + finalOutput: `Error: ${params.message}`, + details: params.details, + }), + }; + } +} + +type RpcRequest = { + readonly jsonrpc: '2.0'; + readonly id: string; + readonly method: 'run'; + readonly params: Record; +}; + +type ParsedRpcOutput = { + readonly result?: unknown; + readonly error?: string; + readonly events: readonly unknown[]; +}; + +function buildRpcRequest(params: { + readonly request: ProviderRequest; + readonly config: PiRpcResolvedConfig; + readonly inputFiles?: readonly string[]; +}): RpcRequest { + const rpcParams: Record = { + prompt: params.request.question, + system_prompt: params.config.systemPrompt ?? params.request.systemPrompt, + model: params.config.model, + subprovider: params.config.subprovider, + tools: params.config.tools, + thinking: params.config.thinking, + input_files: params.inputFiles, + metadata: params.request.metadata, + }; + for (const key of Object.keys(rpcParams)) { + if (rpcParams[key] === undefined) { + delete rpcParams[key]; + } + } + return { + jsonrpc: '2.0', + id: randomUUID(), + method: 'run', + params: rpcParams, + }; +} + +function ensureRpcMode(command: readonly string[]): readonly string[] { + const hasMode = command.some( + (arg, index) => + arg === '--mode' || arg.startsWith('--mode=') || command[index - 1] === '--mode', + ); + return hasMode ? command : [...command, '--mode', 'rpc']; +} + +function parseRpcOutput(stdout: string, requestId: string): ParsedRpcOutput { + const lines = stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0); + if (lines.length === 0) { + throw new Error('Pi RPC produced no protocol messages'); + } + + const events: unknown[] = []; + let result: unknown; + let error: string | undefined; + + for (const line of lines) { + let message: Record; + try { + const parsed = JSON.parse(line); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('message is not an object'); + } + message = parsed as Record; + } catch (parseError) { + throw new Error(`invalid JSON protocol message: ${formatError(parseError)}`); + } + + if (message.id === requestId && Object.prototype.hasOwnProperty.call(message, 'result')) { + result = message.result; + continue; + } + if (message.id === requestId && Object.prototype.hasOwnProperty.call(message, 'error')) { + error = rpcErrorMessage(message.error); + continue; + } + if (message.type === 'result') { + result = message.result ?? message.response ?? message; + continue; + } + if (message.type === 'error') { + error = rpcErrorMessage(message.error ?? message.message); + continue; + } + events.push(message); + } + + if (error) { + return { error, events }; + } + if (result === undefined) { + throw new Error('Pi RPC disconnected before a result message'); + } + return { result, events }; +} + +function messagesFromRpcResult(result: unknown): readonly Message[] { + if (result && typeof result === 'object') { + const record = result as Record; + if (Array.isArray(record.output)) { + return record.output.map(convertRpcMessage).filter((msg): msg is Message => Boolean(msg)); + } + if (Array.isArray(record.messages)) { + return record.messages.map(convertRpcMessage).filter((msg): msg is Message => Boolean(msg)); + } + const content = record.text ?? record.output_text ?? record.content; + if (typeof content === 'string') { + return [{ role: 'assistant', content }]; + } + } + if (typeof result === 'string') { + return [{ role: 'assistant', content: result }]; + } + return [{ role: 'assistant', content: JSON.stringify(result) }]; +} + +function convertRpcMessage(value: unknown): Message | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return undefined; + } + const record = value as Record; + const role = typeof record.role === 'string' ? record.role : undefined; + if (!role) { + return undefined; + } + const content = extractPiTextContent(record.content) ?? stringField(record, 'text'); + return { + role, + content, + metadata: + record.metadata && typeof record.metadata === 'object' + ? (record.metadata as Record) + : undefined, + }; +} + +function tokenUsageFromRpcResult(result: unknown): ProviderTokenUsage | undefined { + if (!result || typeof result !== 'object') { + return undefined; + } + const record = result as Record; + const usage = (record.token_usage ?? record.tokenUsage ?? record.usage) as + | Record + | undefined; + if (!usage || typeof usage !== 'object') { + return undefined; + } + const input = toFiniteNumber(usage.input ?? usage.input_tokens ?? usage.inputTokens); + const output = toFiniteNumber(usage.output ?? usage.output_tokens ?? usage.outputTokens); + if (input === undefined && output === undefined) { + return undefined; + } + const cached = toFiniteNumber(usage.cached ?? usage.cache_read_input_tokens); + const reasoning = toFiniteNumber(usage.reasoning ?? usage.reasoning_tokens); + return { + input: input ?? 0, + output: output ?? 0, + ...(cached !== undefined ? { cached } : {}), + ...(reasoning !== undefined ? { reasoning } : {}), + }; +} + +function stringField(record: Record, key: string): string | undefined { + const value = record[key]; + return typeof value === 'string' ? value : undefined; +} + +function rpcErrorMessage(value: unknown): string { + if (typeof value === 'string') { + return value; + } + if (value && typeof value === 'object') { + const record = value as Record; + if (typeof record.message === 'string') { + return record.message; + } + } + return JSON.stringify(value); +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export const _internal = { + ensureRpcMode, + parseRpcOutput, +}; diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 8e6dbd0d4..64df4a9ab 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -522,6 +522,7 @@ export interface PiCodingAgentResolvedConfig { } export interface PiCliResolvedConfig { + readonly command: readonly string[]; readonly executable: string; readonly subprovider?: string; readonly model?: string; @@ -529,7 +530,6 @@ export interface PiCliResolvedConfig { readonly baseUrl?: string; readonly tools?: string; readonly thinking?: string; - readonly args?: readonly string[]; readonly cwd?: string; readonly timeoutMs?: number; readonly logDir?: string; @@ -537,6 +537,31 @@ export interface PiCliResolvedConfig { /** New stream_log field. false=no stream log (default), 'raw'=per-event, 'summary'=consolidated. */ readonly streamLog?: false | 'raw' | 'summary'; readonly systemPrompt?: string; + readonly runtime: PiRuntimeResolvedConfig; +} + +export interface PiRpcResolvedConfig { + readonly command: readonly string[]; + readonly subprovider?: string; + readonly model?: string; + readonly apiKey?: string; + readonly baseUrl?: string; + readonly tools?: string; + readonly thinking?: string; + readonly cwd?: string; + readonly timeoutMs?: number; + readonly logDir?: string; + readonly logFormat?: 'summary' | 'json'; + readonly streamLog?: false | 'raw' | 'summary'; + readonly systemPrompt?: string; + readonly runtime: PiRuntimeResolvedConfig; +} + +export interface PiRuntimeResolvedConfig { + readonly mode: 'host' | 'profile' | 'sandbox'; + readonly home?: string; + readonly env?: Readonly>; + readonly [key: string]: unknown; } export interface ClaudeResolvedConfig { @@ -861,6 +886,7 @@ export type ResolvedTarget = readonly config: PiCodingAgentResolvedConfig; }) | (ResolvedTargetBase & { readonly kind: 'pi-cli'; readonly config: PiCliResolvedConfig }) + | (ResolvedTargetBase & { readonly kind: 'pi-rpc'; readonly config: PiRpcResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'claude'; readonly config: ClaudeResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'claude-cli'; readonly config: ClaudeResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'claude-sdk'; readonly config: ClaudeResolvedConfig }) @@ -1165,6 +1191,12 @@ export function resolveTargetDefinition( ...base, config: resolvePiCliConfig(parsed, env, evalFilePath), }; + case 'pi-rpc': + return { + kind: 'pi-rpc', + ...base, + config: resolvePiRpcConfig(parsed, env, evalFilePath), + }; case 'claude': case 'claude-cli': return { @@ -2076,7 +2108,7 @@ function resolvePiCliConfig( env: EnvLookup, _evalFilePath?: string, ): PiCliResolvedConfig { - const executableSource = target.executable ?? target.command ?? target.binary; + const command = resolveProcessCommandArgv(target, env, `${target.name} pi-cli command`, ['pi']); const subproviderSource = target.subprovider; const modelSource = target.model ?? target.pi_model; const apiKeySource = target.api_key; @@ -2089,12 +2121,6 @@ function resolvePiCliConfig( const streamLogResult = resolveStreamLog(target); - const executable = - resolveOptionalString(executableSource, env, `${target.name} pi-cli executable`, { - allowLiteral: true, - optionalEnv: true, - }) ?? 'pi'; - const subprovider = resolveOptionalString( subproviderSource, env, @@ -2129,9 +2155,6 @@ function resolvePiCliConfig( optionalEnv: true, }); - const rawArgs = target.args ?? target.arguments; - const args = resolveOptionalStringArray(rawArgs, env, `${target.name} pi-cli args`); - const cwd = resolveOptionalString(cwdSource, env, `${target.name} pi-cli cwd`, { allowLiteral: true, optionalEnv: true, @@ -2150,20 +2173,97 @@ function resolvePiCliConfig( : undefined; return { - executable, + command, + executable: command[0], subprovider: piCliSubprovider, model, apiKey, baseUrl, tools, thinking, - args, cwd, timeoutMs, logDir, logFormat: streamLogResult.logFormat, streamLog: streamLogResult.streamLog, systemPrompt, + runtime: resolvePiRuntimeConfig(target, env), + }; +} + +function resolvePiRpcConfig( + target: z.infer, + env: EnvLookup, + _evalFilePath?: string, +): PiRpcResolvedConfig { + const command = resolveProcessCommandArgv(target, env, `${target.name} pi-rpc command`, ['pi']); + const subproviderSource = target.subprovider; + const modelSource = target.model ?? target.pi_model; + const apiKeySource = target.api_key; + const toolsSource = target.tools ?? target.pi_tools; + const thinkingSource = target.reasoning_effort ?? target.thinking ?? target.pi_thinking; + const cwdSource = target.cwd; + const timeoutSource = target.timeout_seconds; + const logDirSource = target.log_dir ?? target.log_directory; + const systemPromptSource = target.system_prompt; + const streamLogResult = resolveStreamLog(target); + + const subprovider = resolveOptionalString( + subproviderSource, + env, + `${target.name} pi-rpc subprovider`, + { allowLiteral: true, optionalEnv: true }, + ); + const model = resolveOptionalString(modelSource, env, `${target.name} pi-rpc model`, { + allowLiteral: true, + optionalEnv: true, + }); + const apiKey = resolveOptionalString(apiKeySource, env, `${target.name} pi-rpc api key`, { + allowLiteral: false, + optionalEnv: true, + }); + const baseUrlSource = target.base_url ?? target.endpoint; + const baseUrl = resolveOptionalString(baseUrlSource, env, `${target.name} pi-rpc base url`, { + allowLiteral: true, + optionalEnv: true, + }); + const tools = resolveOptionalString(toolsSource, env, `${target.name} pi-rpc tools`, { + allowLiteral: true, + optionalEnv: true, + }); + const thinking = resolveOptionalString(thinkingSource, env, `${target.name} pi-rpc thinking`, { + allowLiteral: true, + optionalEnv: true, + }); + const cwd = resolveOptionalString(cwdSource, env, `${target.name} pi-rpc cwd`, { + allowLiteral: true, + optionalEnv: true, + }); + const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} pi-rpc timeout`); + const logDir = resolveOptionalString(logDirSource, env, `${target.name} pi-rpc log directory`, { + allowLiteral: true, + optionalEnv: true, + }); + const systemPrompt = + typeof systemPromptSource === 'string' && systemPromptSource.trim().length > 0 + ? systemPromptSource.trim() + : undefined; + + return { + command, + subprovider, + model, + apiKey, + baseUrl, + tools, + thinking, + cwd, + timeoutMs, + logDir, + logFormat: streamLogResult.logFormat, + streamLog: streamLogResult.streamLog, + systemPrompt, + runtime: resolvePiRuntimeConfig(target, env), }; } @@ -2181,6 +2281,101 @@ function normalizePiCliSubprovider( return subprovider; } +function resolveProcessCommandArgv( + target: z.infer, + env: EnvLookup, + description: string, + defaultCommand: readonly string[], +): readonly string[] { + const rawCommand = target.command; + if (Array.isArray(rawCommand)) { + const command = resolveOptionalStringArray(rawCommand, env, description); + if (!command || command.length === 0) { + throw new Error(`${description} must be a non-empty argv array`); + } + return command; + } + + const executableSource = target.executable ?? target.command ?? target.binary; + const executable = resolveOptionalString(executableSource, env, `${description} executable`, { + allowLiteral: true, + optionalEnv: true, + }); + const args = resolveOptionalStringArray( + target.args ?? target.arguments, + env, + `${description} args`, + ); + return [...(executable ? [executable] : defaultCommand), ...(args ?? [])]; +} + +function resolvePiRuntimeConfig( + target: z.infer, + env: EnvLookup, +): PiRuntimeResolvedConfig { + const raw = target.runtime; + if (raw === undefined || raw === null) { + return { mode: 'host' }; + } + if (typeof raw === 'string') { + return { mode: normalizeRuntimeMode(raw, target.name) }; + } + if (!isRecord(raw)) { + throw new Error(`${target.name} runtime must be 'host' or an object with mode`); + } + + const mode = normalizeRuntimeMode(String(raw.mode ?? ''), target.name); + const home = resolveOptionalString(raw.home, env, `${target.name} runtime home`, { + allowLiteral: true, + optionalEnv: true, + }); + const runtimeEnv = resolveRuntimeEnv(raw.env, env, target.name); + const rest = Object.fromEntries( + Object.entries(raw).filter(([key]) => key !== 'mode' && key !== 'home' && key !== 'env'), + ); + return { + ...rest, + mode, + ...(home ? { home } : {}), + ...(runtimeEnv ? { env: runtimeEnv } : {}), + }; +} + +function normalizeRuntimeMode(value: string, targetName: string): PiRuntimeResolvedConfig['mode'] { + const mode = value.trim(); + if (mode === 'host' || mode === 'profile' || mode === 'sandbox') { + return mode; + } + throw new Error(`${targetName} runtime.mode must be one of: host, profile, sandbox`); +} + +function resolveRuntimeEnv( + raw: unknown, + env: EnvLookup, + targetName: string, +): Readonly> | undefined { + if (raw === undefined || raw === null) { + return undefined; + } + if (!isRecord(raw)) { + throw new Error(`${targetName} runtime.env must be an object`); + } + const result: Record = {}; + for (const [key, value] of Object.entries(raw)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) { + throw new Error(`${targetName} runtime.env has invalid variable name '${key}'`); + } + const resolved = resolveOptionalString(value, env, `${targetName} runtime.env.${key}`, { + allowLiteral: true, + optionalEnv: true, + }); + if (resolved !== undefined) { + result[key] = resolved; + } + } + return Object.keys(result).length > 0 ? result : undefined; +} + function resolveClaudeConfig( target: z.infer, env: EnvLookup, diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 834b7c5ec..01c6ba20c 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -27,6 +27,7 @@ export type ProviderKind = | 'pi-sdk' | 'pi-coding-agent' | 'pi-cli' + | 'pi-rpc' | 'claude' | 'claude-cli' | 'claude-sdk' @@ -55,6 +56,7 @@ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ 'pi-sdk', 'pi-coding-agent', 'pi-cli', + 'pi-rpc', 'claude', 'claude-cli', 'claude-sdk', @@ -99,6 +101,7 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ 'pi-sdk', 'pi-coding-agent', 'pi-cli', + 'pi-rpc', 'claude', 'claude-cli', 'claude-sdk', diff --git a/packages/core/src/evaluation/transcript-summary.ts b/packages/core/src/evaluation/transcript-summary.ts index 81e6f2bfa..1f53dd8cd 100644 --- a/packages/core/src/evaluation/transcript-summary.ts +++ b/packages/core/src/evaluation/transcript-summary.ts @@ -50,6 +50,7 @@ const PROVIDER_ALIASES: Readonly> = { 'copilot-log': 'copilot-log', pi: 'pi-cli', 'pi-cli': 'pi-cli', + 'pi-rpc': 'pi-rpc', 'pi-coding-agent': 'pi-coding-agent', claude: 'claude', 'claude-cli': 'claude-cli', diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index 3d33dacc2..ae86a6b31 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -165,6 +165,33 @@ const COPILOT_CLI_SETTINGS = new Set([ 'wire_model', ]); +const PI_AGENT_SETTINGS = new Set([ + ...COMMON_SETTINGS, + 'executable', + 'command', + 'binary', + 'args', + 'arguments', + 'subprovider', + 'model', + 'pi_model', + 'api_key', + 'base_url', + 'endpoint', + 'tools', + 'pi_tools', + 'thinking', + 'pi_thinking', + 'reasoning_effort', + 'cwd', + 'timeout_seconds', + 'log_dir', + 'log_directory', + 'stream_log', + 'system_prompt', + 'runtime', +]); + const VSCODE_SETTINGS = new Set([ ...COMMON_SETTINGS, 'executable', @@ -234,6 +261,11 @@ function getKnownSettings(provider: string): Set | null { case 'claude-cli': case 'claude-sdk': return CLAUDE_SETTINGS; + case 'pi-cli': + case 'pi-rpc': + case 'pi-sdk': + case 'pi-coding-agent': + return PI_AGENT_SETTINGS; case 'vscode': case 'vscode-insiders': return VSCODE_SETTINGS; diff --git a/packages/core/test/evaluation/providers/pi-runtime.test.ts b/packages/core/test/evaluation/providers/pi-runtime.test.ts new file mode 100644 index 000000000..961d49a5c --- /dev/null +++ b/packages/core/test/evaluation/providers/pi-runtime.test.ts @@ -0,0 +1,220 @@ +import { describe, expect, it, mock } from 'bun:test'; + +import { + createProvider, + resolveTargetDefinition, +} from '../../../src/evaluation/providers/index.js'; +import { PiCliProvider } from '../../../src/evaluation/providers/pi-cli.js'; +import type { PiProcessRunOptions } from '../../../src/evaluation/providers/pi-process.js'; +import { PiRpcProvider, _internal } from '../../../src/evaluation/providers/pi-rpc.js'; +import type { + PiCliResolvedConfig, + PiRpcResolvedConfig, +} from '../../../src/evaluation/providers/targets.js'; +import { extractLastAssistantContent } from '../../../src/evaluation/providers/types.js'; + +describe('Pi coding-agent runtime providers', () => { + it('passes config.command argv and profile runtime env to pi-cli subprocesses', async () => { + let captured: PiProcessRunOptions | undefined; + const runner = mock(async (options: PiProcessRunOptions) => { + captured = options; + return { + stdout: `${JSON.stringify({ + type: 'agent_end', + messages: [{ role: 'assistant', content: [{ type: 'text', text: 'cli ok' }] }], + })}\n`, + stderr: '', + exitCode: 0, + }; + }); + const provider = new PiCliProvider('pi-cli-target', baseCliConfig(), runner); + + const response = await provider.invoke({ question: 'hello', cwd: '/tmp/workspace' }); + + expect(extractLastAssistantContent(response.output)).toBe('cli ok'); + expect(captured?.command.slice(0, 3)).toEqual(['pi-shim', '--profile', 'clean']); + expect(captured?.command).toContain('--mode'); + expect(captured?.command).toContain('json'); + expect(captured?.env.HOME).toBe('/tmp/pi-profile'); + expect(captured?.env.PI_TEST_FLAG).toBe('enabled'); + expect(response.targetExecution?.status).toBe('success'); + expect(response.targetExecution?.runtimeMode).toBe('profile'); + expect(response.targetExecution?.command?.argv?.slice(0, 3)).toEqual([ + 'pi-shim', + '--profile', + 'clean', + ]); + }); + + it('returns structured pi-cli malformed-output errors', async () => { + const provider = new PiCliProvider('pi-cli-target', baseCliConfig(), async () => ({ + stdout: 'not json\n', + stderr: '', + exitCode: 0, + })); + + const response = await provider.invoke({ question: 'hello', cwd: '/tmp/workspace' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('malformed_output'); + expect(extractLastAssistantContent(response.output)).toMatch(/malformed output/i); + }); + + it('runs pi-rpc over process stdio and returns fake RPC success', async () => { + let captured: PiProcessRunOptions | undefined; + const provider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async (options) => { + captured = options; + const request = JSON.parse(options.stdin?.trim() ?? '{}') as { id: string }; + return { + stdout: `${JSON.stringify({ type: 'event', event: { kind: 'start' } })}\n${JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + result: { + output: [{ role: 'assistant', content: 'rpc ok' }], + token_usage: { input: 2, output: 3 }, + }, + })}\n`, + stderr: '', + exitCode: 0, + }; + }); + + const response = await provider.invoke({ question: 'hello rpc', cwd: '/tmp/workspace' }); + + expect(captured?.command).toEqual(['pi', '--mode', 'rpc']); + expect(captured?.stdin).toContain('"method":"run"'); + expect(extractLastAssistantContent(response.output)).toBe('rpc ok'); + expect(response.tokenUsage).toEqual({ input: 2, output: 3 }); + expect(response.targetExecution?.status).toBe('success'); + expect(response.targetExecution?.providerKind).toBe('pi-rpc'); + }); + + it('maps pi-rpc protocol errors to malformed target envelopes', async () => { + const provider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async () => ({ + stdout: 'not-json\n', + stderr: '', + exitCode: 0, + })); + + const response = await provider.invoke({ question: 'hello rpc', cwd: '/tmp/workspace' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('malformed_output'); + expect(response.targetExecution?.message).toMatch(/malformed protocol/i); + }); + + it('maps pi-rpc result errors to target task failures', async () => { + const provider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async (options) => { + const request = JSON.parse(options.stdin?.trim() ?? '{}') as { id: string }; + return { + stdout: `${JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + error: { message: 'task failed' }, + })}\n`, + stderr: '', + exitCode: 0, + }; + }); + + const response = await provider.invoke({ question: 'hello rpc', cwd: '/tmp/workspace' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('target_task_failure'); + expect(response.targetExecution?.message).toBe('task failed'); + }); + + it('maps pi-rpc timeout and crash failures to target envelopes', async () => { + const timeoutProvider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async () => ({ + stdout: 'partial', + stderr: '', + exitCode: null, + timedOut: true, + signal: 'SIGTERM', + })); + const crashProvider = new PiRpcProvider('pi-rpc-target', baseRpcConfig(), async () => ({ + stdout: 'partial', + stderr: 'crashed', + exitCode: null, + signal: 'SIGSEGV', + })); + + const timeout = await timeoutProvider.invoke({ question: 'hello', cwd: '/tmp/workspace' }); + const crash = await crashProvider.invoke({ question: 'hello', cwd: '/tmp/workspace' }); + + expect(timeout.targetExecution?.errorKind).toBe('timeout'); + expect(crash.targetExecution?.errorKind).toBe('signal_crash'); + expect(crash.targetExecution?.logs?.stderr?.text).toBe('crashed'); + }); + + it('resolves pi-cli and pi-rpc command argv without disturbing plain LLM providers', async () => { + const piCli = resolveTargetDefinition( + { + id: 'pi-cli-id', + name: 'pi-cli-id', + provider: 'pi-cli', + runtime: { mode: 'profile', home: '/tmp/pi-home' }, + config: { + command: ['pi-shim', '--profile', 'clean'], + model: 'gpt-5-codex', + }, + } as never, + {}, + ); + const piRpc = resolveTargetDefinition( + { + id: 'pi-rpc-id', + name: 'pi-rpc-id', + provider: 'pi-rpc', + runtime: 'host', + config: { command: ['pi'] }, + } as never, + {}, + ); + const llm = resolveTargetDefinition( + { name: 'mock-llm', provider: 'mock', response: 'still works' }, + {}, + ); + + expect(piCli.kind).toBe('pi-cli'); + if (piCli.kind !== 'pi-cli') throw new Error('expected pi-cli'); + expect(piCli.config.command).toEqual(['pi-shim', '--profile', 'clean']); + expect(piCli.config.runtime).toEqual({ mode: 'profile', home: '/tmp/pi-home' }); + + expect(piRpc.kind).toBe('pi-rpc'); + if (piRpc.kind !== 'pi-rpc') throw new Error('expected pi-rpc'); + expect(piRpc.config.command).toEqual(['pi']); + + const provider = createProvider(llm); + const response = await provider.invoke({ question: 'hello' }); + expect(extractLastAssistantContent(response.output)).toBe('still works'); + }); + + it('does not append duplicate RPC mode flags', () => { + expect(_internal.ensureRpcMode(['pi', '--mode', 'rpc'])).toEqual(['pi', '--mode', 'rpc']); + expect(_internal.ensureRpcMode(['pi', '--mode=rpc'])).toEqual(['pi', '--mode=rpc']); + }); +}); + +function baseCliConfig(): PiCliResolvedConfig { + return { + command: ['pi-shim', '--profile', 'clean'], + executable: 'pi-shim', + model: 'gpt-5-codex', + runtime: { + mode: 'profile', + home: '/tmp/pi-profile', + env: { PI_TEST_FLAG: 'enabled' }, + }, + timeoutMs: 1_000, + }; +} + +function baseRpcConfig(): PiRpcResolvedConfig { + return { + command: ['pi'], + model: 'gpt-5-codex', + runtime: { mode: 'host' }, + timeoutMs: 1_000, + }; +} From bb33adf2cc53a60ad8561bd1527feee6883190f8 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 17:28:59 +0200 Subject: [PATCH 07/10] feat(providers): align coding agent target contracts --- .../docs/docs/next/targets/coding-agents.mdx | 75 +++-- .../docs/docs/next/targets/configuration.mdx | 2 + .../features/agent-skills-evals/README.md | 6 +- .../multi-provider-skill-trigger.EVAL.yaml | 6 +- .../src/evaluation/providers/claude-cli.ts | 155 ++++++++- .../core/src/evaluation/providers/claude.ts | 10 +- packages/core/src/evaluation/providers/cli.ts | 55 +--- .../src/evaluation/providers/copilot-cli.ts | 304 +++++++++++------- .../core/src/evaluation/providers/index.ts | 6 - .../providers/normalize-tool-call.ts | 17 +- .../evaluation/providers/target-execution.ts | 87 +++++ .../core/src/evaluation/providers/targets.ts | 89 ++--- .../core/src/evaluation/providers/types.ts | 4 - .../core/src/evaluation/transcript-summary.ts | 2 +- .../validation/targets-validator.ts | 8 +- packages/core/src/import/claude-parser.ts | 4 +- .../providers/claude-provider-aliases.test.ts | 50 ++- .../evaluation/providers/copilot-cli.test.ts | 39 +++ .../providers/normalize-tool-call.test.ts | 4 +- .../test/evaluation/providers/targets.test.ts | 60 ++-- .../validation/targets-validator.test.ts | 26 +- 21 files changed, 708 insertions(+), 301 deletions(-) create mode 100644 packages/core/src/evaluation/providers/target-execution.ts diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index d764c9336..89aa0e508 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -109,16 +109,26 @@ The preread block instructs the agent to read input files before processing the ```yaml targets: - - label: claude_agent - provider: claude - grader_target: azure-base + - id: claude-local + provider: claude-cli + runtime: host + config: + command: ["claude"] + model: claude-sonnet-4-20250514 ``` | Field | Required | Description | |-------|----------|-------------| -| `executable` | No | CLI binary name or path (default: `claude`). Accepts a bare name looked up on PATH or an absolute/relative file path. | -| `cwd` | No | Working directory | -| `grader_target` | Yes | LLM target for evaluation | +| `config.command` | No | Claude CLI argv. Defaults to `["claude"]`; use this for host shims or alternate installed binaries. | +| `config.model` | No | Model to pass to the Claude CLI. | +| `config.cwd` | No | Working directory. | +| `config.timeout_seconds` | No | Per-case timeout. | +| `config.max_turns` | No | Claude CLI turn limit. | +| `config.bypass_permissions` | No | Defaults to true for autonomous eval runs. Set false to omit `--dangerously-skip-permissions`. | + +Use `provider: claude-cli` when you want AgentV to spawn the same Claude CLI a user would run locally. It captures Claude's structured stream output when available and stores stdout, stderr, final output, and target execution metadata in the run bundle. + +Use `provider: claude-sdk` only when you intentionally want the Claude Agent SDK path. AgentV runs SDK providers inside an isolated child runner, so SDK crashes, malformed child output, timeouts, and missing optional SDK packages are reported as target execution errors rather than crashing the AgentV process. SDK transcripts may be richer for SDK-native events, while CLI transcripts better match the installed local agent. ## Codex CLI @@ -146,40 +156,44 @@ Use `provider: codex-sdk` only when you intentionally want the Codex SDK path. T ```yaml targets: - - label: copilot + - id: copilot-local provider: copilot-cli - model: gpt-5-mini - grader_target: azure-base + runtime: host + config: + command: ["copilot"] + model: gpt-5-mini ``` | Field | Required | Description | |-------|----------|-------------| -| `model` | No | Model to use (defaults to copilot's default) | -| `cwd` | No | Working directory | -| `subprovider` | No | OpenAI-compatible provider type for `copilot-cli` or `copilot-sdk`, such as `openai` or `azure` | -| `base_url` | No | Provider base URL or Azure resource URL/name | -| `api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. | -| `bearer_token` | No | Provider bearer token. Prefer `${{ ENV_VAR }}` references. Takes precedence over `api_key` when set. | -| `api_version` | No | Provider API version, primarily for Azure endpoints | -| `api_format` | No | Provider API format, such as `responses` | -| `grader_target` | Yes | LLM target for evaluation | +| `config.command` | No | Copilot CLI argv. Defaults to `["copilot"]`; use this for host shims or alternate installed binaries. | +| `config.model` | No | Model to use. Defaults to Copilot's configured default. | +| `config.cwd` | No | Working directory. | +| `config.subprovider` | No | OpenAI-compatible provider type for `copilot-cli` or `copilot-sdk`, such as `openai` or `azure`. | +| `config.base_url` | No | Provider base URL or Azure resource URL/name. | +| `config.api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. | +| `config.bearer_token` | No | Provider bearer token. Prefer `${{ ENV_VAR }}` references. Takes precedence over `api_key` when set. | +| `config.api_version` | No | Provider API version, primarily for Azure endpoints. | +| `config.api_format` | No | Provider API format, such as `responses`. | Route Copilot through an OpenAI-compatible endpoint: ```yaml targets: - - label: copilot-openai + - id: copilot-openai provider: copilot-cli - subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - api_format: responses - grader_target: azure-base + runtime: host + config: + command: ["copilot"] + subprovider: openai + base_url: ${{ OPENAI_ENDPOINT }} + api_key: ${{ OPENAI_API_KEY }} + api_format: responses ``` Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps these flat fields to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged. -Use `provider: copilot-sdk` only when you intentionally want the Copilot SDK path. The SDK package is optional and loaded inside the isolated child runner. +Use `provider: copilot-cli` for active eval runs through Copilot's ACP subprocess. Use `provider: copilot-log` when you want a passive transcript reader for an existing Copilot CLI session under `~/.copilot/session-state/`; it does not run a new agent and is useful for zero-cost transcript grading. Use `provider: copilot-sdk` only when you intentionally want the Copilot SDK path. SDK providers run inside AgentV's isolated child runner; CLI and log providers keep the boundary at the installed process or existing session logs. CLI runs generally provide the best "what the user actually ran" match, SDK runs may expose SDK-native events, and log runs are limited to the transcript richness already present on disk. ## Pi SDK @@ -350,19 +364,20 @@ The VS Code provider uses a **subagent file-messaging architecture**. AgentV pro ### Copilot CLI - **MCP OAuth token expiration**: If your copilot CLI has MCP servers configured that use OAuth authentication, **expired tokens will block eval execution**. The copilot CLI attempts to re-authenticate via a browser OAuth flow, which cannot complete in non-interactive mode and causes the eval to hang indefinitely. Before running evals, either re-authenticate your MCP servers manually (`copilot` → `/mcp`) or remove MCP servers with expired tokens. See [copilot-cli#1797](https://github.com/github/copilot-cli/issues/1797) and [copilot-cli#1491](https://github.com/github/copilot-cli/issues/1491) for upstream tracking. -- **Windows shell shim vs process spawn**: On Windows, `copilot -h` may work in PowerShell while AgentV still fails with `spawn copilot ENOENT`. Shell commands can execute `copilot.ps1`/`copilot.bat`, but AgentV launches a subprocess that expects a directly spawnable executable path. If this occurs, set an explicit target executable (for example via env var): +- **Windows shell shim vs process spawn**: On Windows, `copilot -h` may work in PowerShell while AgentV still fails with `spawn copilot ENOENT`. Shell commands can execute `copilot.ps1`/`copilot.bat`, but AgentV launches a subprocess that expects a directly spawnable executable path. If this occurs, set an explicit command argv with a native executable path: ```yaml targets: - - label: copilot + - id: copilot-local provider: copilot-cli - executable: ${{ COPILOT_EXE }} - grader_target: azure-base + runtime: host + config: + command: ["${{ COPILOT_EXE }}"] ``` Use a native binary path for `COPILOT_EXE` (for example `copilot.exe` from `@github/copilot-win32-x64`). ### Claude Code -- **Run evals externally**: Run agentv evals from **outside** Claude Code. Running `agentv eval` with the `claude` target from within a Claude Code session can cause unintended behavior — the spawned Claude agent may interfere with the parent session. +- **Run evals externally**: Run agentv evals from **outside** Claude Code. Running `agentv eval` with a `claude-cli` target from within a Claude Code session can cause unintended behavior — the spawned Claude agent may interfere with the parent session. - **`ANTHROPIC_API_KEY` overrides subscription auth**: Claude Code loads `.env` from the working directory on startup. If your `.env` contains `ANTHROPIC_API_KEY`, the spawned Claude Code process will use that API key instead of your Claude subscription (Max/Pro). If the API key has insufficient credits, evals will fail with "Credit balance is too low". To use subscription auth, remove `ANTHROPIC_API_KEY` from your `.env` file. diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 6683fcb52..27ac54bfb 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -146,6 +146,8 @@ already-exported secrets into `.env`. | `codex-cli` | Agent | Codex CLI subprocess | | `codex-app-server` | Agent | Codex app-server subprocess | | `codex-sdk` | Agent | Codex SDK in an isolated child runner | +| `copilot-cli` | Agent | Copilot CLI subprocess | +| `copilot-log` | Agent | Passive Copilot CLI session log reader | | `copilot-sdk` | Agent | Copilot SDK in an isolated child runner | | `pi-sdk` | Agent | Pi SDK in an isolated child runner | | `pi-cli` | Agent | Pi CLI subprocess | diff --git a/examples/features/agent-skills-evals/README.md b/examples/features/agent-skills-evals/README.md index 36010e40c..fc8575941 100644 --- a/examples/features/agent-skills-evals/README.md +++ b/examples/features/agent-skills-evals/README.md @@ -102,15 +102,15 @@ The `csv-analyzer` skill is included in this example under `.claude/skills/csv-a ```bash bun apps/cli/src/cli.ts eval multi-provider-skill-trigger.EVAL.yaml \ - --target copilot --targets ../.agentv/targets.yaml + --target copilot-cli --targets ../.agentv/targets.yaml ``` The `skill-trigger` grader automatically handles each provider's tool-call format: | Provider | Detection method | |----------|-----------------| -| claude, claude-cli | `Skill` tool with `skill` input field | -| copilot | `Using skill: ` tool prefix | +| claude-cli | `Skill` tool with `skill` input field | +| copilot-cli | `Using skill: ` tool prefix | | codex | `mcp:/` tool prefix | ## Copilot note diff --git a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml index 6ee9c1660..0aa85d7a4 100644 --- a/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml +++ b/examples/features/agent-skills-evals/multi-provider-skill-trigger.EVAL.yaml @@ -2,7 +2,7 @@ # # Uses an isolated workspace template (workspace/) so no global skill installation # is needed. The workspace contains: -# .claude/skills/acme-deploy/ — for claude/copilot providers +# .claude/skills/acme-deploy/ — for claude-cli/copilot-cli providers # .agents/skills/acme-deploy/ — for codex/pi providers # .codex/skills/acme-deploy/ — for codex provider (fallback) # @@ -12,8 +12,8 @@ # # The same EVAL.yaml works with any provider — just change --target: # -# agentv eval this-file.EVAL.yaml --target claude --targets ../.agentv/targets.yaml -# agentv eval this-file.EVAL.yaml --target copilot --targets ../.agentv/targets.yaml +# agentv eval this-file.EVAL.yaml --target claude-cli --targets ../.agentv/targets.yaml +# agentv eval this-file.EVAL.yaml --target copilot-cli --targets ../.agentv/targets.yaml # agentv eval this-file.EVAL.yaml --target codex-cli --targets ../.agentv/targets.yaml # # The grader automatically resolves the correct tool names for each diff --git a/packages/core/src/evaluation/providers/claude-cli.ts b/packages/core/src/evaluation/providers/claude-cli.ts index b4c18e69e..6a03cd8ed 100644 --- a/packages/core/src/evaluation/providers/claude-cli.ts +++ b/packages/core/src/evaluation/providers/claude-cli.ts @@ -11,6 +11,7 @@ import { recordClaudeLogEntry } from './claude-log-tracker.js'; import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; +import { buildTargetExecutionEnvelope } from './target-execution.js'; import type { ClaudeResolvedConfig } from './targets.js'; import type { Message, @@ -54,7 +55,8 @@ export class ClaudeCliProvider implements Provider { const inputFiles = normalizeInputFiles(request.inputFiles); const prompt = buildPromptDocument(request, inputFiles); - const args = this.buildArgs(); + const command = this.resolveCommand(); + const args = this.buildArgs(command); const cwd = this.resolveCwd(request.cwd); const env = sanitizeEnvForClaude(request.braintrustSpanIds); @@ -67,6 +69,7 @@ export class ClaudeCliProvider implements Provider { try { const result = await this.runClaude({ + command, args, cwd, prompt, @@ -140,15 +143,37 @@ export class ClaudeCliProvider implements Provider { }); if (result.timedOut) { - throw new Error( - `Claude CLI timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`, - ); + const message = `Claude CLI timed out${formatTimeoutSuffix(this.config.timeoutMs ?? undefined)}`; + return this.errorResponse({ + message, + errorKind: 'timeout', + startMs, + startTime, + command, + args, + cwd, + result, + output, + loggerFile: logger?.filePath, + }); } if (result.exitCode !== 0) { const detail = result.stderr.trim() || result.stdout.trim(); const prefix = `Claude CLI exited with code ${result.exitCode}`; - throw new Error(detail ? `${prefix}: ${detail}` : prefix); + const message = detail ? `${prefix}: ${detail}` : prefix; + return this.errorResponse({ + message, + errorKind: 'nonzero_exit', + startMs, + startTime, + command, + args, + cwd, + result, + output, + loggerFile: logger?.filePath, + }); } const endTime = new Date().toISOString(); @@ -158,6 +183,7 @@ export class ClaudeCliProvider implements Provider { raw: { model: this.config.model, logFile: logger?.filePath, + command, args, exitCode: result.exitCode, }, @@ -167,15 +193,51 @@ export class ClaudeCliProvider implements Provider { durationMs: totalDurationMs, startTime, endTime, + targetExecution: buildTargetExecutionEnvelope({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + status: 'success', + commandArgv: [command, ...args], + cwd, + timeoutMs: this.config.timeoutMs, + startedAt: startMs, + endedAt: Date.now(), + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + output, + finalOutput: extractFinalOutput(output), + details: { logFile: logger?.filePath }, + }), }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return this.errorResponse({ + message, + errorKind: classifyClaudeError(message), + startMs, + startTime, + command, + args, + cwd, + result: { stdout: '', stderr: message, exitCode: null, timedOut: false }, + output, + loggerFile: logger?.filePath, + }); } finally { await logger?.close(); } } - private buildArgs(): string[] { + private resolveCommand(): string { + return this.config.command?.[0] ?? this.config.executable; + } + + private buildArgs(command = this.resolveCommand()): string[] { // --verbose is required when combining -p with --output-format stream-json const args = [ + ...(this.config.command?.[0] === command ? (this.config.command?.slice(1) ?? []) : []), '-p', '--output-format', 'stream-json', @@ -208,6 +270,62 @@ export class ClaudeCliProvider implements Provider { return undefined; } + private errorResponse(params: { + readonly message: string; + readonly errorKind: NonNullable['errorKind']; + readonly startMs: number; + readonly startTime: string; + readonly command: string; + readonly args: readonly string[]; + readonly cwd: string | undefined; + readonly result: { + readonly stdout: string; + readonly stderr: string; + readonly exitCode: number | null; + readonly timedOut: boolean; + }; + readonly output: readonly Message[]; + readonly loggerFile?: string; + }): ProviderResponse { + const content = `Error: ${params.message}`; + const output = params.output.length > 0 ? params.output : [{ role: 'assistant', content }]; + const endTime = new Date().toISOString(); + const endMs = Date.now(); + return { + raw: { + model: this.config.model, + logFile: params.loggerFile, + command: params.command, + args: params.args, + exitCode: params.result.exitCode, + error: params.message, + }, + output, + durationMs: endMs - params.startMs, + startTime: params.startTime, + endTime, + targetExecution: buildTargetExecutionEnvelope({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + status: 'error', + commandArgv: [params.command, ...params.args], + cwd: params.cwd, + timeoutMs: this.config.timeoutMs, + startedAt: params.startMs, + endedAt: endMs, + exitCode: params.result.exitCode, + errorKind: params.errorKind, + message: params.message, + stdout: params.result.stdout, + stderr: params.result.stderr, + output, + finalOutput: extractFinalOutput(output) || content, + details: { logFile: params.loggerFile }, + }), + }; + } + private resolveLogDirectory(request: ProviderRequest): string | undefined { if (this.config.streamLog === false) { return undefined; @@ -262,6 +380,7 @@ export class ClaudeCliProvider implements Provider { } private async runClaude(options: { + readonly command: string; readonly args: string[]; readonly cwd: string | undefined; readonly prompt: string; @@ -278,7 +397,7 @@ export class ClaudeCliProvider implements Provider { spawnOptions.cwd = options.cwd; } - const child = spawn(this.config.executable, options.args, spawnOptions); + const child = spawn(options.command, options.args, spawnOptions); trackChild(child); let stdout = ''; @@ -374,6 +493,28 @@ export class ClaudeCliProvider implements Provider { } } +function classifyClaudeError( + message: string, +): NonNullable['errorKind'] { + if (/not found|enoent/i.test(message)) { + return 'spawn_failure'; + } + if (/timed out|timeout/i.test(message)) { + return 'timeout'; + } + return 'target_task_failure'; +} + +function extractFinalOutput(output: readonly Message[]): string { + for (let i = output.length - 1; i >= 0; i--) { + const content = output[i]?.content; + if (typeof content === 'string') { + return content; + } + } + return ''; +} + class ClaudeCliStreamLogger { readonly filePath: string; private readonly stream: WriteStream; diff --git a/packages/core/src/evaluation/providers/claude.ts b/packages/core/src/evaluation/providers/claude.ts index 64e15ed88..033f7fb06 100644 --- a/packages/core/src/evaluation/providers/claude.ts +++ b/packages/core/src/evaluation/providers/claude.ts @@ -36,23 +36,23 @@ async function loadClaudeSdk(): Promise INLINE_LOG_LIMIT_BYTES) { - stored = stored.slice(0, Math.max(0, stored.length - 1024)); - } - const storedBytes = Buffer.byteLength(stored, 'utf8'); - return { - text: stored, - truncated: true, - bytes, - storedBytes, - }; -} - function commandEnvelopeBase(params: { targetName: string; providerId: string; @@ -340,28 +314,23 @@ function commandEnvelopeBase(params: { process.platform === 'win32' ? ['powershell.exe', '-NoProfile', '-Command', params.command] : ['/bin/sh', '-lc', params.command]; - return { - schemaVersion: 'agentv.target_execution.v1', - targetId: params.targetName, + return buildTargetExecutionEnvelope({ + targetName: params.targetName, providerId: params.providerId, providerKind: params.providerKind, + status: 'success', runtimeMode: params.runtimeMode ?? 'host', - command: { - argv, - commandLine: params.command, - cwd: params.cwd, - }, + commandArgv: argv, + commandLine: params.command, + cwd: params.cwd, timeoutMs: params.timeoutMs, - startedAt: new Date(params.startedAt).toISOString(), - endedAt: new Date(params.endedAt).toISOString(), - durationMs: params.endedAt - params.startedAt, + startedAt: params.startedAt, + endedAt: params.endedAt, exitCode: params.result?.exitCode, signal: params.result?.signal ?? null, - logs: { - stdout: captureLog(params.result?.stdout ?? ''), - stderr: captureLog(params.result?.stderr ?? ''), - }, - }; + stdout: params.result?.stdout ?? '', + stderr: params.result?.stderr ?? '', + }); } function classifyCommandFailure( diff --git a/packages/core/src/evaluation/providers/copilot-cli.ts b/packages/core/src/evaluation/providers/copilot-cli.ts index 244e202c1..1d0f7aa55 100644 --- a/packages/core/src/evaluation/providers/copilot-cli.ts +++ b/packages/core/src/evaluation/providers/copilot-cli.ts @@ -23,6 +23,7 @@ import { import { resolveDefaultProviderLogDir } from './log-directory.js'; import { normalizeToolCall } from './normalize-tool-call.js'; import { buildPromptDocument, normalizeInputFiles } from './preread.js'; +import { buildTargetExecutionEnvelope } from './target-execution.js'; import type { CopilotCliResolvedConfig, CopilotCustomProviderConfig } from './targets.js'; import type { Message, @@ -111,19 +112,19 @@ export class CopilotCliProvider implements Provider { const logger = await this.createStreamLogger(request, 'acp').catch(() => undefined); // Build command args - const executable = this.resolveExecutable(); - const args = this.buildCliArgs(); + const command = this.resolveCommand(); + const executable = this.resolveExecutable(command); + const args = this.buildCliArgs(command); + const cwd = this.resolveCwd(request.cwd) ?? process.cwd(); // Spawn the CLI process const agentProcess = this.spawnAcpProcess(executable, args, { env: buildCopilotCliProviderEnv(process.env, this.config.customProvider), - cwd: this.resolveCwd(request.cwd) ?? process.cwd(), + cwd, stdio: ['pipe', 'pipe', 'inherit'], }); trackChild(agentProcess); - await waitForProcessSpawn(agentProcess, executable, this.targetName); - // Track events const toolCallsInProgress = new Map(); const completedToolCalls: ToolCall[] = []; @@ -131,121 +132,123 @@ export class CopilotCliProvider implements Provider { let tokenUsage: ProviderTokenUsage | undefined; let costUsd: number | undefined; - // Set up ACP connection - if (!agentProcess.stdin || !agentProcess.stdout) { - throw new Error('Copilot CLI process missing stdin/stdout (stdio: pipe required)'); - } - const input = Writable.toWeb(agentProcess.stdin); - const output = Readable.toWeb(agentProcess.stdout) as ReadableStream; - const stream = acp.ndJsonStream(input, output); - const customProvider = this.config.customProvider; - - const client: acp.Client = { - async requestPermission(): Promise { - // Auto-approve all permissions for autonomous execution - return { - outcome: { outcome: 'selected', optionId: 'allow' }, - }; - }, - async sessionUpdate(params: acp.SessionNotification): Promise { - const update = params.update; - const sessionUpdate = update.sessionUpdate; - - logger?.handleEvent(sessionUpdate, sanitizeSensitiveValue(update, customProvider)); - - if (sessionUpdate === 'tool_call') { - const callId = update.toolCallId ?? randomUUID(); - // Track new or in-progress tool calls - if (!update.status || update.status === 'pending' || update.status === 'in_progress') { - toolCallsInProgress.set(callId, { - tool: update.title ?? update.kind ?? 'unknown', - input: update.rawInput, - id: callId, - startTime: new Date().toISOString(), - startMs: Date.now(), - }); - } - // Tool call arrived already completed - if (update.status === 'completed' || update.status === 'failed') { - const toolName = update.title ?? update.kind ?? 'unknown'; - completedToolCalls.push( - normalizeToolCall('copilot-cli', { - tool: toolName, + try { + await waitForProcessSpawn(agentProcess, executable, this.targetName); + + // Set up ACP connection + if (!agentProcess.stdin || !agentProcess.stdout) { + throw new Error('Copilot CLI process missing stdin/stdout (stdio: pipe required)'); + } + const input = Writable.toWeb(agentProcess.stdin); + const output = Readable.toWeb(agentProcess.stdout) as ReadableStream; + const stream = acp.ndJsonStream(input, output); + const customProvider = this.config.customProvider; + + const client: acp.Client = { + async requestPermission(): Promise { + // Auto-approve all permissions for autonomous execution + return { + outcome: { outcome: 'selected', optionId: 'allow' }, + }; + }, + async sessionUpdate(params: acp.SessionNotification): Promise { + const update = params.update; + const sessionUpdate = update.sessionUpdate; + + logger?.handleEvent(sessionUpdate, sanitizeSensitiveValue(update, customProvider)); + + if (sessionUpdate === 'tool_call') { + const callId = update.toolCallId ?? randomUUID(); + // Track new or in-progress tool calls + if (!update.status || update.status === 'pending' || update.status === 'in_progress') { + toolCallsInProgress.set(callId, { + tool: update.title ?? update.kind ?? 'unknown', input: update.rawInput, - output: update.rawOutput, id: callId, startTime: new Date().toISOString(), - endTime: new Date().toISOString(), - durationMs: 0, - }), - ); - request.streamCallbacks?.onToolCallEnd?.( - toolName, - update.rawInput, - update.rawOutput, - 0, - callId, - ); - } - } - - if (sessionUpdate === 'tool_call_update') { - const callId = update.toolCallId; - if (callId && (update.status === 'completed' || update.status === 'failed')) { - const inProgress = toolCallsInProgress.get(callId); - if (inProgress) { - toolCallsInProgress.delete(callId); - const duration = Date.now() - inProgress.startMs; + startMs: Date.now(), + }); + } + // Tool call arrived already completed + if (update.status === 'completed' || update.status === 'failed') { + const toolName = update.title ?? update.kind ?? 'unknown'; completedToolCalls.push( normalizeToolCall('copilot-cli', { - tool: inProgress.tool, - input: inProgress.input, + tool: toolName, + input: update.rawInput, output: update.rawOutput, - id: inProgress.id, - startTime: inProgress.startTime, + id: callId, + startTime: new Date().toISOString(), endTime: new Date().toISOString(), - durationMs: duration, + durationMs: 0, }), ); request.streamCallbacks?.onToolCallEnd?.( - inProgress.tool, - inProgress.input, + toolName, + update.rawInput, update.rawOutput, - duration, - inProgress.id, + 0, + callId, ); } } - } - if (sessionUpdate === 'agent_message_chunk') { - const content = update.content; - if (content?.type === 'text' && typeof content.text === 'string') { - finalContent += content.text; + if (sessionUpdate === 'tool_call_update') { + const callId = update.toolCallId; + if (callId && (update.status === 'completed' || update.status === 'failed')) { + const inProgress = toolCallsInProgress.get(callId); + if (inProgress) { + toolCallsInProgress.delete(callId); + const duration = Date.now() - inProgress.startMs; + completedToolCalls.push( + normalizeToolCall('copilot-cli', { + tool: inProgress.tool, + input: inProgress.input, + output: update.rawOutput, + id: inProgress.id, + startTime: inProgress.startTime, + endTime: new Date().toISOString(), + durationMs: duration, + }), + ); + request.streamCallbacks?.onToolCallEnd?.( + inProgress.tool, + inProgress.input, + update.rawOutput, + duration, + inProgress.id, + ); + } + } } - } - if (sessionUpdate === 'usage_update') { - // ACP UsageUpdate provides { size, used, cost? } where `used` is - // cumulative context window tokens. This does NOT separate input vs - // output tokens, so we report `used` as input with output 0. - // Copilot CLI does not currently emit this event via ACP (events are - // marked ephemeral internally — see github/copilot-cli#1152), but - // this handler is ready for when it does. See #683. - tokenUsage = { input: update.used, output: 0 }; - // Cost may arrive across multiple events — accumulate - if (update.cost && update.cost.currency === 'USD') { - costUsd = (costUsd ?? 0) + update.cost.amount; + if (sessionUpdate === 'agent_message_chunk') { + const content = update.content; + if (content?.type === 'text' && typeof content.text === 'string') { + finalContent += content.text; + } } - // Stream callback for LLM usage - request.streamCallbacks?.onLlmCallEnd?.('copilot', tokenUsage); - } - }, - }; - const connection = new acp.ClientSideConnection((_agent) => client, stream); + if (sessionUpdate === 'usage_update') { + // ACP UsageUpdate provides { size, used, cost? } where `used` is + // cumulative context window tokens. This does NOT separate input vs + // output tokens, so we report `used` as input with output 0. + // Copilot CLI does not currently emit this event via ACP (events are + // marked ephemeral internally — see github/copilot-cli#1152), but + // this handler is ready for when it does. See #683. + tokenUsage = { input: update.used, output: 0 }; + // Cost may arrive across multiple events — accumulate + if (update.cost && update.cost.currency === 'USD') { + costUsd = (costUsd ?? 0) + update.cost.amount; + } + // Stream callback for LLM usage + request.streamCallbacks?.onLlmCallEnd?.('copilot', tokenUsage); + } + }, + }; + + const connection = new acp.ClientSideConnection((_agent) => client, stream); - try { // Initialize the ACP connection await connection.initialize({ protocolVersion: acp.PROTOCOL_VERSION, @@ -360,7 +363,9 @@ export class CopilotCliProvider implements Provider { return { raw: { model: this.config.model, + command, executable, + args, logFile: logger?.filePath, }, output: outputMessages, @@ -369,15 +374,76 @@ export class CopilotCliProvider implements Provider { durationMs, startTime, endTime, + targetExecution: buildTargetExecutionEnvelope({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + status: 'success', + commandArgv: [executable, ...args], + cwd, + timeoutMs: resolveCopilotTimeoutMs(this.config.timeoutMs), + startedAt: startMs, + endedAt: Date.now(), + exitCode: 0, + stdout: finalContent, + stderr: '', + output: outputMessages, + finalOutput: finalContent, + details: { logFile: logger?.filePath, mode: 'acp' }, + }), ...(fileChanges ? { fileChanges } : {}), }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const content = `Error: ${message}`; + const output = finalContent + ? [{ role: 'assistant' as const, content: finalContent }] + : [{ role: 'assistant' as const, content }]; + const endMs = Date.now(); + return { + raw: { + model: this.config.model, + command, + executable, + args, + logFile: logger?.filePath, + error: message, + }, + output, + durationMs: endMs - startMs, + startTime, + endTime: new Date(endMs).toISOString(), + targetExecution: buildTargetExecutionEnvelope({ + targetName: this.targetName, + providerId: this.id, + providerKind: this.kind, + status: 'error', + commandArgv: [executable, ...args], + cwd, + timeoutMs: resolveCopilotTimeoutMs(this.config.timeoutMs), + startedAt: startMs, + endedAt: endMs, + exitCode: null, + errorKind: classifyCopilotError(message, request.signal?.aborted), + message, + stdout: finalContent, + stderr: message, + output, + finalOutput: finalContent || content, + details: { logFile: logger?.filePath, mode: 'acp' }, + }), + }; } finally { await logger?.close(); killProcess(agentProcess); } } - private buildCliArgs(): string[] { + private resolveCommand(): readonly string[] { + return this.config.command ?? [this.config.executable, ...(this.config.args ?? [])]; + } + + private buildCliArgs(command: readonly string[]): string[] { // --yolo bypasses copilot's permission system so file reads and tool calls // are not rejected during eval runs (see #421). const args = ['--acp', '--stdio', '--allow-all-tools', '--yolo']; @@ -386,10 +452,7 @@ export class CopilotCliProvider implements Provider { args.push('--model', this.config.model); } - // Append user-provided extra args - if (this.config.args) { - args.push(...this.config.args); - } + args.push(...command.slice(1)); return args; } @@ -400,12 +463,13 @@ export class CopilotCliProvider implements Provider { startMs: number, ): Promise { const logger = await this.createStreamLogger(request, 'prompt').catch(() => undefined); - const executable = this.resolveExecutable(); + const command = this.resolveCommand(); + const executable = this.resolveExecutable(command); const inputFiles = normalizeInputFiles(request.inputFiles); const prompt = buildPromptDocument(request, inputFiles); const systemPrompt = this.resolveSystemPrompt(request); const promptText = systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt; - const args = this.buildPromptModeArgs(promptText); + const args = this.buildPromptModeArgs(command, promptText); const cwd = this.resolveCwd(request.cwd) ?? process.cwd(); const env = buildCopilotCliProviderEnv(process.env, this.config.customProvider); @@ -453,6 +517,7 @@ export class CopilotCliProvider implements Provider { return { raw: { model: this.config.model, + command, executable, args, stdout: result.stdout, @@ -476,17 +541,14 @@ export class CopilotCliProvider implements Provider { } } - private buildPromptModeArgs(prompt: string): string[] { + private buildPromptModeArgs(command: readonly string[], prompt: string): string[] { const args = ['-s', '--allow-all-tools', '--no-color']; if (this.config.model) { args.push('--model', this.config.model); } - if (this.config.args) { - args.push(...this.config.args); - } - + args.push(...command.slice(1)); args.push('-p', prompt); return args; @@ -528,9 +590,10 @@ export class CopilotCliProvider implements Provider { return undefined; } - private resolveExecutable(): string { - if (this.config.executable !== 'copilot') { - return this.config.executable; + private resolveExecutable(command: readonly string[]): string { + const executable = command[0] ?? this.config.executable; + if (executable !== 'copilot') { + return executable; } // Try to resolve the platform-specific native binary @@ -601,6 +664,17 @@ export class CopilotCliProvider implements Provider { } } +function classifyCopilotError( + message: string, + aborted: boolean | undefined, +): NonNullable['errorKind'] { + if (aborted) return 'cancelled'; + if (/not found|enoent|failed to start/i.test(message)) return 'spawn_failure'; + if (/timed out|timeout/i.test(message)) return 'timeout'; + if (/malformed|protocol/i.test(message)) return 'malformed_output'; + return 'target_task_failure'; +} + export function buildCopilotCliProviderEnv( baseEnv: NodeJS.ProcessEnv, customProvider: CopilotCustomProviderConfig | undefined, diff --git a/packages/core/src/evaluation/providers/index.ts b/packages/core/src/evaluation/providers/index.ts index 8e1a053fb..a916d6e59 100644 --- a/packages/core/src/evaluation/providers/index.ts +++ b/packages/core/src/evaluation/providers/index.ts @@ -234,17 +234,11 @@ export function createBuiltinProviderRegistry(): ProviderRegistry { ? unsupportedSandboxProvider(t) : new PiRpcProvider(t.name, t.config as never), ) - // claude-cli is the new default subprocess provider; claude is an alias .register('claude-cli', (t) => usesSandboxRuntime(t) ? unsupportedSandboxProvider(t) : new ClaudeCliProvider(t.name, t.config as never), ) - .register('claude', (t) => - usesSandboxRuntime(t) - ? unsupportedSandboxProvider(t) - : new ClaudeCliProvider(t.name, t.config as never), - ) // Explicit SDK providers are isolated behind an AgentV child runner. .register('claude-sdk', (t) => usesSandboxRuntime(t) diff --git a/packages/core/src/evaluation/providers/normalize-tool-call.ts b/packages/core/src/evaluation/providers/normalize-tool-call.ts index 80558f7a2..4584e64ae 100644 --- a/packages/core/src/evaluation/providers/normalize-tool-call.ts +++ b/packages/core/src/evaluation/providers/normalize-tool-call.ts @@ -44,11 +44,6 @@ type CanonicalTool = 'Skill' | 'Read' | 'Write' | 'Edit' | 'Bash'; */ const TOOL_NAME_MAP = new Map([ // --- Claude (already canonical) --- - ['claude::Skill', 'Skill'], - ['claude::Read', 'Read'], - ['claude::Write', 'Write'], - ['claude::Edit', 'Edit'], - ['claude::Bash', 'Bash'], ['claude-cli::Skill', 'Skill'], ['claude-cli::Read', 'Read'], ['claude-cli::Write', 'Write'], @@ -122,8 +117,12 @@ const TOOL_NAME_MAP = new Map([ ['vscode-insiders::runTerminalCommand', 'Bash'], // --- Codex --- - ['codex::command_execution', 'Bash'], - ['codex::file_change', 'Edit'], + ['codex-cli::command_execution', 'Bash'], + ['codex-cli::file_change', 'Edit'], + ['codex-app-server::command_execution', 'Bash'], + ['codex-app-server::file_change', 'Edit'], + ['codex-sdk::command_execution', 'Bash'], + ['codex-sdk::file_change', 'Edit'], // --- Pi --- ['pi-coding-agent::read', 'Read'], @@ -166,7 +165,9 @@ const TOOL_PREFIX_MAP = new Map([ ['copilot-log', COPILOT_PREFIXES], ['vscode', COPILOT_PREFIXES], ['vscode-insiders', COPILOT_PREFIXES], - ['codex', CODEX_PREFIXES], + ['codex-cli', CODEX_PREFIXES], + ['codex-app-server', CODEX_PREFIXES], + ['codex-sdk', CODEX_PREFIXES], ]); // --------------------------------------------------------------------------- diff --git a/packages/core/src/evaluation/providers/target-execution.ts b/packages/core/src/evaluation/providers/target-execution.ts new file mode 100644 index 000000000..2d1481dbd --- /dev/null +++ b/packages/core/src/evaluation/providers/target-execution.ts @@ -0,0 +1,87 @@ +import type { Message, TargetExecutionEnvelope, TargetExecutionLogCapture } from './types.js'; + +const INLINE_LOG_LIMIT_BYTES = 128 * 1024; + +export function captureTargetExecutionLog(text: string): TargetExecutionLogCapture { + const bytes = Buffer.byteLength(text, 'utf8'); + if (bytes <= INLINE_LOG_LIMIT_BYTES) { + return { + text, + truncated: false, + bytes, + storedBytes: bytes, + }; + } + + let stored = text; + while (Buffer.byteLength(stored, 'utf8') > INLINE_LOG_LIMIT_BYTES) { + stored = stored.slice(0, Math.max(0, stored.length - 1024)); + } + const storedBytes = Buffer.byteLength(stored, 'utf8'); + return { + text: stored, + truncated: true, + bytes, + storedBytes, + }; +} + +export function buildTargetExecutionEnvelope(params: { + readonly targetName: string; + readonly providerId: string; + readonly providerKind: string; + readonly status: 'success' | 'error'; + readonly commandArgv?: readonly string[]; + readonly commandLine?: string; + readonly cwd?: string; + readonly runtimeMode?: string; + readonly timeoutMs?: number; + readonly startedAt: number; + readonly endedAt: number; + readonly exitCode?: number | null; + readonly signal?: string | null; + readonly errorKind?: TargetExecutionEnvelope['errorKind']; + readonly message?: string; + readonly stdout?: string; + readonly stderr?: string; + readonly output?: readonly Message[]; + readonly finalOutput?: string; + readonly details?: Record; +}): TargetExecutionEnvelope { + return { + schemaVersion: 'agentv.target_execution.v1', + status: params.status, + targetId: params.targetName, + providerId: params.providerId, + providerKind: params.providerKind, + runtimeMode: params.runtimeMode ?? 'host', + command: + params.commandArgv || params.commandLine || params.cwd + ? { + argv: params.commandArgv, + commandLine: params.commandLine, + cwd: params.cwd, + } + : undefined, + timeoutMs: params.timeoutMs, + startedAt: new Date(params.startedAt).toISOString(), + endedAt: new Date(params.endedAt).toISOString(), + durationMs: params.endedAt - params.startedAt, + exitCode: params.exitCode, + signal: params.signal ?? null, + errorKind: params.errorKind, + message: params.message, + logs: { + stdout: captureTargetExecutionLog(params.stdout ?? ''), + stderr: captureTargetExecutionLog(params.stderr ?? ''), + }, + transcript: + params.output || params.finalOutput + ? { + messages: params.output, + finalOutput: params.finalOutput, + } + : undefined, + details: params.details, + }; +} diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 64df4a9ab..171a2b756 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -452,6 +452,7 @@ export interface CodingAgentRuntimeConfig { } export interface CopilotCliResolvedConfig { + readonly command: readonly string[]; readonly executable: string; readonly model?: string; readonly args?: readonly string[]; @@ -565,6 +566,7 @@ export interface PiRuntimeResolvedConfig { } export interface ClaudeResolvedConfig { + readonly command: readonly string[]; readonly executable: string; readonly model?: string; readonly systemPrompt?: string; @@ -887,7 +889,6 @@ export type ResolvedTarget = }) | (ResolvedTargetBase & { readonly kind: 'pi-cli'; readonly config: PiCliResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'pi-rpc'; readonly config: PiRpcResolvedConfig }) - | (ResolvedTargetBase & { readonly kind: 'claude'; readonly config: ClaudeResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'claude-cli'; readonly config: ClaudeResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'claude-sdk'; readonly config: ClaudeResolvedConfig }) | (ResolvedTargetBase & { readonly kind: 'mock'; readonly config: MockResolvedConfig }) @@ -1101,6 +1102,11 @@ export function resolveTargetDefinition( `${parsed.name} provider`, true, ).toLowerCase(); + if (provider === 'claude' || provider === 'copilot') { + throw new Error( + `Target "${parsed.name}" uses ambiguous provider '${provider}'. Choose an explicit provider such as '${provider}-cli' or '${provider}-sdk'.`, + ); + } const providerBatching = resolveOptionalBoolean(parsed.batch_requests); const subagentModeAllowed = resolveOptionalBoolean(parsed.subagent_mode_allowed); @@ -1197,7 +1203,6 @@ export function resolveTargetDefinition( ...base, config: resolvePiRpcConfig(parsed, env, evalFilePath), }; - case 'claude': case 'claude-cli': return { kind: 'claude-cli', @@ -1674,35 +1679,46 @@ function normalizeCodingAgentRuntimeMode( throw new Error(`Target "${targetName}" runtime.mode must be one of: host, profile, sandbox.`); } -function resolveRuntimeEnv( - value: unknown, - env: EnvLookup, - targetName: string, -): Readonly> | undefined { +function resolveRuntimeEnvAllowlist(value: unknown): readonly string[] | undefined { if (value === undefined || value === null) { return undefined; } - if (!isRecord(value)) { - throw new Error(`Target "${targetName}" runtime.env must be an object of string values.`); + if (!Array.isArray(value) || !value.every((entry) => typeof entry === 'string')) { + throw new Error('runtime.env_allowlist must be an array of strings.'); } - const resolved: Record = {}; - for (const [key, raw] of Object.entries(value)) { - if (typeof raw !== 'string') { - throw new Error(`Target "${targetName}" runtime.env.${key} must be a string.`); + return value.map((entry) => entry.trim()).filter((entry) => entry.length > 0); +} + +function assertNoProcessCommandAliases( + target: z.infer, + provider: 'claude-cli' | 'copilot-cli', +): void { + const raw = target as Record; + for (const field of ['executable', 'binary', 'args', 'arguments']) { + if (Object.prototype.hasOwnProperty.call(raw, field)) { + throw new Error( + `Target "${target.name}" (provider: ${provider}) uses removed field '${field}'. Use config.command as a non-empty argv array instead.`, + ); } - resolved[key] = resolveString(raw, env, `${targetName} runtime.env.${key}`, true); } - return resolved; } -function resolveRuntimeEnvAllowlist(value: unknown): readonly string[] | undefined { +function resolveCommandArgv( + value: unknown, + env: EnvLookup, + label: string, + defaultCommand: readonly string[], +): readonly string[] { if (value === undefined || value === null) { - return undefined; + return defaultCommand; } - if (!Array.isArray(value) || !value.every((entry) => typeof entry === 'string')) { - throw new Error('runtime.env_allowlist must be an array of strings.'); + if (!Array.isArray(value)) { + throw new Error(`${label} must be a non-empty argv array of strings.`); } - return value.map((entry) => entry.trim()).filter((entry) => entry.length > 0); + if (value.length === 0) { + throw new Error(`${label} must be a non-empty argv array of strings.`); + } + return resolveOptionalStringArray(value, env, label) ?? defaultCommand; } function normalizeCodexModelReasoningEffort( @@ -1957,9 +1973,11 @@ function resolveCopilotCliConfig( env: EnvLookup, _evalFilePath?: string, ): CopilotCliResolvedConfig { - const executableSource = target.executable ?? target.command ?? target.binary; + assertNoProcessCommandAliases(target, 'copilot-cli'); + const command = resolveCommandArgv(target.command, env, `${target.name} copilot-cli command`, [ + 'copilot', + ]); const modelSource = target.model; - const argsSource = target.args ?? target.arguments; const cwdSource = target.cwd; const timeoutSource = target.timeout_seconds; const logDirSource = target.log_dir ?? target.log_directory; @@ -1967,19 +1985,11 @@ function resolveCopilotCliConfig( const streamLogResult = resolveStreamLog(target); - const executable = - resolveOptionalString(executableSource, env, `${target.name} copilot-cli executable`, { - allowLiteral: true, - optionalEnv: true, - }) ?? 'copilot'; - const model = resolveOptionalString(modelSource, env, `${target.name} copilot-cli model`, { allowLiteral: true, optionalEnv: true, }); - const args = resolveOptionalStringArray(argsSource, env, `${target.name} copilot-cli args`); - const cwd = resolveOptionalString(cwdSource, env, `${target.name} copilot-cli cwd`, { allowLiteral: true, optionalEnv: true, @@ -2004,9 +2014,10 @@ function resolveCopilotCliConfig( const customProvider = resolveCopilotFlatProviderConfig(target, env); return { - executable, + command, + executable: command[0] ?? 'copilot', + args: command.slice(1), model, - args, cwd, timeoutMs, logDir, @@ -2381,7 +2392,10 @@ function resolveClaudeConfig( env: EnvLookup, _evalFilePath?: string, ): ClaudeResolvedConfig { - const executableSource = target.executable ?? target.command ?? target.binary; + assertNoProcessCommandAliases(target, 'claude-cli'); + const command = resolveCommandArgv(target.command, env, `${target.name} claude-cli command`, [ + 'claude', + ]); const modelSource = target.model; const cwdSource = target.cwd; const timeoutSource = target.timeout_seconds; @@ -2390,12 +2404,6 @@ function resolveClaudeConfig( const streamLogResult = resolveStreamLog(target); - const executable = - resolveOptionalString(executableSource, env, `${target.name} claude-cli executable`, { - allowLiteral: true, - optionalEnv: true, - }) ?? 'claude'; - const model = resolveOptionalString(modelSource, env, `${target.name} claude model`, { allowLiteral: true, optionalEnv: true, @@ -2429,7 +2437,8 @@ function resolveClaudeConfig( : undefined; return { - executable, + command, + executable: command[0] ?? 'claude', model, systemPrompt, cwd, diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 01c6ba20c..541dbc2d2 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -28,7 +28,6 @@ export type ProviderKind = | 'pi-coding-agent' | 'pi-cli' | 'pi-rpc' - | 'claude' | 'claude-cli' | 'claude-sdk' | 'cli' @@ -57,7 +56,6 @@ export const AGENT_PROVIDER_KINDS: readonly ProviderKind[] = [ 'pi-coding-agent', 'pi-cli', 'pi-rpc', - 'claude', 'claude-cli', 'claude-sdk', 'vscode', @@ -102,7 +100,6 @@ export const KNOWN_PROVIDERS: readonly ProviderKind[] = [ 'pi-coding-agent', 'pi-cli', 'pi-rpc', - 'claude', 'claude-cli', 'claude-sdk', 'cli', @@ -458,7 +455,6 @@ export interface TargetDefinition { readonly api_key?: string | unknown | undefined; readonly deployment?: string | unknown | undefined; readonly model?: string | unknown | undefined; - readonly runtime?: unknown | undefined; readonly version?: string | unknown | undefined; readonly api_version?: string | unknown | undefined; readonly api_format?: string | unknown | undefined; diff --git a/packages/core/src/evaluation/transcript-summary.ts b/packages/core/src/evaluation/transcript-summary.ts index 1f53dd8cd..9fea5d35f 100644 --- a/packages/core/src/evaluation/transcript-summary.ts +++ b/packages/core/src/evaluation/transcript-summary.ts @@ -52,7 +52,7 @@ const PROVIDER_ALIASES: Readonly> = { 'pi-cli': 'pi-cli', 'pi-rpc': 'pi-rpc', 'pi-coding-agent': 'pi-coding-agent', - claude: 'claude', + claude: 'claude-sdk', 'claude-cli': 'claude-cli', 'claude-sdk': 'claude-sdk', vscode: 'vscode', diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index ae86a6b31..b4e254ed5 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -257,7 +257,6 @@ function getKnownSettings(provider: string): Set | null { return COPILOT_SDK_SETTINGS; case 'copilot-cli': return COPILOT_CLI_SETTINGS; - case 'claude': case 'claude-cli': case 'claude-sdk': return CLAUDE_SETTINGS; @@ -661,6 +660,13 @@ export async function validateTargetsFile(filePath: string): Promise { expect(provider.id).toBe('claude-cli:test-target'); }); - it('creates a ClaudeCliProvider for claude kind (alias for claude-cli)', () => { - const provider = registry.create({ - name: 'test-target', - kind: 'claude', - config: mockClaudeConfig, - }); - expect(provider).toBeInstanceOf(ClaudeCliProvider); - expect(provider.kind).toBe('claude-cli'); + it('does not register a bare claude provider alias', () => { + expect(() => + registry.create({ + name: 'test-target', + kind: 'claude' as never, + config: mockClaudeConfig, + }), + ).toThrow(/Unknown provider kind: "claude"/); }); it('creates an isolated child provider for claude-sdk kind', () => { @@ -59,7 +60,7 @@ describe('Claude provider alias resolution', () => { expect(cliProvider).toBeInstanceOf(ClaudeCliProvider); expect(sdkProvider).toBeInstanceOf(ClaudeProvider); expect(cliProvider.kind).toBe('claude-cli'); - expect(sdkProvider.kind).toBe('claude'); + expect(sdkProvider.kind).toBe('claude-sdk'); }); }); @@ -91,3 +92,32 @@ describe('ClaudeCliProvider buildArgs', () => { expect(args).not.toContain('--dangerously-skip-permissions'); }); }); + +describe('ClaudeCliProvider target execution envelopes', () => { + const originalLogEnv = process.env.AGENTV_CLAUDE_STREAM_LOGS; + + afterEach(() => { + process.env.AGENTV_CLAUDE_STREAM_LOGS = originalLogEnv; + }); + + it('maps subprocess nonzero exits to target execution errors', async () => { + process.env.AGENTV_CLAUDE_STREAM_LOGS = 'false'; + const provider = new ClaudeCliProvider('target', { + ...mockClaudeConfig, + command: ['node', '--eval', 'process.stderr.write("boom"); process.exit(7)', '--'], + executable: 'node', + }); + + const response = await provider.invoke({ question: 'hello' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('nonzero_exit'); + expect(response.targetExecution?.exitCode).toBe(7); + expect(response.targetExecution?.command?.argv?.slice(0, 3)).toEqual([ + 'node', + '--eval', + 'process.stderr.write("boom"); process.exit(7)', + ]); + expect(response.targetExecution?.logs?.stderr?.text).toContain('boom'); + }); +}); diff --git a/packages/core/test/evaluation/providers/copilot-cli.test.ts b/packages/core/test/evaluation/providers/copilot-cli.test.ts index 0ce05426e..0aa5ca625 100644 --- a/packages/core/test/evaluation/providers/copilot-cli.test.ts +++ b/packages/core/test/evaluation/providers/copilot-cli.test.ts @@ -35,6 +35,21 @@ function createMockChildProcess(): ChildProcess { return child as unknown as ChildProcess; } +function createErroredChildProcess(error: NodeJS.ErrnoException): ChildProcess { + const child = new EventEmitter() as EventEmitter & { + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + kill: ReturnType; + }; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = mock(() => true); + queueMicrotask(() => child.emit('error', error)); + return child as unknown as ChildProcess; +} + beforeAll(async () => { spawnMock = mock(() => createMockChildProcess()); mock.module('@agentclientprotocol/sdk', () => ({ @@ -194,6 +209,8 @@ describe('CopilotCliProvider custom provider ACP mode', () => { }); expect(extractLastAssistantContent(response.output)).toBe('agentv-copilot-gateway-ok'); + expect(response.targetExecution?.status).toBe('success'); + expect(response.targetExecution?.providerKind).toBe('copilot-cli'); expect(runner).not.toHaveBeenCalled(); expect(spawnMock).toHaveBeenCalledTimes(1); @@ -226,6 +243,28 @@ describe('CopilotCliProvider custom provider ACP mode', () => { expect(options.env.COPILOT_PROVIDER_WIRE_API).toBe('responses'); }); + it('maps spawn failures to target execution errors', async () => { + const error = new Error('spawn copilot ENOENT') as NodeJS.ErrnoException; + error.code = 'ENOENT'; + const failingSpawn = mock(() => createErroredChildProcess(error)); + const provider = new CopilotCliProvider( + 'copilot-cli-missing', + { + command: ['copilot-missing'], + executable: 'copilot-missing', + }, + undefined, + failingSpawn as unknown as typeof spawn, + ); + + const response = await provider.invoke({ question: 'Return done' }); + + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('spawn_failure'); + expect(response.targetExecution?.command?.argv?.[0]).toBe('copilot-missing'); + expect(extractLastAssistantContent(response.output)).toContain('Error:'); + }); + it('uses configured cwd for ACP spawn when request cwd is omitted', async () => { const provider = new CopilotCliProvider( 'copilot-cli-custom', diff --git a/packages/core/test/evaluation/providers/normalize-tool-call.test.ts b/packages/core/test/evaluation/providers/normalize-tool-call.test.ts index 6fa5a6e38..65215e49a 100644 --- a/packages/core/test/evaluation/providers/normalize-tool-call.test.ts +++ b/packages/core/test/evaluation/providers/normalize-tool-call.test.ts @@ -12,7 +12,7 @@ describe('normalizeToolCall', () => { // Claude providers (already canonical — should be identity) // ------------------------------------------------------------------------- describe('claude providers (identity)', () => { - for (const provider of ['claude', 'claude-cli', 'claude-sdk'] as ProviderKind[]) { + for (const provider of ['claude-cli', 'claude-sdk'] as ProviderKind[]) { it(`${provider}: Skill → Skill`, () => { const result = normalizeToolCall(provider, tc('Skill', { skill: 'my-skill' })); expect(result.tool).toBe('Skill'); @@ -157,7 +157,7 @@ describe('normalizeToolCall', () => { // ------------------------------------------------------------------------- describe('input field normalization', () => { it('Read: copies path → file_path when file_path missing', () => { - const result = normalizeToolCall('claude', tc('Read', { path: '/foo.ts' })); + const result = normalizeToolCall('claude-sdk', tc('Read', { path: '/foo.ts' })); expect((result.input as Record).file_path).toBe('/foo.ts'); expect((result.input as Record).path).toBe('/foo.ts'); }); diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 7d78b2941..b5ebf9aaa 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -349,7 +349,7 @@ describe('resolveTargetDefinition', () => { resolveTargetDefinition( { name: 'claude-log-output-format', - provider: 'claude', + provider: 'claude-cli', log_output_format: 'summary', } as never, {}, @@ -376,7 +376,7 @@ describe('resolveTargetDefinition', () => { const summary = resolveTargetDefinition( { name: 'claude-summary-log', - provider: 'claude', + provider: 'claude-cli', stream_log: 'summary', }, {}, @@ -994,24 +994,29 @@ describe('resolveTargetDefinition', () => { ).toThrow(/ambiguous provider 'codex'/); }); - it('does not canonicalize removed provider aliases to built-ins', () => { - const target = resolveTargetDefinition( - { - name: 'copilot-alias-removed', - provider: 'copilot', - }, - {}, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected discovered cli target'); - } + it('rejects ambiguous Claude and Copilot provider aliases', () => { + expect(() => + resolveTargetDefinition( + { + name: 'copilot-alias-removed', + provider: 'copilot', + }, + {}, + ), + ).toThrow(/ambiguous provider 'copilot'.*copilot-cli.*copilot-sdk/i); - expect(target.config.command).toBe('bun run .agentv/providers/copilot.ts {PROMPT}'); + expect(() => + resolveTargetDefinition( + { + name: 'claude-alias-removed', + provider: 'claude', + }, + {}, + ), + ).toThrow(/ambiguous provider 'claude'.*claude-cli.*claude-sdk/i); }); - it('claude-cli defaults executable to claude', () => { + it('claude-cli defaults command to claude', () => { const target = resolveTargetDefinition( { name: 'claude-default', @@ -1026,14 +1031,15 @@ describe('resolveTargetDefinition', () => { } expect(target.config.executable).toBe('claude'); + expect(target.config.command).toEqual(['claude']); }); - it('claude-cli accepts custom executable', () => { + it('claude-cli accepts custom command argv', () => { const target = resolveTargetDefinition( { name: 'claude-custom', provider: 'claude-cli', - executable: 'claude-custom', + command: ['claude-custom', '--config', 'profile=eval'], }, {}, ); @@ -1044,6 +1050,20 @@ describe('resolveTargetDefinition', () => { } expect(target.config.executable).toBe('claude-custom'); + expect(target.config.command).toEqual(['claude-custom', '--config', 'profile=eval']); + }); + + it('claude-cli rejects removed executable and args fields', () => { + expect(() => + resolveTargetDefinition( + { + name: 'claude-custom', + provider: 'claude-cli', + executable: 'claude-custom', + } as never, + {}, + ), + ).toThrow(/executable.*config.command/i); }); it('resolves copilot-cli as its own provider kind', () => { @@ -1063,6 +1083,7 @@ describe('resolveTargetDefinition', () => { } expect(target.config.executable).toBe('copilot'); + expect(target.config.command).toEqual(['copilot']); expect(target.config.model).toBe('claude-haiku-4.5'); expect(target.config.timeoutMs).toBe(600000); }); @@ -1082,6 +1103,7 @@ describe('resolveTargetDefinition', () => { } expect(target.config.executable).toBe('copilot'); + expect(target.config.command).toEqual(['copilot']); }); it('resolves copilot-cli flat base_url/api_key as custom provider', () => { diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index 20d3e5319..cc9f21f39 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -170,6 +170,8 @@ targets: provider: google-gemini - label: copilot-alias provider: copilot + - label: claude-alias + provider: claude - label: copilot-sdk-alias provider: copilot_sdk - label: pi-alias @@ -189,7 +191,6 @@ targets: 'azure-openai', 'google', 'google-gemini', - 'copilot', 'copilot_sdk', 'pi', 'claude-code', @@ -205,6 +206,27 @@ targets: ), ).toBe(true); } + + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[3].provider' && + error.message.includes("Ambiguous provider 'copilot'") && + error.message.includes('copilot-cli') && + error.message.includes('copilot-sdk'), + ), + ).toBe(true); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[4].provider' && + error.message.includes("Ambiguous provider 'claude'") && + error.message.includes('claude-cli') && + error.message.includes('claude-sdk'), + ), + ).toBe(true); }); it('rejects camelCase target aliases', async () => { @@ -471,7 +493,7 @@ targets: provider: copilot-cli log_format: json - label: claude-agent - provider: claude + provider: claude-cli log_output_format: summary `, ); From a96f3a601e9e7b515d0240841c19b1de411af3ca Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 18:03:33 +0200 Subject: [PATCH 08/10] docs: document target runtime config graph --- README.md | 47 +- .../docs/docs/next/evaluation/eval-files.mdx | 53 ++- .../docs/next/evaluation/running-evals.mdx | 8 +- .../docs/docs/next/graders/llm-graders.mdx | 8 +- .../docs/docs/next/targets/cli-provider.mdx | 79 +++- .../docs/docs/next/targets/coding-agents.mdx | 447 ++++++++---------- .../docs/docs/next/targets/configuration.mdx | 76 ++- .../docs/next/targets/custom-providers.mdx | 25 +- .../docs/docs/next/targets/llm-providers.mdx | 81 ++-- .../content/docs/docs/next/targets/retry.mdx | 20 +- .../docs/v4.42.4/evaluation/eval-files.mdx | 434 ++++++++++++++--- .../docs/v4.42.4/evaluation/running-evals.mdx | 412 +++++++++++----- .../docs/docs/v4.42.4/graders/llm-graders.mdx | 8 +- .../docs/v4.42.4/targets/cli-provider.mdx | 89 ++-- .../docs/v4.42.4/targets/coding-agents.mdx | 428 +++++++++-------- .../docs/v4.42.4/targets/configuration.mdx | 345 ++++++++++---- .../docs/v4.42.4/targets/custom-providers.mdx | 28 +- .../docs/v4.42.4/targets/llm-providers.mdx | 92 ++-- .../docs/docs/v4.42.4/targets/retry.mdx | 23 +- .../docs/docs/v4.42.4/tools/dashboard.mdx | 137 +++--- examples/features/readme-quickstart/README.md | 3 +- .../readme-quickstart/evals/my-eval.eval.yaml | 2 +- .../features/readme-quickstart/targets.yaml | 13 +- 23 files changed, 1854 insertions(+), 1004 deletions(-) diff --git a/README.md b/README.md index 849ac742a..1a3069a24 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ Test AI targets on real repo tasks and measure what actually works. - **Eval suite / imports / tests** are the task corpus: the prompts, cases, datasets, and imported benchmarks you want to evaluate. - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval. - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts. -- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by label from `targets.yaml` or with an eval-local target object. +- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object. - **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag. -- **Evaluate options** configure runner-level behavior such as repeat policy, optional timeouts, and `max_concurrency` under `evaluate_options`. +- **Execution** configures runner-level behavior such as general suite concurrency with `execution.max_concurrency`; `evaluate_options` holds eval behaviors such as repeat policy and budgets. - **Default test** configures inherited per-test defaults such as score `threshold`. - **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend. @@ -31,16 +31,31 @@ npm install -g agentv agentv init ``` -**2. Configure targets** in `.agentv/targets.yaml` — point to the system under test, such as an agent, provider, gateway, replay source, or CLI wrapper. Provider-specific budgets belong here: +**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags: ```yaml targets: - - label: local-openai + - id: local-openai provider: openai - api_format: chat - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + runtime: host + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + +graders: + - id: local-openai-grader + provider: openai + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + +defaults: + target: local-openai + grader: local-openai-grader ``` **3. Create shared test defaults** in `evals/default-test.yaml`. This is a promptfoo-style partial test config that AgentV applies to each test: @@ -68,7 +83,7 @@ description: Code generation quality tags: experiment: with-skills target: local-openai -evaluate_options: +execution: max_concurrency: 1 default_test: file://./default-test.yaml @@ -104,8 +119,14 @@ description: Code generation quality with eval-local target settings tags: experiment: with-skills target: - extends: local-openai - model: gpt-5.4-mini + id: local-mini + provider: openai + runtime: host + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: gpt-5.4-mini evaluate_options: repeat: count: 2 @@ -119,7 +140,7 @@ tests: input: Write FizzBuzz in Python ``` -`target: local-openai` resolves the target label from `.agentv/targets.yaml` or `targets.yaml` and uses its default provider, model, hooks, and provider settings. The object form above starts from `local-openai`, then applies the eval-local fields for this eval. If `extends` is omitted, the object defines the full target inline and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata. +`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata. Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file, matching promptfoo's external defaults pattern: @@ -150,7 +171,7 @@ agentv results compare .agentv/results//index.jsonl .agentv/res ## Results -Each run writes a portable bundle directly under `.agentv/results//`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `targets.yaml`; both are recorded as metadata, not path segments. The root `index.jsonl` manifest is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run. +Each run writes a portable bundle directly under `.agentv/results//`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The root `index.jsonl` manifest is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run. ```bash agentv eval evals/my-eval.eval.yaml diff --git a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx index 05fd933c1..8c521f972 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/eval-files.mdx @@ -5,20 +5,32 @@ sidebar: order: 1 --- -Evaluation files define the test cases, graders, workspace lifecycle, and run controls for an evaluation run. The reserved `tags.experiment` key is the run/result grouping label, top-level `target` identifies the system under test, and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`, `evaluate_options.budget_usd`, and `evaluate_options.max_concurrency` control repeated attempts and gates. Workspace lifetime belongs under `workspace.scope`; repository provenance belongs under `workspace.repos`; Docker/container binding belongs under `workspace.docker`. Non-provisioning setup commands belong in top-level `extensions`; reset policy stays under `workspace.hooks.after_each.reset`; runner-specific setup belongs in the `target` object or `targets.yaml`. AgentV supports two eval data formats: YAML and JSONL. +Evaluation files define the test cases, graders, workspace lifecycle, and run +controls for an evaluation run. The reserved `tags.experiment` key is the +run/result grouping label, top-level `target` identifies the system under test, +and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`, +`evaluate_options.budget_usd`, and `execution.max_concurrency` control repeated +attempts and gates. Workspace lifetime belongs under `workspace.scope`; +repository provenance belongs under `workspace.repos`; Docker/container binding +belongs under `workspace.docker`. Non-provisioning setup commands belong in +top-level `extensions`; reset policy stays under +`workspace.hooks.after_each.reset`; runner-specific setup belongs in the +`target` object, in `targets`, or in project config. AgentV supports two eval +data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. Eval files describe the task, target binding, and run controls. Use -`execution.max_concurrency` or `evaluate_options.max_concurrency` for authored -suite concurrency. Operators can still override concurrency with -`agentv eval --workers N`; do not author legacy `workers` fields in eval YAML. +`execution.max_concurrency` for authored suite concurrency. Operators can still +override concurrency with `agentv eval --workers N`; do not author legacy +`workers` fields in eval YAML. ## Authoring Shapes -Eval YAML is AgentV's composable and runnable authoring primitive. Use ordinary -`*.eval.yaml` files for direct task suites and for wrapper evals that compose -other suites. Raw case files are reusable data inputs, not a second runnable -experiment format. +Eval YAML is AgentV's composable and runnable authoring primitive. It is a +focused, shareable slice of the same config graph as `.agentv/config.yaml`. +Use ordinary `*.eval.yaml` files for direct task suites and for wrapper evals +that compose other suites. Raw case files are reusable data inputs, not a +second runnable experiment format. - A **task suite** is eval YAML that owns task context: `workspace`, shared `input`, shared `assert`, fixtures, graders, and test cases. It can run @@ -68,6 +80,9 @@ A wrapper eval stays ordinary eval YAML while choosing a target and run controls # experiments/refunds-codex.eval.yaml name: refunds-codex target: codex-gpt5 +execution: + max_concurrency: 3 + evaluate_options: repeat: count: 2 @@ -120,13 +135,14 @@ tests: | `description` | Human-readable description of the evaluation | | `suite` | Optional suite identifier | | `category` | Optional slash-delimited analytics taxonomy path. Overrides the category derived from the eval file path. | -| `target` | Named system under test from `.agentv/targets.yaml` or `--targets` | +| `target` | System under test by configured target `id` or inline target object | | `tags` | Optional promptfoo-style metadata map. Use `tags.experiment` as the run/result grouping label. | | `prompts` | Optional top-level prompt matrix. Entries can be strings, chat message arrays, files, or generated prompt functions. | -| `targets` | Optional target matrix. Entries reference target labels or inline target objects. | +| `targets` | Optional target matrix. Entries reference target ids or inline target objects. | | `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` | | `timeout_seconds` | Optional per-case timeout | -| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `max_concurrency` | +| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `repeat` | +| `execution.max_concurrency` | Optional general eval parallelism for this suite | | `threshold` | Optional suite quality threshold | | `workspace` | Suite-level task environment — inline object or string path to an external workspace file. Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). | | `extensions` | Promptfoo-style lifecycle hooks: `file://path/to/hooks.mjs:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, plus the built-in `agentv:agent-rules`. Hooks run after `workspace.repos` materializes. | @@ -166,10 +182,17 @@ prompts: prompt: "In one sentence, summarize {{ vars.topic }}." targets: - - label: local-mini - id: openai:gpt-5.4-mini - - label: local-codex - id: codex-auto-review + - id: local-mini + provider: openai + runtime: host + config: + model: gpt-5.4-mini + - id: local-codex + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex tests: - id: release-notes diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index ecaa07778..8148f884c 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -620,9 +620,11 @@ targets: - id: replay_coding_agent provider: replay - fixtures: ../fixtures/legal-review-target-output.jsonl - source_target: live_coding_agent - suite: legal-review + runtime: host + config: + fixtures: ../fixtures/legal-review-target-output.jsonl + source_target: live_coding_agent + suite: legal-review ``` Run the same eval against the replay alias: diff --git a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx index 4c6332b08..da6a8d270 100644 --- a/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/llm-graders.mdx @@ -112,7 +112,8 @@ tests: ## Per-Grader Target -By default, an `llm-rubric` uses the suite target's `grader_target`. Override it per grader when you need multiple grader models in one run: +By default, an `llm-rubric` uses `defaults.grader` from the resolved config graph. +Override it per assertion when you need multiple grader models in one run: ```yaml assert: @@ -126,7 +127,8 @@ assert: prompt: ./prompts/pass-fail.md ``` -Each `target:` value must match a named LLM target in `.agentv/targets.yaml`. +Each `target:` value must match a grader `id` from `.agentv/config.yaml`, an +eval-local `graders` entry, or a directly referenced `graders: file://...` field. ### TypeScript Template @@ -168,7 +170,7 @@ Evaluate and provide a score from 0 to 1.`; ## How It Works 1. AgentV renders the prompt template with variables from the test -2. The rendered prompt is sent to the grader target (configured in targets.yaml) +2. The rendered prompt is sent to the selected grader target 3. The LLM returns a structured evaluation with score, assertions array, and reasoning 4. Results are recorded in the output JSONL diff --git a/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx b/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx index 072ac6b4c..9248f576a 100644 --- a/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx +++ b/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx @@ -12,12 +12,25 @@ Because the contract is "we invoke a command and read a file," almost any useful ## Minimal example ```yaml -# .agentv/targets.yaml +# .agentv/config.yaml targets: - - label: my_agent + - id: my-agent provider: cli - command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} - grader_target: azure-base # required if your evals use LLM graders + runtime: host + config: + command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} + +graders: + - id: azure-grader + provider: azure + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + +defaults: + target: my-agent + grader: azure-grader ``` Your `agent.py` reads the prompt, writes its response to the path passed as `--out`, and exits `0`. That's it. @@ -70,18 +83,17 @@ echo "Hello, world!" > {OUTPUT_FILE} | Field | Type | Required | Default | Description | |---|---|---|---|---| -| `label` | string | yes | — | AgentV target name used by eval `target`, CLI `--target`, and comparisons. | +| `id` | string | yes | — | AgentV target identity used by eval `target`, CLI `--target`, and comparisons. | | `provider` | literal `"cli"` | yes | — | Selects this provider. | -| `command` | string | yes | — | Shell command template. | -| `timeout_seconds` | number | no | — | Kill the process if it runs longer than this. | -| `cwd` | string | no | eval dir | Working directory. Relative paths resolve against the eval file. | -| `files_format` | string | no | `{path}` | How each entry in `{FILES}` is formatted. Placeholders: `{path}`, `{basename}`. | -| `verbose` | boolean | no | `false` | Log the rendered command and cwd to stdout. Useful for debugging template substitution. | -| `keep_temp_files` | boolean | no | `false` | Preserve `{PROMPT_FILE}` / `{OUTPUT_FILE}` after the run — handy while iterating on your command. | -| `healthcheck` | object | no | — | Pre-run health check (HTTP or command); the eval aborts if it fails. | -| `workers` | number | no | — | Concurrent test-case executions against this target. | -| `batch_requests` | boolean | no | `false` | Run all cases in one command invocation — see [Batching](#batching). | -| `grader_target` | string | no | — | LLM target used by this target's LLM graders. Required if your evals use LLM-based graders. | +| `runtime` | string or object | no | `host` | Target runtime placement. | +| `config.command` | string | yes | — | Shell command template. | +| `config.timeout_seconds` | number | no | — | Kill the process if it runs longer than this. | +| `config.cwd` | string | no | eval dir | Working directory. Relative paths resolve against the eval file. | +| `config.files_format` | string | no | `{path}` | How each entry in `{FILES}` is formatted. Placeholders: `{path}`, `{basename}`. | +| `config.verbose` | boolean | no | `false` | Log the rendered command and cwd to stdout. Useful for debugging template substitution. | +| `config.keep_temp_files` | boolean | no | `false` | Preserve `{PROMPT_FILE}` / `{OUTPUT_FILE}` after the run — handy while iterating on your command. | +| `config.healthcheck` | object | no | — | Pre-run health check (HTTP or command); the eval aborts if it fails. | +| `config.batch_requests` | boolean | no | `false` | Run all cases in one command invocation — see [Batching](#batching). | ## Batching @@ -89,10 +101,12 @@ For targets where spin-up cost dominates per-case work (e.g. loading a model, au ```yaml targets: - - label: batched_agent + - id: batched-agent provider: cli - batch_requests: true - command: python agent.py --batch-in {PROMPT_FILE} --batch-out {OUTPUT_FILE} + runtime: host + config: + batch_requests: true + command: python agent.py --batch-in {PROMPT_FILE} --batch-out {OUTPUT_FILE} ``` `{PROMPT_FILE}` contains one JSON object per line with an `id` and the case's inputs; your command writes one line per case to `{OUTPUT_FILE}`, each carrying the matching `id` plus the same output shape as the non-batched case. @@ -104,17 +118,30 @@ A common question when building a new eval: **"if my grader scores my agent poor AgentV has no dedicated "oracle" feature because the `cli` provider already composes into one. Declare a second target that prints your known-good answer into `{OUTPUT_FILE}`, run the same eval against it, and assert a perfect score: ```yaml -# .agentv/targets.yaml +# .agentv/config.yaml targets: - - label: my_agent + - id: my-agent provider: cli - command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} - grader_target: azure-base + runtime: host + config: + command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} - - label: oracle + - id: oracle provider: cli - command: cp fixtures/{EVAL_ID}.expected.txt {OUTPUT_FILE} - grader_target: azure-base + runtime: host + config: + command: cp fixtures/{EVAL_ID}.expected.txt {OUTPUT_FILE} + +graders: + - id: azure-grader + provider: azure + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + +defaults: + grader: azure-grader ``` ```bash @@ -123,7 +150,7 @@ targets: agentv eval my.EVAL.yaml --target oracle # Then run the real target. -agentv eval my.EVAL.yaml --target my_agent +agentv eval my.EVAL.yaml --target my-agent ``` A few practical notes: diff --git a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx index 89aa0e508..cbbbcabe5 100644 --- a/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/next/targets/coding-agents.mdx @@ -5,36 +5,77 @@ sidebar: order: 3 --- -Coding agent targets evaluate AI coding assistants and CLI-based agents. Use `defaults.grader` or an evaluator-level grader override for LLM-based grading. +Coding agent targets evaluate assistants that run against a real workspace. They +use the target shape `id`, `provider`, `runtime`, and `config`. The target `id` +is AgentV's stable selection and artifact identity. `provider` names the +adapter or control boundary. `runtime` names where the target runs. Provider +settings live under `config`. -AgentV's recommended coding-agent targets use process or protocol boundaries such as `claude-cli`, `codex-cli`, `copilot-cli`, `pi-cli`, and `pi-rpc`. SDK-backed targets are opt-in provider kinds: `claude-sdk`, `codex-sdk`, `copilot-sdk`, and `pi-sdk`. When selected, AgentV runs the SDK adapter in an AgentV-owned child process and communicates with it through a structured protocol, so SDK crashes, malformed output, timeouts, and missing optional SDK packages are scoped to that target run instead of the AgentV orchestrator process. +Use `defaults.grader`, CLI `--grader` / `--grader-target`, or an +evaluator-specific target override for LLM-based grading. Grader selection is +separate from the coding-agent target, so target definitions do not carry a +grader field. -## Runtime isolation +```yaml +targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high + +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini + +defaults: + target: codex-local + grader: openai-grader + +execution: + max_concurrency: 3 +``` -Coding-agent targets can use three runtime placements: +Process-backed coding-agent providers use `config.command` as a non-empty argv +array. The first element is the executable or shim, and the remaining elements +are arguments. -- `runtime: host` runs the installed CLI or child runner on the current machine - with the configured local environment. Use this to evaluate the same agent - profile you use manually. -- `runtime.mode: profile` keeps host process execution but points the agent at - an explicit home/config/profile directory. -- `runtime.mode: sandbox` runs through a separate substrate such as Docker. The - first built-in sandbox path supports `provider: cli` with explicit - `runtime.image`, `runtime.mounts`, `runtime.env`, and `runtime.secrets`. - Provider-specific adapters such as `codex-cli`, `claude-cli`, `copilot-cli`, - and `pi-cli` currently return a structured unsupported target error in - sandbox mode until their transcript handling is sandbox-aware. +## Runtime placement -Sandbox mode never reuses host credentials implicitly. For reproducible CI, -prefer API keys or explicit secrets injected through `runtime.secrets`. -Subscription OAuth can be evaluated only by intentionally mounting or seeding -the profile directory the agent needs, which trades reproducibility for fidelity -to a local authenticated setup. +| Runtime | Boundary | Best fit | +| --- | --- | --- | +| `host` | Runs the installed CLI or child runner on the current machine. | Local research, subscription OAuth, and evaluating the same profile an engineer uses manually. | +| `profile` | Runs a host process with isolated home/config/env such as `HOME`, `CODEX_HOME`, or temp dirs. | Cleaner local evals without container cost. | +| `sandbox` | Runs through a separate substrate such as Docker or a managed sandbox. | CI, reproducibility, untrusted tasks, and stronger filesystem containment. | + +Sandbox mode does not inherit host credentials. For CI, prefer API keys or +explicit secrets injected through sandbox configuration. Subscription OAuth can +be evaluated only by intentionally mounting or seeding the profile directory +the agent needs, which trades portability for fidelity to the local agent setup. + +```yaml +targets: + - id: codex-clean-profile + provider: codex-cli + runtime: + mode: profile + codex_home: .agentv/profiles/codex-clean + tmp_dir: .agentv/tmp/codex-clean + config: + command: ["codex", "exec", "--json"] + model: gpt-5-codex + sandbox_mode: workspace-write + approval_policy: never +``` ```yaml targets: - id: codex-ci-sandbox - provider: cli + provider: codex-cli runtime: mode: sandbox engine: docker @@ -50,156 +91,113 @@ targets: secrets: OPENAI_API_KEY: ${{ OPENAI_API_KEY }} config: - command: "codex exec --json --output-file {OUTPUT_FILE} {PROMPT_FILE}" + command: ["codex", "exec", "--json"] + model: gpt-5-codex timeout_seconds: 300 ``` -## Prompt format - -Agent providers receive a structured prompt document with two sections: a **preread block** listing files the agent must read, and the **user query** containing the eval input. - -### File handling - -When an eval test includes `type: file` inputs, agent providers do **not** receive the file content inline. Instead, they receive: - -1. A preread block with `file://` URIs pointing to absolute paths on disk -2. The user query with `` reference tags - -The agent is expected to read the files itself using its filesystem tools. - -This differs from [LLM providers](/docs/targets/llm-providers), which receive file content embedded directly in the prompt as XML: - -```xml - -// file content is inlined here - -``` - -### Example prompt - -Given an eval with file inputs: - -```yaml -input: - - role: user - content: - - type: file - value: ./src/example.ts - - type: text - value: Review this code -``` - -The agent receives a prompt like: +## Provider tradeoffs -``` -Read all input files: -* [example.ts](file:///abs/path/src/example.ts). +| Provider | Boundary | Transcript and isolation notes | +| --- | --- | --- | +| `codex-app-server` | Codex app-server subprocess. | Preferred Codex path for rich protocol events, session control, cancellation, and structured transcripts. | +| `codex-cli` | Codex CLI subprocess. | Best for simple local/CI process isolation and evaluating an installed Codex shim or profile. | +| `codex-sdk` | Codex SDK in an AgentV child runner. | Explicit SDK path. SDK crashes, malformed child output, and missing optional SDK packages are target execution errors. | +| `pi-rpc` | Pi launched in RPC mode over stdio. | Preferred rich Pi control boundary; AgentV launches the configured command with RPC mode when needed. | +| `pi-cli` | Pi CLI subprocess. | Simple process boundary; transcript richness depends on Pi CLI output. | +| `pi-sdk` | Pi SDK in an AgentV child runner. | Explicit SDK path for SDK-native events with child-process isolation. | +| `claude-cli` | Claude CLI subprocess. | Default Claude path; captures structured stream output when available. | +| `claude-sdk` | Claude Agent SDK in an AgentV child runner. | Explicit SDK path; useful when SDK-native events matter more than matching a local CLI invocation. | +| `copilot-cli` | Copilot CLI subprocess/protocol path. | Active Copilot eval run through the installed process. | +| `copilot-log` | Passive Copilot session-log reader. | Zero-cost transcript grading for existing sessions; it does not run a new agent. | +| `copilot-sdk` | Copilot SDK in an AgentV child runner. | Explicit SDK path with child-process isolation. | -If any file is missing, fail with ERROR: missing-file and stop. -Then apply system_instructions on the user query below. - -[[ ## user_query ## ]] - -Review this code -``` +Every coding-agent provider returns a structured target execution envelope. +Run bundles preserve target id, provider kind, runtime mode, command argv, cwd, +stdout/stderr, transcripts or logs, final output when available, timing, +timeouts, exit codes, signals, and partial artifacts on failure. -The preread block instructs the agent to read input files before processing the query. If a `system_prompt` is configured on the target, it is passed separately via the provider SDK (not in the prompt document). +## Codex -## Claude +Use `codex-app-server` when you want rich protocol control: ```yaml targets: - - id: claude-local - provider: claude-cli + - id: codex-local + provider: codex-app-server runtime: host config: - command: ["claude"] - model: claude-sonnet-4-20250514 + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high + model_verbosity: medium ``` -| Field | Required | Description | -|-------|----------|-------------| -| `config.command` | No | Claude CLI argv. Defaults to `["claude"]`; use this for host shims or alternate installed binaries. | -| `config.model` | No | Model to pass to the Claude CLI. | -| `config.cwd` | No | Working directory. | -| `config.timeout_seconds` | No | Per-case timeout. | -| `config.max_turns` | No | Claude CLI turn limit. | -| `config.bypass_permissions` | No | Defaults to true for autonomous eval runs. Set false to omit `--dangerously-skip-permissions`. | - -Use `provider: claude-cli` when you want AgentV to spawn the same Claude CLI a user would run locally. It captures Claude's structured stream output when available and stores stdout, stderr, final output, and target execution metadata in the run bundle. - -Use `provider: claude-sdk` only when you intentionally want the Claude Agent SDK path. AgentV runs SDK providers inside an isolated child runner, so SDK crashes, malformed child output, timeouts, and missing optional SDK packages are reported as target execution errors rather than crashing the AgentV process. SDK transcripts may be richer for SDK-native events, while CLI transcripts better match the installed local agent. - -## Codex CLI +Use `codex-cli` when a simple CLI boundary is enough or when you want an +operator-specific shim: ```yaml targets: - - id: codex-target + - id: codex-eng provider: codex-cli runtime: host config: - command: ["codex-eng"] + command: ["codex-eng", "exec", "--json"] model: ${{ CODEX_MODEL }} - reasoning_effort: ${{ CODEX_REASONING_EFFORT }} ``` -| Field | Required | Description | -|-------|----------|-------------| -| `config.command` | Yes | Codex binary or profile shim argv, such as `["codex-eng"]` | -| `config.model` | No | Model to use | -| `config.reasoning_effort` | No | Codex reasoning effort: `minimal`, `low`, `medium`, `high`, or `xhigh` | -| `config.cwd` | No | Working directory | - -Use `provider: codex-sdk` only when you intentionally want the Codex SDK path. The SDK package is optional and loaded inside the isolated child runner. - -## Copilot CLI +Use `codex-sdk` only when you intentionally want the Codex SDK path: ```yaml targets: - - id: copilot-local - provider: copilot-cli + - id: codex-sdk-isolated + provider: codex-sdk runtime: host config: - command: ["copilot"] - model: gpt-5-mini + model: gpt-5-codex ``` -| Field | Required | Description | -|-------|----------|-------------| -| `config.command` | No | Copilot CLI argv. Defaults to `["copilot"]`; use this for host shims or alternate installed binaries. | -| `config.model` | No | Model to use. Defaults to Copilot's configured default. | -| `config.cwd` | No | Working directory. | -| `config.subprovider` | No | OpenAI-compatible provider type for `copilot-cli` or `copilot-sdk`, such as `openai` or `azure`. | -| `config.base_url` | No | Provider base URL or Azure resource URL/name. | -| `config.api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. | -| `config.bearer_token` | No | Provider bearer token. Prefer `${{ ENV_VAR }}` references. Takes precedence over `api_key` when set. | -| `config.api_version` | No | Provider API version, primarily for Azure endpoints. | -| `config.api_format` | No | Provider API format, such as `responses`. | +Common Codex config fields include `command`, `model`, `reasoning_effort`, +`model_verbosity`, `base_url`, `api_key`, `api_format`, `sandbox_mode`, +`approval_policy`, `cwd`, `timeout_seconds`, `log_dir`, `stream_log`, and +`system_prompt`. -Route Copilot through an OpenAI-compatible endpoint: +## Pi + +Use `pi-rpc` for the rich stdio/RPC boundary: ```yaml targets: - - id: copilot-openai - provider: copilot-cli + - id: pi-rpc-local + provider: pi-rpc runtime: host config: - command: ["copilot"] - subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - api_format: responses + command: ["pi"] + model: gpt-5-codex + thinking: medium ``` -Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps these flat fields to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged. +Use `pi-cli` for simple subprocess execution: -Use `provider: copilot-cli` for active eval runs through Copilot's ACP subprocess. Use `provider: copilot-log` when you want a passive transcript reader for an existing Copilot CLI session under `~/.copilot/session-state/`; it does not run a new agent and is useful for zero-cost transcript grading. Use `provider: copilot-sdk` only when you intentionally want the Copilot SDK path. SDK providers run inside AgentV's isolated child runner; CLI and log providers keep the boundary at the installed process or existing session logs. CLI runs generally provide the best "what the user actually ran" match, SDK runs may expose SDK-native events, and log runs are limited to the transcript richness already present on disk. +```yaml +targets: + - id: pi-cli-local + provider: pi-cli + runtime: + mode: profile + home: .agentv/profiles/pi-local + config: + command: ["pi"] + subprovider: openrouter + model: ${{ OPENROUTER_MODEL }} + api_key: ${{ OPENROUTER_API_KEY }} +``` -## Pi SDK +Use `pi-sdk` only when you intentionally want the SDK path: ```yaml targets: - - id: pi_target + - id: pi-sdk-isolated provider: pi-sdk runtime: host config: @@ -208,176 +206,139 @@ targets: thinking: medium ``` -| Field | Required | Description | -|-------|----------|-------------| -| `subprovider` | No | Pi provider to use, such as `google`, `openai`, `openai-codex`, `azure`, `anthropic`, or `openrouter`. Defaults to Pi's default provider. | -| `model` | No | Model to use. For OpenAI subscription auth through Pi, use `subprovider: openai-codex` with a subscription model such as `gpt-5.5`. | -| `thinking` | No | Pi reasoning level: `off`, `minimal`, `low`, `medium`, `high`, or `xhigh`. Passed to the Pi SDK as `thinkingLevel`. | -| `tools` | No | Comma-separated Pi tool allowlist, such as `read,bash,edit,write`. | -| `api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. Omit for subscription auth handled by Pi. | -| `base_url` | No | Provider base URL or Azure resource URL/name. | -| `cwd` | No | Working directory | -| `timeout_seconds` | No | Per-case timeout | -| `grader_target` | Yes | LLM target for evaluation | - -For `provider: pi-sdk`, `base_url` is passed through the Pi SDK model -configuration. This works for OpenAI-compatible endpoints: +Pi config fields include `command`, `subprovider`, `model`, `thinking`, +`tools`, `api_key`, `base_url`, `cwd`, `timeout_seconds`, `log_dir`, +`stream_log`, and `system_prompt`. With `pi-cli`, the built-in OpenAI provider +does not expose a CLI base-url option; use a Pi custom provider name or Pi's +Azure provider path for custom gateways. + +## Claude ```yaml targets: - - id: pi-sdk-openai - provider: pi-sdk + - id: claude-local + provider: claude-cli runtime: host config: - subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} + command: ["claude"] + model: claude-sonnet-4-20250514 + max_turns: 10 ``` -Use `provider: pi-rpc` when you want AgentV to launch Pi in Kata-style RPC mode and communicate over process stdio. AgentV starts the configured command with `--mode rpc` unless the command already includes a mode flag. The Pi runtime must already be installed or reachable through `config.command`; AgentV does not install workers or provision a fleet for this target. +Use `claude-cli` when you want AgentV to spawn the same Claude CLI a user runs +locally. Use `claude-sdk` only when you intentionally want the Claude Agent SDK +path: ```yaml targets: - - id: pi-rpc-local - provider: pi-rpc + - id: claude-sdk-isolated + provider: claude-sdk runtime: host config: - command: ["pi"] - model: gpt-5-codex + model: claude-sonnet-4-20250514 + max_turns: 10 ``` -Use `provider: pi-cli` when you want AgentV to spawn the `pi` binary for simple JSONL subprocess execution: +Claude config fields include `command`, `model`, `cwd`, `timeout_seconds`, +`max_turns`, `max_budget_usd`, `bypass_permissions`, `log_dir`, `stream_log`, +and `system_prompt`. + +## Copilot ```yaml targets: - - id: pi-cli-local - provider: pi-cli - runtime: - mode: profile - home: .agentv/profiles/pi-local + - id: copilot-local + provider: copilot-cli + runtime: host config: - command: ["pi"] - model: gpt-5-codex + command: ["copilot"] + model: gpt-5-mini ``` -`pi-cli` and `pi-rpc` accept the same Pi fields above plus: - -| Field | Required | Description | -|-------|----------|-------------| -| `command` | No | Non-empty argv array for the Pi binary, shim, or remote command. Defaults to `["pi"]`. | - -Pi CLI has one important difference from the SDK path: the built-in `openai` -provider does not currently expose a CLI base-url option. With `provider: pi-cli` -and `subprovider: openai`, AgentV can pass the API key and model, but `base_url` -does not re-route the built-in OpenAI provider. For custom endpoints, either -configure a Pi custom provider in Pi's own `models.json` and reference that -provider name as `subprovider`, or use Pi's Azure provider path when your gateway -is compatible with Azure OpenAI Responses: +Route Copilot through an OpenAI-compatible endpoint: ```yaml targets: - - id: pi-cli-gateway - provider: pi-cli + - id: copilot-openai + provider: copilot-cli runtime: host config: - command: ["pi"] - subprovider: azure + command: ["copilot"] + subprovider: openai base_url: ${{ OPENAI_ENDPOINT }} api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} -``` - -## VS Code - -```yaml -targets: - - label: vscode_dev - provider: vscode - grader_target: azure-base + api_format: responses ``` -| Field | Required | Description | -|-------|----------|-------------| -| `executable` | No | Path to VS Code binary. Supports `${{ ENV_VAR }}` syntax or literal paths. Defaults to `code` (or `code-insiders` for the insiders provider). | -| `grader_target` | Yes | LLM target for evaluation | - -Using a custom executable path: +Read an existing Copilot session log without running a new agent: ```yaml targets: - - label: vscode_dev - provider: vscode - executable: ${{ VSCODE_CMD }} - grader_target: azure-base + - id: copilot-session-log + provider: copilot-log + runtime: host + config: + discover: latest ``` -## VS Code Insiders +Use `copilot-sdk` only when you intentionally want the SDK path: ```yaml targets: - - label: vscode_insiders - provider: vscode-insiders - grader_target: azure-base + - id: copilot-sdk-isolated + provider: copilot-sdk + runtime: host + config: + model: gpt-5-mini ``` -Same configuration as VS Code. +Copilot config fields include `command`, `model`, `cwd`, `timeout_seconds`, +`subprovider`, `base_url`, `api_key`, `bearer_token`, `api_version`, +`api_format`, `log_dir`, `stream_log`, `system_prompt`, and session-log fields +such as `discover`, `session_id`, and `session_dir` for `copilot-log`. -## Custom CLI Agent +## File inputs -Evaluate any command-line agent: +Agent providers receive file inputs as paths, not inline file content. The +prompt includes a preread block with `file://` URIs pointing to absolute paths +on disk, then the user query references each file: ```yaml -targets: - - label: local_agent - provider: cli - command: 'python agent.py --prompt-file {PROMPT_FILE} --output {OUTPUT_FILE}' - grader_target: azure-base +input: + - role: user + content: + - type: file + value: ./src/example.ts + - type: text + value: Review this code ``` -| Field | Required | Description | -|-------|----------|-------------| -| `command` | Yes | Command to run. `{PROMPT}` is inline prompt text and `{PROMPT_FILE}` is a temp file path containing the prompt. | -| `cwd` | No | Working directory | -| `grader_target` | Yes | LLM target for evaluation | +The agent receives a prompt like: -## Mock Provider +```text +Read all input files: +* [example.ts](file:///abs/path/src/example.ts). -For testing the evaluation harness without calling real providers: +If any file is missing, fail with ERROR: missing-file and stop. +Then apply system_instructions on the user query below. -```yaml -targets: - - label: mock_target - provider: mock +[[ ## user_query ## ]] + +Review this code ``` -## Known limitations - -### VS Code - -The VS Code provider uses a **subagent file-messaging architecture**. AgentV provisions pre-configured VS Code workspace directories (subagents), dispatches requests by writing prompt files, and the AI agent writes its response to a file. Lock files control concurrency. +LLM providers receive file content inline instead; see +[LLM providers](/docs/targets/llm-providers). -- **Per-target worker limit**: VS Code evals run with 1 worker per target because the provider requires window focus to dispatch requests. When multiple targets are configured (e.g., `vscode` + `copilot`), they run concurrently — the single-worker limit only applies within each VS Code target. Subagents are provisioned automatically if needed. -- **Windows only**: VS Code is not available on Linux CI. E2E testing must be done on a Windows machine. -- **`.code-workspace` support**: When your eval uses `workspace.template` with a `.code-workspace` file, the template folders are opened in the VS Code window alongside the subagent directory. +## Mock provider -### Copilot CLI - -- **MCP OAuth token expiration**: If your copilot CLI has MCP servers configured that use OAuth authentication, **expired tokens will block eval execution**. The copilot CLI attempts to re-authenticate via a browser OAuth flow, which cannot complete in non-interactive mode and causes the eval to hang indefinitely. Before running evals, either re-authenticate your MCP servers manually (`copilot` → `/mcp`) or remove MCP servers with expired tokens. See [copilot-cli#1797](https://github.com/github/copilot-cli/issues/1797) and [copilot-cli#1491](https://github.com/github/copilot-cli/issues/1491) for upstream tracking. -- **Windows shell shim vs process spawn**: On Windows, `copilot -h` may work in PowerShell while AgentV still fails with `spawn copilot ENOENT`. Shell commands can execute `copilot.ps1`/`copilot.bat`, but AgentV launches a subprocess that expects a directly spawnable executable path. If this occurs, set an explicit command argv with a native executable path: +For deterministic harness checks without a real provider: ```yaml targets: - - id: copilot-local - provider: copilot-cli + - id: mock-target + provider: mock runtime: host config: - command: ["${{ COPILOT_EXE }}"] + response: ok ``` - -Use a native binary path for `COPILOT_EXE` (for example `copilot.exe` from `@github/copilot-win32-x64`). - -### Claude Code - -- **Run evals externally**: Run agentv evals from **outside** Claude Code. Running `agentv eval` with a `claude-cli` target from within a Claude Code session can cause unintended behavior — the spawned Claude agent may interfere with the parent session. -- **`ANTHROPIC_API_KEY` overrides subscription auth**: Claude Code loads `.env` from the working directory on startup. If your `.env` contains `ANTHROPIC_API_KEY`, the spawned Claude Code process will use that API key instead of your Claude subscription (Max/Pro). If the API key has insufficient credits, evals will fail with "Credit balance is too low". To use subscription auth, remove `ANTHROPIC_API_KEY` from your `.env` file. diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index 27ac54bfb..c66fc5a6c 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -5,10 +5,22 @@ sidebar: order: 1 --- -Targets define which agent or LLM provider to evaluate. Project manifests can -keep targets inline in `.agentv/config.yaml` or decompose them into a direct -field reference such as `targets: file://targets.yaml`. Both forms normalize to -the same config graph. +Targets define which agent or LLM provider to evaluate. AgentV uses one +composable config graph across project manifests and eval files: + +- `.agentv/config.yaml` is the project-local discovery and composition root. It + can hold targets, graders, tests, defaults, execution policy, results + settings, and repo-local project policy. +- `$AGENTV_HOME/config.yaml` is the user/operator config. Use it for defaults + that apply across projects, project registry data, default result locations, + and provider defaults that should not be copied into each repo. +- `eval.yaml` is a focused, shareable slice of the same graph. Use it for a + suite-specific target, grader, tests, evaluator settings, or run controls + that should travel with the eval. + +Any supported top-level field can stay inline or become a direct field +reference such as `targets: file://targets.yaml`. Both forms normalize to the +same config graph. ## Structure @@ -45,7 +57,7 @@ Use `id` for the stable AgentV target identity. `provider` selects the adapter or control boundary. `runtime` describes where the provider runs; use `host` as the shorthand for the current machine, or object form when you need `mode: host | profile | sandbox` plus runtime-specific settings. Provider -settings belong under `config`. Process-backed providers use +settings belong under `config`. Process-backed coding-agent providers use `config.command` as a non-empty argv array. ## Runtime Modes @@ -68,7 +80,7 @@ error until their transcript parsers are wired through sandbox-aware runners. ```yaml targets: - id: codex-sandbox - provider: cli + provider: codex-cli runtime: mode: sandbox engine: docker @@ -87,7 +99,7 @@ targets: secrets: OPENAI_API_KEY: ${{ OPENAI_API_KEY }} config: - command: "codex exec --json --output-file {OUTPUT_FILE} {PROMPT_FILE}" + command: ["codex", "exec", "--json"] timeout_seconds: 300 ``` @@ -104,16 +116,62 @@ or seed the relevant profile directory into the sandbox. That makes the run less portable than API-key CI and should be reserved for workflows where matching a local subscription profile is the point of the evaluation. -Any supported top-level field can be moved to a file reference: +Inline and decomposed forms are equivalent. This single-file config: + +```yaml +targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini + +tests: + - id: smoke + input: Fix the failing test. + +defaults: + target: codex-local + grader: openai-grader +``` + +can be decomposed like this: ```yaml targets: file://targets.yaml graders: file://graders.yaml +tests: file://tests.yaml defaults: file://defaults.yaml ``` Referenced field files contain the field value directly. `targets.yaml` contains -a bare array, not an object wrapped in `targets:`. +a bare array, not an object wrapped in `targets:`: + +```yaml +# .agentv/targets.yaml +- id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex +``` + +```yaml +# .agentv/defaults.yaml +target: codex-local +grader: openai-grader +``` + +File refs are optional. Use them when a config field is large, reused, or owned +by a separate team; keep fields inline when that is easier to read. ## Environment Variables diff --git a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx index 1310747d8..7e810a233 100644 --- a/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx +++ b/apps/web/src/content/docs/docs/next/targets/custom-providers.mdx @@ -175,11 +175,24 @@ registry.register('http-agent', (target: ResolvedTarget) => { Then reference it in your targets file: ```yaml -# .agentv/targets.yaml +# .agentv/config.yaml targets: - - label: my_http_agent + - id: my-http-agent provider: http-agent - grader_target: azure-base + runtime: host + config: + base_url: http://localhost:8080 + +graders: + - id: azure-grader + provider: azure + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + +defaults: + grader: azure-grader ``` :::note @@ -208,9 +221,11 @@ Use `provider: cli` when: ```yaml targets: - - label: python_agent + - id: python-agent provider: cli - command: 'python agent.py --prompt-file {PROMPT_FILE} --output {OUTPUT_FILE}' + runtime: host + config: + command: 'python agent.py --prompt-file {PROMPT_FILE} --output {OUTPUT_FILE}' ``` ### When to use native providers diff --git a/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx b/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx index 4a3bad4c4..0a0df092b 100644 --- a/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx +++ b/apps/web/src/content/docs/docs/next/targets/llm-providers.mdx @@ -11,10 +11,12 @@ LLM provider targets call language model APIs directly. These are used both as e ```yaml targets: - - label: openai-target + - id: openai-target provider: openai - api_key: ${{ OPENAI_API_KEY }} - model: gpt-4o + runtime: host + config: + api_key: ${{ OPENAI_API_KEY }} + model: gpt-4o ``` | Field | Required | Description | @@ -38,19 +40,23 @@ Most users should leave this unset. The default `chat` format is universally sup ```yaml # OpenAI-compatible endpoint (default chat format works) targets: - - label: github-models + - id: github-models provider: openai - api_format: chat - base_url: https://models.github.ai/inference/v1 - api_key: ${{ GH_MODELS_TOKEN }} - model: ${{ GH_MODELS_MODEL }} + runtime: host + config: + api_format: chat + base_url: https://models.github.ai/inference/v1 + api_key: ${{ GH_MODELS_TOKEN }} + model: ${{ GH_MODELS_MODEL }} # Opt in to Responses API for api.openai.com - - label: openai-responses + - id: openai-responses provider: openai - api_format: responses - api_key: ${{ OPENAI_API_KEY }} - model: gpt-4o + runtime: host + config: + api_format: responses + api_key: ${{ OPENAI_API_KEY }} + model: gpt-4o ``` ### Local OpenAI-compatible endpoints @@ -59,13 +65,14 @@ For smoke tests and dogfood runs against a local OpenAI-compatible proxy, keep the endpoint, model, and placeholder key in environment variables: ```yaml -targets: - - label: local-openai-grader +graders: + - id: local-openai-grader provider: openai - api_format: chat - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} ``` If the local proxy does not require authentication, set @@ -77,11 +84,13 @@ shared eval files. ```yaml targets: - - label: azure-base + - id: azure-base provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + runtime: host + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} ``` | Field | Required | Description | @@ -99,12 +108,14 @@ If your Azure deployment only exposes `/chat/completions` (older deployments, ce ```yaml targets: - - label: azure-chat + - id: azure-chat provider: openai - base_url: https://.openai.azure.com/openai/deployments/ - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: - api_format: chat + runtime: host + config: + base_url: https://.openai.azure.com/openai/deployments/ + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: + api_format: chat ``` The `api_format` field was previously available on `provider: azure` but has been removed — Azure targets always go through the Responses API. @@ -113,10 +124,12 @@ The `api_format` field was previously available on `provider: azure` but has bee ```yaml targets: - - label: claude_target + - id: claude-target provider: anthropic - api_key: ${{ ANTHROPIC_API_KEY }} - model: claude-sonnet-4-20250514 + runtime: host + config: + api_key: ${{ ANTHROPIC_API_KEY }} + model: claude-sonnet-4-20250514 ``` | Field | Required | Description | @@ -128,10 +141,12 @@ targets: ```yaml targets: - - label: gemini_target + - id: gemini-target provider: gemini - api_key: ${{ GEMINI_API_KEY }} - model: gemini-2.0-flash + runtime: host + config: + api_key: ${{ GEMINI_API_KEY }} + model: gemini-2.0-flash ``` | Field | Required | Description | diff --git a/apps/web/src/content/docs/docs/next/targets/retry.mdx b/apps/web/src/content/docs/docs/next/targets/retry.mdx index 05fb3638a..725a5ded5 100644 --- a/apps/web/src/content/docs/docs/next/targets/retry.mdx +++ b/apps/web/src/content/docs/docs/next/targets/retry.mdx @@ -13,16 +13,18 @@ Add retry fields to any target: ```yaml targets: - - label: azure-base + - id: azure-base provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - max_retries: 5 - retry_initial_delay_ms: 2000 - retry_max_delay_ms: 120000 - retry_backoff_factor: 2 - retry_status_codes: [500, 408, 429, 502, 503, 504] + runtime: host + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + max_retries: 5 + retry_initial_delay_ms: 2000 + retry_max_delay_ms: 120000 + retry_backoff_factor: 2 + retry_status_codes: [500, 408, 429, 502, 503, 504] ``` ## Fields diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx index df2e3b6a3..8c521f972 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/eval-files.mdx @@ -3,36 +3,127 @@ title: Eval Files description: YAML and JSONL evaluation file formats sidebar: order: 1 -slug: docs/v4.42.4/evaluation/eval-files -editUrl: false -pagefind: false --- -Evaluation files define the test cases, targets, and graders for an evaluation run. AgentV supports two formats: YAML and JSONL. +Evaluation files define the test cases, graders, workspace lifecycle, and run +controls for an evaluation run. The reserved `tags.experiment` key is the +run/result grouping label, top-level `target` identifies the system under test, +and fields such as `evaluate_options.repeat`, `threshold`, `timeout_seconds`, +`evaluate_options.budget_usd`, and `execution.max_concurrency` control repeated +attempts and gates. Workspace lifetime belongs under `workspace.scope`; +repository provenance belongs under `workspace.repos`; Docker/container binding +belongs under `workspace.docker`. Non-provisioning setup commands belong in +top-level `extensions`; reset policy stays under +`workspace.hooks.after_each.reset`; runner-specific setup belongs in the +`target` object, in `targets`, or in project config. AgentV supports two eval +data formats: YAML and JSONL. YAML is the canonical portable model. TypeScript helpers, generated fixtures, and Python scripts should lower to the same YAML/JSONL shapes rather than inventing a separate eval contract. +Eval files describe the task, target binding, and run controls. Use +`execution.max_concurrency` for authored suite concurrency. Operators can still +override concurrency with `agentv eval --workers N`; do not author legacy +`workers` fields in eval YAML. + +## Authoring Shapes + +Eval YAML is AgentV's composable and runnable authoring primitive. It is a +focused, shareable slice of the same config graph as `.agentv/config.yaml`. +Use ordinary `*.eval.yaml` files for direct task suites and for wrapper evals +that compose other suites. Raw case files are reusable data inputs, not a +second runnable experiment format. + +- A **task suite** is eval YAML that owns task context: `workspace`, shared + `input`, shared `assert`, fixtures, graders, and test cases. It can run + directly or be imported through `imports.suites`. +- A **raw case file** is a YAML, JSON, JSONL, CSV, script-backed dataset, + directory, or glob of cases. Import it with `imports.tests`, + `tests: ./cases.yaml`, `tests: file://cases.csv`, or string shorthand; parent + suite context applies because raw cases do not carry their own suite context. +- A **wrapper eval** is eval YAML that imports one or more suites with + `imports.suites` and binds run controls with top-level `target`, `threshold`, + `timeout_seconds`, and `evaluate_options`. + Wrapper evals can live anywhere in the repo. A wrapper that imports suites + with `imports.suites` must not define parent `workspace`; imported suites own + task environment. Machine-local existing workspace paths belong in CLI flags + or `config.local.yaml`, not eval YAML. + +For example, a reusable task suite can keep the task contract in one file: -## Suites +```yaml +# evals/suites/refunds.eval.yaml +suite: refunds +workspace: + repos: + - path: ./support-app + repo: acme/support-app + commit: main +input: Answer using the refund policy in the workspace. +assert: + - Applies the refund policy correctly +tests: + - id: missing-receipt + input: Can this customer get a refund without a receipt? +``` + +Raw cases are just case data: + +```yaml +# evals/cases/refund-smoke.cases.yaml +- id: damaged-item + input: The item arrived damaged. What should support do? + expected_output: Offer a replacement or refund path. +``` + +A wrapper eval stays ordinary eval YAML while choosing a target and run controls: + +```yaml +# experiments/refunds-codex.eval.yaml +name: refunds-codex +target: codex-gpt5 +execution: + max_concurrency: 3 + +evaluate_options: + repeat: + count: 2 + strategy: pass_any -An eval file is a **suite**: it binds test cases to execution context (workspace, hooks, targets, trials). Test cases can be inline or loaded from an external file via `tests: ./cases.yaml` for reuse across suites. +imports: + suites: + - path: ../evals/suites/refunds.eval.yaml + tests: + - path: ../evals/cases/refund-smoke.cases.yaml + +tests: + - id: local-edge-case + input: Can a final-sale item be refunded after damage in transit? + expected_output: Explain the final-sale exception for damaged transit. +``` + +The `experiments/` directory in that example is optional and user-owned. AgentV +does not infer behavior from the path; the wrapper runs because it is eval YAML +with tests or imports. The wrapper owns target selection and run controls. Put +workspace setup in imported child suites. Parent workspace-affecting fields, +including top-level `workspace`, are for parent-owned raw cases, including +cases imported with `imports.tests`. Runtime workspace path overrides belong in +CLI flags or `.agentv/config.local.yaml`; repos, hooks, templates, Docker +config, env checks, and workspace scope belong in top-level or case-level +`workspace`. ## YAML Format -The primary format. A single file contains metadata, execution config, and tests: +The primary format. A single file contains metadata, inline runtime config, and tests: ```yaml description: Math problem solving evaluation -execution: - target: default +target: default -assertions: - - name: correctness - type: llm-grader - prompt: ./graders/correctness.md +assert: + - Correctly calculates the answer + - Explains the calculation briefly tests: - id: addition - criteria: Correctly calculates 15 + 27 = 42 input: What is 15 + 27? expected_output: "42" ``` @@ -43,12 +134,117 @@ tests: |-------|-------------| | `description` | Human-readable description of the evaluation | | `suite` | Optional suite identifier | -| `execution` | Default execution config (`target`, `fail_on_error`, `threshold`, etc.) | -| `workspace` | Suite-level workspace config — inline object or string path to an [external workspace file](/docs/v4.42.4/guides/workspace-pool/#external-workspace-config). Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/v4.42.4/guides/workspace-architecture/#repo-provenance-vs-acquisition). | -| `tests` | Array of individual tests, or a string path to an external file or directory | -| `assertions` | Suite-level graders appended to each test unless `execution.skip_defaults: true` is set on the test | +| `category` | Optional slash-delimited analytics taxonomy path. Overrides the category derived from the eval file path. | +| `target` | System under test by configured target `id` or inline target object | +| `tags` | Optional promptfoo-style metadata map. Use `tags.experiment` as the run/result grouping label. | +| `prompts` | Optional top-level prompt matrix. Entries can be strings, chat message arrays, files, or generated prompt functions. | +| `targets` | Optional target matrix. Entries reference target ids or inline target objects. | +| `evaluate_options.repeat` | Optional repeat policy as a positive integer shorthand or object with `count`, `strategy`, `early_exit`, and `cost_limit_usd` | +| `timeout_seconds` | Optional per-case timeout | +| `evaluate_options` | Optional evaluation runtime options such as `budget_usd` and `repeat` | +| `execution.max_concurrency` | Optional general eval parallelism for this suite | +| `threshold` | Optional suite quality threshold | +| `workspace` | Suite-level task environment — inline object or string path to an external workspace file. Repo entries declare identity and checkout pins; acquisition is covered in [Workspace Architecture](/docs/guides/workspace-architecture/#repo-provenance-vs-acquisition). | +| `extensions` | Promptfoo-style lifecycle hooks: `file://path/to/hooks.mjs:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, plus the built-in `agentv:agent-rules`. Hooks run after `workspace.repos` materializes. | +| `imports` | Optional import groups. `imports.suites` imports full child eval suites with their task context. `imports.tests` imports raw test rows into this file's context. Import entries may use scoped `run:` overrides for `threshold`, `repeat`, `timeout_seconds`, and `budget_usd`. | +| `tests` | Inline raw tests or a string path to an external raw-case file or directory. Legacy `tests[].include` entries still load with a migration warning; prefer `imports.suites` or `imports.tests`. | +| `assert` | Suite-level graders appended to each test unless `execution.skip_defaults: true` is set on the test | | `input` | Suite-level input messages prepended to each test's input unless `execution.skip_defaults: true` is set on the test | +`workspace` is what the agent can inspect or modify through tools, not prompt +input. Put instructions in `input`; put repos, templates, Docker config, env +checks, scope, and repo provenance in `workspace`. Put lifecycle setup that +does not acquire repos in `extensions`. + +For historical or repo-state evals, put the checkout under +`workspace.repos[].commit`. A commit SHA in the prompt or metadata is useful +context, but it does not materialize a repo for the agent to inspect. + +### Prompts, Vars, and Target Expansion + +Use top-level `prompts` when you want promptfoo-style prompt variants. AgentV +renders each prompt with each test's `vars`, then expands the run as +`prompts x targets x tests x repeat` before execution. Each expanded row keeps +the original `test_id` plus prompt and target identity for Dashboard filtering, +reruns, and comparisons. + +```yaml +description: Release-note summarization +tags: + experiment: prompt-matrix + +prompts: + - id: direct + label: Direct + prompt: "Summarize {{ vars.topic }}." + - id: terse + label: Terse + prompt: "In one sentence, summarize {{ vars.topic }}." + +targets: + - id: local-mini + provider: openai + runtime: host + config: + model: gpt-5.4-mini + - id: local-codex + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + +tests: + - id: release-notes + vars: + topic: the July release notes + expected_output: concise release-note summary + assert: + - Identifies the most important change + - Avoids unsupported details +``` + +If `prompts` is present, put per-case data in `tests[].vars` rather than +`tests[].input`. For direct task suites, `input` remains the supported shorthand +for the target task and can be a string, object, or message array. Use +`prompts` only when you want a prompt matrix rendered from `tests[].vars`. + +### Lifecycle Extensions + +`extensions` uses Promptfoo-compatible lifecycle names. File hooks are local +JavaScript or TypeScript modules resolved relative to the eval file: + +```yaml +extensions: + - file://scripts/setup.mjs:beforeAll + - file://scripts/setup.mjs:beforeEach + - file://scripts/setup.mjs:afterEach + - file://scripts/setup.mjs:afterAll +``` + +Each exported function receives a context object with snake_case keys such as +`workspace_path`, `test_id`, `eval_run_id`, `case_input`, and `case_metadata`. +Setup hook failures (`beforeAll`, `beforeEach`) fail the affected run; teardown +hook failures (`afterEach`, `afterAll`) are non-fatal. + +`agentv:agent-rules` is the only built-in extension in this slice. It runs after +workspace materialization and exposes staged rule paths to providers and result +metadata as `agent_rules_paths`: + +```yaml +extensions: + - id: agentv:agent-rules + hook: beforeAll + skills: agent-rules/skills + hooks: agent-rules/hooks + agents: agent-rules/agents + rules: agent-rules/AGENTS.md +``` + +If `agentv:agent-rules` is authored as a string, it defaults to `beforeAll` and +discovers conventional rule locations already present in the materialized +workspace. It does not clone repositories or replace `workspace.repos`. + ### Metadata Fields You can add structured metadata to your eval file using these optional top-level fields. Metadata is parsed when the `name` field is present: @@ -79,32 +275,50 @@ tests: input: Screen "Acme Corp" against denied parties list ``` -### Suite-level Assertions +When `category` is omitted, AgentV derives it from the eval file path. Generic +filenames do not add a leaf: `security/eval.yaml` becomes `security`, and +`security/network/suite.yaml` becomes `security/network`. A meaningful +named eval file contributes a leaf, so `security/network.eval.yaml` becomes +`security/network`. Existing flat category strings remain valid one-node +category paths. + +### Suite-level Assert -The `assertions` field is the canonical way to define suite-level graders. Suite-level assertions are appended to every test's graders unless a test sets `execution.skip_defaults: true`. +The `assert` field is the canonical way to define suite-level graders. Suite-level assertions are appended to every test's graders unless a test sets `execution.skip_defaults: true`. +For semantic or agent-behavior checks, prefer plain assertion strings first; +AgentV treats them as rubric criteria. Use deterministic assertions or script +graders when the expected output is exact or requires programmatic inspection. +If the assertion strings already state the grading contract, omit a duplicate +`criteria` field on each test. Use explicit `type: llm-rubric` entries only +when you need a custom prompt, a custom grader target, or a deliberately +separate grader panel. ```yaml description: API response validation -assertions: +assert: - type: is-json required: true - type: contains value: "status" + - Correctly answers the user's question + - Explains the reasoning clearly tests: - id: health-check - criteria: Returns health status input: Check API health ``` -`assertions` supports all grader types, including deterministic assertion types (`contains`, `regex`, `is_json`, `equals`) and `rubrics`. See [Tests](/docs/v4.42.4/evaluation/eval-cases/#per-test-assertions) for per-test assertions usage. +`assert` supports rubric shorthand strings, deterministic assertion types +(`contains`, `regex`, `is-json`, `equals`), `llm-rubric`, and script +graders. See [Tests](/docs/evaluation/eval-cases/#per-test-assert) for +per-test assert usage. -### Assertion Includes +### Assert Includes -Reusable assertion sets can be factored into template files and referenced from any `assertions` array: +Reusable assertion sets can be factored into template files and referenced from any `assert` array: ```yaml -assertions: +assert: - include: safe-response - include: ./shared/format.yaml ``` @@ -117,15 +331,21 @@ Resolution rules: ### Suite-level Input -The `input` field defines messages that are **prepended** to every test's input. This avoids repeating the same prompt or system context in each test case — following the same pattern as suite-level `assertions`. +The `input` field defines messages that are **prepended** to every test's input. This avoids repeating the same prompt or system context in each test case, following the same pattern as suite-level `assert`. ```yaml description: Travel assistant evaluation -input: - - role: user - content: - - type: file - value: ./system-prompt.md +input: "Answer as a concise travel assistant." + +tests: ./cases.yaml +``` + +Use a block scalar for multi-line shared instructions: + +```yaml +input: | + Read AGENTS.md before answering. + Explain the tradeoffs clearly. tests: ./cases.yaml ``` @@ -142,9 +362,23 @@ The effective input at runtime becomes `[...suite input, ...test input]`. Suite-level `input` accepts the same formats as test-level `input`: - **String** — wrapped as `[{ role: "user", content: "..." }]` -- **Message array** — used as-is, including file references +- **Object without a top-level `role` key** — wrapped as structured user-message content +- **Single message object** — a `{ role, content }` object using a supported message role +- **Message array** — used as-is, including system messages and file references -To opt out for a specific test, set `execution.skip_defaults: true` (same flag that skips suite-level `assertions`). +The top-level `role` key is reserved for message objects. If your structured payload needs a field named `role`, nest it under another key. + +```yaml +input: + - role: system + content: You are a careful reviewer. + - role: user + content: + - type: file + value: ./system-prompt.md +``` + +To opt out for a specific test, set `execution.skip_defaults: true` (same flag that skips suite-level `assert`). ### Suite-level Input Files @@ -169,21 +403,94 @@ Each test's effective input becomes a single user message with `[file blocks..., Per-test `input_files` overrides the suite-level value (it does not merge). To opt out, set `execution.skip_defaults: true` on the test. -### Tests as String Path +### PROMPT.md Fallback + +For directory-style evals, a test may omit `input` and keep the task prompt in +Markdown instead. AgentV resolves the prompt in this order: + +1. If the effective `input_files` contains a file named exactly `PROMPT.md`, that file becomes the test prompt. +2. Otherwise, if a `PROMPT.md` exists beside the `EVAL.yaml`, that file becomes the test prompt. +3. Other `input_files` remain attachments. `PROMPT.md` is removed from the attachment list so the prompt is not duplicated. -Instead of inlining tests in the same file, you can point `tests` to an external YAML or JSONL file. This is the inverse of the sidecar pattern — the metadata file references the test data: +```text +agent-001-fix-bug/ + EVAL.yaml + PROMPT.md + fixtures/ + failing-test.log +``` + +```yaml +tests: + - id: fix-bug + criteria: Fixes the regression described in the prompt + input_files: + - ./fixtures/failing-test.log +``` + +Use explicit `input` when the prompt is short or generated from YAML variables. +Use `PROMPT.md` when the task text is long enough that duplicating it inside +YAML would make the eval hard to review. + +### Raw Cases as String Paths + +Instead of inlining tests in the same file, you can point `tests` to an external YAML or JSONL file of raw cases. This is the inverse of the sidecar pattern — the metadata file references the test data: ```yaml name: my-eval description: My evaluation suite -execution: - target: default +target: default tests: ./cases.yaml ``` -The path is resolved relative to the eval file's directory. The external file should contain a YAML array of test objects or a JSONL file with one test per line. +The path is resolved relative to the eval file's directory. The external raw +case file can be a YAML or JSON array of test objects, a JSONL file with one +test per line, a promptfoo-compatible CSV file, or an explicit JavaScript or +Python dataset function such as `file://generate-tests.mjs:createTests` or +`file://generate_tests.py:create_tests`. String entries inside a `tests:` list +work the same way and may use direct paths, `file://` paths, directories, or +globs: + +```yaml +tests: + - ./cases/*.cases.yaml +``` + +CSV datasets support promptfoo-style magic columns. `__expected` and +`__expectedN` create AgentV assertions using the supported expected-column +mini-DSL (`contains:*`, `icontains:*`, `contains-any:*`, `contains-all:*`, +`icontains-any:*`, `icontains-all:*`, `starts-with:*`, `ends-with:*`, +`regex:*`, `equals:*`, `is-json`, `latency()`, `cost()`, +`grade:*`, `llm-rubric:*`, `javascript:*`, `fn:*`, `eval:*`, `python:*`, and +`file://*.py`; file paths inside CSV cells are resolved relative to the CSV +file). Unsupported promptfoo assertion forms such as `similar:*` are rejected +during validation instead of being skipped at runtime. +`__provider_output` becomes first-class `expected_output` reference data, +`__metric` names the generated assertions, `__threshold` sets the test threshold, +`__metadata:` adds metadata, and `__config:__expectedN:threshold` sets an +assertion `min_score`. Ordinary columns become `vars`, so CSV rows can rely on +suite-level `input` that interpolates those variables. + +String shorthand is raw-case-only. Import reusable task suites through +`imports.suites`; use `imports.tests` when you want to drop suite context and +import only raw cases into the parent context: -### Tests as Directory Path +```yaml +imports: + suites: + - path: ./suites/*.eval.yaml + tests: + - path: ./cases/regression.jsonl + +tests: + - id: local-edge-case + input: ... +``` + +Legacy `tests[].include` entries still load with a migration warning for older +eval files, but new evals should use `imports.suites` or `imports.tests`. + +### Raw Cases as Directory Paths When `tests` points to a directory, AgentV auto-discovers test cases from subdirectories. Each subdirectory containing a `case.yaml` (or `case.yml`) becomes a test case: @@ -219,34 +526,34 @@ input: Fix the null check bug in parser.ts - **Alphabetical ordering:** Subdirectories are sorted alphabetically for deterministic order - **Per-case workspace:** A `workspace/` subdirectory inside the case directory automatically sets `workspace.template` to that path, unless the case already defines a `workspace` field - **Skipped directories:** Subdirectories without `case.yaml` are skipped with a warning -- **Suite-level config applies:** Suite-level `assertions`, `input`, `workspace`, and `execution` still apply to directory-discovered cases +- **Suite-level config applies:** Suite-level `assert`, `input`, `workspace`, `target`, and top-level run controls still apply to directory-discovered cases This pattern is useful for benchmarks with many cases, where each case benefits from its own directory for workspace templates, supporting files, or documentation. For guidance on keeping provenance metadata, patches, oracle files, and generated -dataset rows out of oversized inline YAML, see [Benchmark Provenance](/docs/v4.42.4/guides/benchmark-provenance/). +dataset rows out of oversized inline YAML, see [Benchmark Provenance](/docs/guides/benchmark-provenance/). ## Environment Variable Interpolation -All string fields in eval files support `${{ VAR }}` syntax for environment variable interpolation. This enables portable eval configs that work across machines and CI environments without hardcoded paths. +All string fields in eval files support `{{ env.VAR }}` syntax for environment variable interpolation. This enables portable eval configs that work across machines and CI environments without hardcoded paths. ```yaml workspace: repos: - path: ./RepoA - repo: "${{ REPO_A_URL }}" - commit: "${{ REPO_A_COMMIT }}" + repo: "{{ env.REPO_A_URL }}" + commit: "{{ env.REPO_A_COMMIT }}" tests: - id: test-1 - input: "Evaluate the code in ${{ PROJECT_NAME }}" - criteria: "${{ EVAL_CRITERIA }}" + input: "Evaluate the code in {{ env.PROJECT_NAME }}" + criteria: "{{ env.EVAL_CRITERIA }}" ``` ### Behavior -- **Syntax:** `${{ VARIABLE_NAME }}` with optional whitespace around the name +- **Syntax:** `{{ env.VARIABLE_NAME }}` with optional whitespace around the name - **Missing variables** resolve to an empty string -- **Partial interpolation** is supported: `${{ HOME }}/repos/${{ PROJECT }}` becomes `/home/user/repos/myproject` +- **Partial interpolation** is supported: `{{ env.HOME }}/repos/{{ env.PROJECT }}` becomes `/home/user/repos/myproject` - **Non-string values** (numbers, booleans) are not affected - Interpolation is applied recursively to all nested objects and arrays - Works in YAML eval files, external YAML/JSONL case files, and external workspace config files @@ -258,8 +565,8 @@ tests: # workspace.yaml — works on any machine repos: - path: ./my-repo - repo: "${{ MY_REPO_URL }}" - commit: "${{ MY_REPO_COMMIT }}" + repo: "{{ env.MY_REPO_URL }}" + commit: "{{ env.MY_REPO_COMMIT }}" ``` ```bash @@ -270,31 +577,31 @@ MY_REPO_COMMIT=main ## Per-Test Template Variables -Eval YAML also supports per-test `vars` for data-driven prompt templates. Use `{{name}}` placeholders in test-facing text fields, and AgentV resolves them when the suite loads. +Eval YAML also supports per-test `vars` for data-driven prompt templates. Use `{{ vars.name }}` placeholders in test-facing text fields, and AgentV resolves them when the suite loads. ```yaml -input: "Answer clearly: {{question}}" +input: "Answer clearly: {{ vars.question }}" tests: - id: capital vars: question: What is the capital of France? expected_answer: Paris - criteria: "Answers {{question}} correctly" + criteria: "Answers {{ vars.question }} correctly" input: - role: user - content: "Question: {{question}}" - expected_output: "{{expected_answer}}" + content: "Question: {{ vars.question }}" + expected_output: "{{ vars.expected_answer }}" ``` ### Behavior - `vars` is defined per test as an object -- `{{name}}` and dotted paths like `{{ user.name }}` are supported -- Substitution applies to suite-level `input`, test `input`, `input_files`, `criteria`, `expected_output`, and conversation turn `input` / `expected_output` +- `{{ vars.name }}` and dotted paths like `{{ vars.user.name }}` are supported +- Substitution applies to suite-level `input`, test `input`, `input_files`, `criteria`, `expected_output`, assertion values/metrics, and conversation turn `input` / `expected_output` / assertions - When the whole string is a single placeholder, the original JSON value is preserved -- Missing variables are left unchanged, so unrelated template syntax is not silently blanked out -- `vars` interpolation is separate from environment interpolation: `{{question}}` uses test data, `${{ PROJECT_NAME }}` uses environment variables +- Missing variables render as empty strings following Nunjucks semantics +- `vars` interpolation is separate from environment interpolation: `{{ vars.question }}` uses test data, `{{ env.PROJECT_NAME }}` uses environment variables ## JSONL Format @@ -314,11 +621,10 @@ An optional YAML sidecar file provides metadata and execution config. Place it a ```yaml description: Math evaluation dataset suite: math-tests -execution: - target: azure-base -assertions: +target: azure-base +assert: - name: correctness - type: llm-grader + type: llm-rubric prompt: ./graders/correctness.md ``` diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx index 1fe98c7f0..8148f884c 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx @@ -3,9 +3,6 @@ title: Running Evaluations description: CLI commands for running and managing evaluations sidebar: order: 4 -slug: docs/v4.42.4/evaluation/running-evals -editUrl: false -pagefind: false --- ## Run an Evaluation @@ -14,7 +11,14 @@ pagefind: false agentv eval evals/my-eval.yaml ``` -Results are written to `.agentv/results/runs//index.jsonl`. Each line is a JSON object with one result per test case, and the run workspace also stores the manifest and related artifacts. Use this generated run folder as the portable audit surface: copy or sync the run directory, not a hand-authored parallel bundle. +Results are written to `.agentv/results//.internal/index.jsonl`. Each CLI +invocation writes one run bundle. The experiment label is stored in +`summary.json` and row metadata. Each line is a JSON object with one result per +test case, and the run workspace also stores the summary and related artifacts. +Use this generated run folder as the portable audit surface: copy or sync the +run directory, not a hand-authored parallel bundle. See the +[Result Artifact Contract](/docs/reference/result-artifacts/) for the complete +run layout and reader rules. Each `scores[]` entry includes per-grader timing: @@ -23,7 +27,7 @@ Each `scores[]` entry includes per-grader timing: "scores": [ { "name": "format_structure", - "type": "llm-grader", + "type": "llm-rubric", "score": 0.9, "verdict": "pass", "assertions": [ @@ -38,7 +42,7 @@ Each `scores[]` entry includes per-grader timing: } ``` -The `duration_ms`, `started_at`, and `ended_at` fields are present on every grader result (including `code-grader`), enabling per-grader bottleneck analysis. +The `duration_ms`, `started_at`, and `ended_at` fields are present on every grader result (including `script`), enabling per-grader bottleneck analysis. ## Common Options @@ -52,14 +56,18 @@ agentv eval --target my-target evals/**/*.yaml ### Experiment Label -Tag a pipeline run with an experiment name to track different conditions (e.g. with vs without skills): +Tag a run with an experiment name to track different conditions (e.g. with vs without skills): ```bash -agentv pipeline run evals/my-eval.yaml --experiment with_skills -agentv pipeline run evals/my-eval.yaml --experiment without_skills +agentv eval evals/my-eval.yaml --experiment with_skills +agentv eval evals/my-eval.yaml --experiment without_skills ``` -The experiment label is written to `manifest.json` and propagated to each entry in `index.jsonl` by `pipeline bench`. The eval file stays the same across experiments — what changes is the environment. Dashboards can filter and compare results by experiment. +The experiment label chooses the result bucket and is propagated to each entry +in `index.jsonl`. CLI `--experiment` wins over `experiment.name` in the eval +file. If neither is set, AgentV writes to the `default` bucket. The eval file +stays the same across experiments; what changes is the runtime condition. +Dashboards can filter and compare results by experiment. ### Run Specific Test @@ -69,41 +77,49 @@ Run a single test by ID: agentv eval --test-id case-123 evals/my-eval.yaml ``` -### Dry Run +### Validate Without Running -Test the harness flow with mock responses (does not call real providers): +Use `agentv validate` when you want a cheap schema and config check without +executing targets or graders: ```bash -agentv eval --dry-run evals/my-eval.yaml +agentv validate evals/my-eval.yaml ``` :::note -Dry-run returns mock responses that don't match grader output schemas. Use it only for testing harness flow, not grader logic. +Eval execution no longer has a `--dry-run` mock-target mode. That mode produced +normal quality failures against fake candidate answers, which made cheap +validation look like a grader or agent result. For no-live-LLM quality +validation, run the eval against an oracle/reference target or a replayed/frozen +transcript so graders see real candidate output. Dry-run preview flags on other +commands, such as `agentv results export --dry-run` and import preview flows, +are unchanged. ::: ### Custom Output Directory -Write all artifacts (index.jsonl, benchmark.json, per-test grading/timing) to a specific directory: +Write all artifacts (`.internal/index.jsonl`, `summary.json`, per-test grading, metrics, and transcripts) to a specific directory: ```bash agentv eval evals/my-eval.yaml --output ./my-results ``` `--output` is a run directory, not a file path. The canonical manifest is always -`/index.jsonl`. +`/.internal/index.jsonl`; the aggregate summary is +`/summary.json`. -### Read Results from the Run Index +### Read Results from the Run Manifest -The run directory is the complete artifact boundary. Use `/index.jsonl` for scripts, CI summaries, and downstream tools: +The run directory is the complete artifact boundary. Use `/.internal/index.jsonl` for scripts, CI summaries, and downstream tools: ```bash agentv eval evals/my-eval.yaml --output ./my-results -cat ./my-results/index.jsonl +cat ./my-results/.internal/index.jsonl ``` -### Generated Task Bundles +### Generated Test Bundles -Each result can also include a generated task bundle inside its per-test artifact +Each result can also include a generated test bundle inside its per-test result directory. The bundle captures the eval slice and target settings that produced that row, so reviewers and rerun tooling can inspect the exact run-local source instead of relying on a mutable checkout. @@ -112,14 +128,20 @@ Typical layout: ```text my-results/ - index.jsonl - benchmark.json + summary.json + .internal/ + index.jsonl / - grading.json - timing.json - input.md - outputs/response.md - task/ + summary.json + sample-1/ + result.json + grading.json + metrics.json + transcript.json + transcript-raw.jsonl + outputs/answer.md + outputs/file_changes.diff # when workspace changes are captured + test/ EVAL.yaml targets.yaml files/ # copied input files when the case references them @@ -127,13 +149,24 @@ my-results/ ``` The `index.jsonl` row links to these generated paths with snake_case fields such -as `artifact_dir`, `task_dir`, `eval_path`, `targets_path`, `files_path`, and -`graders_path`. Treat those paths as relative to the run directory. When you need -a portable artifact for audit, review, Dashboard inspection, or rerun workflows, -share the generated run directory and its `index.jsonl` manifest. Source-side -case directories are still useful for organizing bulky prompts, fixtures, or -tests while authoring an eval, but they are optional input organization rather -than a separate artifact schema. +as `result_dir`, `test_dir`, `eval_path`, `targets_path`, `files_path`, +`file_changes_path`, and `graders_path`. Treat those paths as relative to the +run directory. When you need a portable artifact for audit, review, Dashboard +inspection, or rerun workflows, share the generated run directory and its +`index.jsonl` manifest. Source-side case directories are still useful for +organizing bulky prompts, fixtures, or tests while authoring an eval, but they +are optional input organization rather than a separate artifact schema. + +For the full root layout, per-attempt sidecars, pointer rules, and integration +guidance, use the [Result Artifact Contract](/docs/reference/result-artifacts/). + +Use repo-relative `eval_path`, `test_id`, and `target` as the source identity +for a result row. `suite` and `name` are display metadata only; do not use them +to infer storage paths or pick a Dashboard detail row. + +If the source eval uses the `PROMPT.md` fallback instead of inline `input`, +AgentV records the generated test bundle metadata when source artifacts are +available. It no longer emits a generated prompt sidecar for result rows. ### Manual or External-Agent Attempts @@ -141,7 +174,7 @@ Use `agentv prepare` when you want AgentV to set up one eval case but a human, external agent, or separate harness should perform the work. The workflow is: prepare the workspace and prompt, run the external attempt in that workspace, then grade the final state with `agentv grade --prepared` without rerunning the -target provider. See [Prepare](/docs/v4.42.4/tools/prepare/) for the full workflow, +target provider. See [Prepare](/docs/tools/prepare/) for the full workflow, manifest shape, and optional trace/session input with `--trace`. ### Trace Correlation @@ -154,10 +187,10 @@ OpenTelemetry/OpenInference spans directly to that backend during execution. ```bash # Summary-level inspection from the run manifest -agentv inspect stats .agentv/results/runs//index.jsonl +agentv inspect stats .agentv/results//.internal/index.jsonl # Inspect AgentV-owned per-case artifacts and transcript sidecars -agentv inspect show .agentv/results/runs//index.jsonl --tree +agentv inspect show .agentv/results//index.jsonl --tree ``` `index.jsonl` contains aggregate metrics such as score, latency, cost, token @@ -170,7 +203,7 @@ primary observability path. ### Parallelism -The `--workers N` flag controls how many **test cases run in parallel within each eval file** (default: 3). Eval files always run sequentially — one file completes before the next starts. +The `--workers N` flag controls the in-process worker pool for a single eval file (default: 3). Eval files always run sequentially — one file completes before the next starts. In target-matrix runs, selected targets share that worker budget instead of each target creating its own full pool. ```bash agentv eval evals/my-eval.yaml --workers 4 @@ -178,46 +211,46 @@ agentv eval evals/my-eval.yaml --workers 4 agentv eval evals/file1.yaml evals/file2.yaml evals/file3.yaml --workers 3 # Files run one at a time; within each file, up to 3 test cases run in parallel + +agentv eval evals/my-eval.yaml --target gpt --target claude --workers 4 +# The target matrix shares the same 4-worker budget ``` This matches the standard model used by eval frameworks (promptfoo, deepeval, OpenAI Evals) and avoids cross-file workspace races without any special configuration. -### Workspace Modes and Finish Policy +### Workspace Path and Finish Policy -Use workspace mode and finish policies instead of multiple conflicting booleans: +Use `--workspace-path` only when a run should use an existing machine-local +directory as-is. Use finish policies to control cleanup for harness-managed temp +workspaces: ```bash -# Mode: pooled | temp | static -agentv eval evals/my-eval.yaml --workspace-mode pooled - -# Static mode path -agentv eval evals/my-eval.yaml --workspace-mode static --workspace-path /path/to/workspace - -# Pooled reset policy override: standard | full (CLI override) -agentv eval evals/my-eval.yaml --workspace-clean full +# Existing local workspace path for this run +agentv eval evals/my-eval.yaml --workspace-path /path/to/workspace # Finish policy overrides: keep | cleanup (CLI) agentv eval evals/my-eval.yaml --retain-on-success cleanup --retain-on-failure keep ``` -Equivalent eval YAML: +Portable eval YAML keeps workspace intent under templates, repos, env, Docker, +and folder isolation. Use top-level extensions for executable setup: ```yaml +extensions: + - file://scripts/setup.mjs:beforeAll + workspace: - mode: pooled # pooled | temp | static - path: null # workspace path for mode=static; auto-materialised when empty/missing + scope: suite # suite | attempt hooks: - enabled: true # set false to skip all hooks after_each: reset: fast # none | fast | strict ``` Notes: -- Pooling is default for shared workspaces with repos when mode is not specified. -- `mode: static` (or `--workspace-mode static`) uses `path` / `--workspace-path`. When the path is empty or missing, the workspace is auto-materialised (template copied + repos cloned). Populated directories are reused as-is. -- Static mode is incompatible with `isolation: per_test`. -- `hooks.enabled: false` skips all lifecycle hooks (setup, teardown, reset). -- Pool slots are managed separately (`agentv workspace list|clean`). +- Temp workspace materialization is the default for harness-managed workspaces with repos. +- `--workspace-path` uses an existing machine-local directory as-is. +- Runtime workspace paths are incompatible with `workspace.scope: attempt`. +- `workspace.hooks.after_each.reset` resets file state after each case. ### Resume an Interrupted Run @@ -226,45 +259,28 @@ AgentV ships three flags for picking up a partial run. They differ only in **whi | Flag | What it skips | What it re-runs | Use when | |------|---------------|-----------------|----------| | `--resume` | Anything that finished without an `execution_error` (passes, fails, threshold misses) | Errors and missing cases | The run was interrupted (Ctrl-C, crash, OOM) and you just want it to finish | -| `--rerun-failed` | Only cases with `executionStatus === 'ok'` | Errors **and** test failures (assertion misses, threshold misses) | A grader change or model swap means you want to re-grade everything that wasn't already passing | +| `--rerun-failed ` | Only cases with `executionStatus === 'ok'` | Errors **and** test failures (assertion misses, threshold misses) | A grader change or model swap means you want to re-grade everything that wasn't already passing | | `--retry-errors ` | Anything that completed without an `execution_error` (same set as `--resume`) | Errors and missing cases | You want to point at an arbitrary prior run/manifest by path, instead of resuming the run dir you're currently writing to | -`--resume` and `--rerun-failed` both append to the existing `index.jsonl`. When `--output ` is given they target that directory; when omitted they default to the **last run dir for the current cwd**, recorded in `.agentv/cache.json` and updated after every eval. This matches promptfoo's `--resume [evalId]` and OpenCompass's `-r [timestamp]` "latest by default" convention. `--retry-errors` takes the prior run's path directly (a directory or an `index.jsonl`). +`--resume` appends to the existing `index.jsonl` in `--output `; when omitted it defaults to the **last run dir for the current cwd**, recorded in `.agentv/cache.json` and updated after every eval. `--rerun-failed ` reads a specific canonical run bundle from `.agentv/results/` and, when `--output` is omitted, appends replacement rows to that same bundle. You can also pass a run workspace path or `index.jsonl` path instead of a bare run ID. `--retry-errors` takes the prior run's path directly and re-runs only execution errors or missing cases. ```bash # Resume the last run — no args needed; AgentV finds it from .agentv/cache.json agentv eval evals/my-eval.yaml --resume # Or target a specific run dir explicitly -agentv eval evals/my-eval.yaml --output .agentv/results/runs/ --resume +agentv eval evals/my-eval.yaml --output .agentv/results/ --resume -# Re-run errors AND failed cases against the last run dir -agentv eval evals/my-eval.yaml --rerun-failed +# Re-run errors AND failed cases from a specific canonical run +agentv eval evals/my-eval.yaml --rerun-failed # Re-run only execution errors from any prior run by path -agentv eval evals/my-eval.yaml --retry-errors .agentv/results/runs//index.jsonl -``` - -After any failing run, the CLI prints the exact `--rerun-failed` command for the run dir that just completed — copy/paste it. If the process or pod disappeared before you could access the local run directory and results auto-push was enabled, recover the partial run from [WIP checkpoints](/docs/v4.42.4/tools/wip-checkpoints/) first, then use the same `--resume` flow. - -The interactive wizard (`agentv eval` with no arguments) remembers the last run's artifact directory and surfaces a **"Resume last run"** entry in the main menu when one exists. - -### Execution Error Tolerance - -Control whether the eval run halts on execution errors using `execution.fail_on_error` in the eval YAML: - -```yaml -execution: - fail_on_error: false # never halt on errors (default) - # fail_on_error: true # halt on first execution error +agentv eval evals/my-eval.yaml --retry-errors .agentv/results//.internal/index.jsonl ``` -| Value | Behavior | -|-------|----------| -| `true` | Halt immediately on first execution error | -| `false` | Continue despite errors (default) | +After any failing run, the CLI prints the exact `--rerun-failed` command for the run dir that just completed — copy/paste it. If the process or pod disappeared before you could access the local run directory and results auto-push was enabled, recover the partial run from [WIP checkpoints](/docs/tools/wip-checkpoints/) first, then use the same `--resume` flow. -When halted, remaining tests are recorded with `failureReasonCode: 'error_threshold_exceeded'`. With concurrency > 1, a few additional tests may complete before halting takes effect. +The interactive wizard (`agentv eval` with no arguments) remembers the last run directory and surfaces a **"Resume last run"** entry in the main menu when one exists. ### Suite-Level Quality Threshold @@ -279,8 +295,7 @@ agentv eval evals/ --threshold 0.8 **YAML config:** ```yaml -execution: - threshold: 0.8 +threshold: 0.8 ``` The CLI `--threshold` flag overrides the YAML value. The threshold is a number between 0 and 1 (default: 0.8). Execution errors are excluded from the count. @@ -301,9 +316,14 @@ Check eval files for schema errors without executing: agentv validate evals/my-eval.yaml ``` +Validation catches schema, target-reference, and grader configuration problems. +It does not produce quality scores. To validate grader quality behavior without +calling a live agent, use a reference target, imported transcript, or replay +fixture so AgentV still runs graders against real or frozen candidate output. + ## Run a Single Assertion -Run a code-grader assertion in isolation without executing a full eval suite: +Run a script assertion in isolation without executing a full eval suite: ```bash agentv eval assert --agent-output --agent-input @@ -325,7 +345,7 @@ The `--file` option reads a JSON file with `{ "output": "...", "input": "..." }` **Exit codes:** 0 if score >= 0.5 (pass), 1 if score < 0.5 (fail). -This is the same interface that agent-orchestrated evals use — the EVAL.yaml transpiler emits `assertions` instructions for code graders so external grading agents can execute them directly. +This is the same interface that agent-orchestrated evals use — the EVAL.yaml transpiler emits `assertions` instructions for script graders so external grading agents can execute them directly. ## Offline Grading @@ -340,31 +360,61 @@ agentv import claude --session-id agentv eval evals/my-eval.yaml --transcript .agentv/transcripts/claude-.jsonl ``` -See the [Import tool docs](/docs/v4.42.4/tools/import/) for all providers and options. - -## Transcript And Trace Artifacts - -Each result row's `artifact_dir` can include both `outputs/trace.json` and -`outputs/transcript.jsonl`. The run root does not contain a mixed transcript -artifact; use each index row's `transcript_path` to find the per-result -transcript. - -`outputs/trace.json` is the full-fidelity `agentv.trace.v1` sidecar. -It stores the canonical span graph, source metadata, capture/redaction policy, -conversion warnings, score provenance, and opaque evidence references. - -`outputs/transcript.jsonl` is a derived compatibility artifact for reading and -replay. It uses provider-neutral `agentv.transcript.v1` rows with stable -top-level fields for message order, role/content, tool calls and paired results, -timing, token usage, cost, source metadata, capture state, and trace pointers. +See the [Import tool docs](/docs/tools/import/) for all providers and options. + +## Transcript And Result Artifacts + +Each result row's `result_dir` is an allocated folder under the timestamped run +bundle, usually with a readable test-id prefix plus a short hash suffix. It can +include `transcript.json`, `transcript-raw.jsonl`, `grading.json`, +``metrics.json`, and generated outputs under `outputs/`. The run +root does not contain target, model, or `cases/` folders, and it does not contain +a mixed transcript artifact; use each index row's `transcript_path` to find the +per-result transcript. + +Rows also include `artifact_pointers` for AgentV-owned artifact storage. Pointer +entries such as `artifact_pointers.transcript` carry the storage `ref`, artifact +`key`, canonical run-relative `path`, `object_version`, `sha256`, `size`, +`schema_version`, and `media_type` so viewers and exports can migrate from git +refs to object storage without changing the run record contract. + +When automatic remote publishing sees pointers whose `ref` is +`agentv/artifacts/v1`, it also pushes those payload bytes to the +`agentv/artifacts/v1` branch in the same results remote at +`runs//` and rewrites the published pointer `key` to +that backend object key. The configured results branch is the metadata/control +plane for `index.jsonl`, `summary.json`, tags, and pointers; it does not +duplicate canonical transcript payload bodies when those rows name +`agentv/artifacts/v1`. Dashboard resolves the published pointers lazily when a +transcript view requests the payload. AgentV keeps this explicit pointer/backend +contract instead of using Git LFS as the core abstraction so S3, B2, or other +object stores can use the same `key`, `object_version`, `sha256`, `size`, +`media_type`, and `schema_version` fields later. + +AgentV does not persist a public `trace.json` sidecar in run bundles. Use +`external_trace` metadata for link-out correlation when another observability +system already owns spans. + +`transcript.json` is the canonical AgentV transcript/timeline artifact. +It uses provider-neutral `agentv.normalized_transcript.v1` data with stable +fields for message order, role/content, canonical `tool_name` values, paired +tool results, and `transcript_summary`. Provider-native payloads can appear only inside opaque nested fields such as `metadata`, `source.metadata`, tool `input`, or tool `output`. +When an agent provider captures a native stream or session log, AgentV writes +that byte-for-byte evidence to `transcript-raw.jsonl` and records it with +`transcript_raw_path`. New eval runs do not also copy the same stream to +`provider.log`; `raw_provider_log_path` is only a legacy/imported pointer when +older bundles or external sources already provide one. AgentV does not write or +maintain a parallel `outputs/transcript.json` source of truth. + Use the transcript when you need a compact portable message/event projection over the trace, including exports to role/content arrays for chat-template or Hugging Face-style workflows. Use the trace when you need full lifecycle, span, -raw evidence, redaction, or adapter conversion details. The transcript is not a -second canonical trace source and is not a provider-native Pi session dump. +raw evidence pointers, redaction, or adapter conversion details. The transcript +is not a second canonical trace source and is not a provider-native Pi session +dump. Older transcript rows without `schema_version`, `capture`, or `trace` remain accepted for replay. @@ -399,22 +449,110 @@ agentv eval --strict evals/my-eval.yaml ## Config File Defaults -Set default execution options so you don't have to pass them on every CLI invocation. Project-local `.agentv/config.yaml`, home/global `$AGENTV_HOME/config.yaml` (or `~/.agentv/config.yaml`), and `agentv.config.ts` are supported. +Set default execution options so you don't have to pass them on every CLI invocation. Project-local `.agentv/config.yaml`, project-local `.agentv/config.local.yaml`, home/global `$AGENTV_HOME/config.yaml` plus `$AGENTV_HOME/config.local.yaml` (or `~/.agentv/...`), and `agentv.config.ts` are supported. + +Project-local YAML config takes precedence over home/global YAML config. AgentV uses the first config directory it finds; it does not merge project and global YAML directories. + +Within one config directory, AgentV reads `config.yaml` first and `config.local.yaml` second. The local overlay wins: plain objects deep-merge, arrays replace, and scalar values from `config.local.yaml` override `config.yaml`. -Project-local YAML config takes precedence over home/global YAML config. AgentV uses the first config file it finds; it does not merge project and global YAML files. +Use `config.yaml` for portable defaults and shared eval-definition fields that can be committed with the eval project. Use `config.local.yaml` for machine-local overrides such as private paths, local result remotes, Dashboard project registry entries, or temporary execution defaults. Project-local `config.local.yaml` is gitignored by default. -### YAML config (`.agentv/config.yaml` or `$AGENTV_HOME/config.yaml`) +### YAML config (`config.yaml` plus optional `config.local.yaml`) + +Project config and eval YAML share the same eval-definition graph for targets, +graders, tests, defaults, and execution policy. Small projects can keep that +graph inline: + +```yaml +targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini + +tests: + - id: smoke + input: Fix the failing test + +defaults: + target: codex-local + grader: openai-grader + +execution: + max_concurrency: 3 +``` + +Larger projects can decompose any supported top-level field with a direct +`file://...` reference. The referenced file contains that field's value +directly: + +```yaml +targets: file://targets.yaml +graders: file://graders.yaml +tests: file://tests.yaml +defaults: file://defaults.yaml +execution: file://execution.yaml +``` + +```yaml +# .agentv/targets.yaml +- id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] +``` + +```yaml +# .agentv/defaults.yaml +target: codex-local +grader: openai-grader +``` + +Do not wrap referenced field files in another object. For example, +`targets: file://targets.yaml` expects `targets.yaml` to contain a bare array, +not `{ targets: [...] }`. + +`execution.max_concurrency` is AgentV's general eval parallelism field for this +config graph. It is AgentV's run-policy shape, aligned with the general +max-concurrency concept in eval runners, not a copied Promptfoo YAML path. + +Other project defaults can live beside the graph: ```yaml execution: verbose: true keep_workspaces: false +refs: + global-default: file://{{ env.AGENTV_REPO_ROOT }}/.agentv/default-test.yaml +``` + +Example local overlay: + +```yaml +execution: + keep_workspaces: true + # Machine-local existing workspace binding. Do not commit this file. + workspace_path: /home/user/workspaces/my-eval +eval_patterns: + - "local-evals/**/*.eval.yaml" ``` | Field | CLI equivalent | Type | Default | Description | |-------|---------------|------|---------|-------------| | `verbose` | `--verbose` | boolean | `false` | Enable verbose logging | +| `max_concurrency` | `--workers` | integer | none | Default eval parallelism for the composable config graph | | `keep_workspaces` | `--keep-workspaces` | boolean | `false` | Always keep temp workspaces after eval | +| `workspace_path` | `--workspace-path` | string | none | Machine-local existing workspace directory | +| `refs` | none | object | none | Project-defined named references for fields that support `ref://name`, such as shared `default_test` files | ### TypeScript config (`agentv.config.ts`) @@ -429,7 +567,7 @@ export default defineConfig({ }); ``` -**Precedence:** CLI flags > project-local `.agentv/config.yaml` > home/global `$AGENTV_HOME/config.yaml` (or `~/.agentv/config.yaml`) > `agentv.config.ts` > built-in defaults. +**Precedence:** CLI flags > project-local `.agentv/config.local.yaml` over `.agentv/config.yaml` > home/global `$AGENTV_HOME/config.local.yaml` over `$AGENTV_HOME/config.yaml` (or `~/.agentv/...`) > `agentv.config.ts` > built-in defaults. ## Response Cache @@ -440,14 +578,6 @@ agentv eval evals/suite.yaml --cache agentv eval evals/suite.yaml --cache-path .agentv/response-cache ``` -Eval YAML can enable the same cache per suite: - -```yaml -execution: - cache: true - cache_path: .agentv/response-cache -``` - Project TypeScript config can set the project default: ```typescript @@ -461,7 +591,7 @@ export default defineConfig({ }); ``` -`--no-cache` disables response caching regardless of CLI, eval YAML, or TypeScript config. Cache path precedence is `--cache-path` > eval YAML `execution.cache_path` > TypeScript config `cache.path` > `.agentv/cache`. +`--no-cache` disables response caching regardless of CLI or TypeScript config. Cache path precedence is `--cache-path` > TypeScript config `cache.path` > `.agentv/cache`. Response cache and replay are separate concepts. The response cache is an iteration aid for repeated live provider calls. Transcript or fixture replay is target substitution from curated artifacts, and graders still run fresh against the replayed output. @@ -490,9 +620,11 @@ targets: - id: replay_coding_agent provider: replay - fixtures: ../fixtures/legal-review-target-output.jsonl - source_target: live_coding_agent - suite: legal-review + runtime: host + config: + fixtures: ../fixtures/legal-review-target-output.jsonl + source_target: live_coding_agent + suite: legal-review ``` Run the same eval against the replay alias: @@ -509,7 +641,7 @@ The replay provider never invokes the live target. It only returns the recorded ### AGENTV_HOME -Override AgentV's lightweight home/config directory. This directory stores files such as `config.yaml`, `version-check.json`, `last-config.json`, and managed helper binaries. Registered Dashboard projects live under `projects:` in this home `config.yaml`. +Override AgentV's lightweight home/config directory. This directory stores files such as `config.yaml`, `config.local.yaml`, `version-check.json`, `last-config.json`, and managed helper binaries. Registered Dashboard projects live under `projects:` in the home config pair. ```bash # Linux/macOS @@ -524,9 +656,41 @@ set AGENTV_HOME=D:\agentv-config When unset, AgentV uses `~/.agentv`. +For local workspaces, put portable registry defaults in `$AGENTV_HOME/config.yaml` and machine-local project paths or result remotes in `$AGENTV_HOME/config.local.yaml`: + +```yaml +projects: + - id: agentv + path: /home/user/projects/agentv + results: + path: /home/user/agentv-results + branch: agentv/results/v1 +``` + +For larger registries, `$AGENTV_HOME/config.yaml` can point `projects` at a +bare project array: + +```yaml +projects: file://projects.yaml +``` + +```yaml +# $AGENTV_HOME/projects.yaml +- id: agentv + path: /home/user/projects/agentv +``` + +When running AgentV from a worktree that needs environment from a primary checkout, load the primary `.env` through the runtime instead of shell-sourcing it: + +```bash +bun --env-file /home/user/projects/agentv/.env apps/cli/src/cli.ts eval evals/smoke.eval.yaml +``` + +This keeps `.env` parsing in Bun's dotenv loader and avoids executing shell syntax from an environment file. + ### AGENTV_DATA_DIR -Override the heavy runtime data directory for workspaces, workspace pool, subagents, trace state, git caches, downloaded dependencies, and results repository clones. If `AGENTV_DATA_DIR` is unset, AgentV stores heavy data in `AGENTV_HOME` (or `~/.agentv`) for backward compatibility. +Override the heavy runtime data directory for workspaces, subagents, trace state, git caches, downloaded dependencies, and results repository clones. If `AGENTV_DATA_DIR` is unset, AgentV stores heavy data in `AGENTV_HOME` (or `~/.agentv`) for backward compatibility. ```bash # Linux/macOS diff --git a/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx b/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx index 91c8d68ea..b05ef7d25 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/graders/llm-graders.mdx @@ -109,7 +109,8 @@ tests: ## Per-Grader Target -By default, an `llm-grader` uses the suite target's `grader_target`. Override it per grader when you need multiple grader models in one run: +By default, an `llm-grader` uses `defaults.grader` from the resolved config graph. +Override it per grader when you need multiple grader models in one run: ```yaml assertions: @@ -123,7 +124,8 @@ assertions: prompt: ./prompts/pass-fail.md ``` -Each `target:` value must match a named LLM target in `.agentv/targets.yaml`. +Each `target:` value must match a grader `id` from `.agentv/config.yaml`, an +eval-local `graders` entry, or a directly referenced `graders: file://...` field. ### TypeScript Template @@ -165,7 +167,7 @@ Evaluate and provide a score from 0 to 1.`; ## How It Works 1. AgentV renders the prompt template with variables from the test -2. The rendered prompt is sent to the grader target (configured in targets.yaml) +2. The rendered prompt is sent to the selected grader target 3. The LLM returns a structured evaluation with score, assertions array, and reasoning 4. Results are recorded in the output JSONL diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx index d0c5345a9..9248f576a 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx @@ -3,9 +3,6 @@ title: CLI Provider description: Wrap any shell command as an evaluation target sidebar: order: 4 -slug: docs/v4.42.4/targets/cli-provider -editUrl: false -pagefind: false --- The `cli` provider runs an arbitrary shell command per test case and captures its output as the target's response. It's the escape hatch that lets you evaluate *anything* that exposes a command-line entry point — your own agent, a third-party CLI, a stub that prints a fixed answer, a script that calls an in-house microservice, etc. @@ -15,12 +12,25 @@ Because the contract is "we invoke a command and read a file," almost any useful ## Minimal example ```yaml -# .agentv/targets.yaml +# .agentv/config.yaml targets: - - name: my_agent + - id: my-agent provider: cli - command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} - grader_target: azure-base # required if your evals use LLM graders + runtime: host + config: + command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} + +graders: + - id: azure-grader + provider: azure + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + +defaults: + target: my-agent + grader: azure-grader ``` Your `agent.py` reads the prompt, writes its response to the path passed as `--out`, and exits `0`. That's it. @@ -73,29 +83,30 @@ echo "Hello, world!" > {OUTPUT_FILE} | Field | Type | Required | Default | Description | |---|---|---|---|---| -| `name` | string | yes | — | Target identifier used in eval configs. | +| `id` | string | yes | — | AgentV target identity used by eval `target`, CLI `--target`, and comparisons. | | `provider` | literal `"cli"` | yes | — | Selects this provider. | -| `command` | string | yes | — | Shell command template. | -| `timeout_seconds` | number | no | — | Kill the process if it runs longer than this. | -| `cwd` | string | no | eval dir | Working directory. Relative paths resolve against the eval file. | -| `files_format` | string | no | `{path}` | How each entry in `{FILES}` is formatted. Placeholders: `{path}`, `{basename}`. | -| `verbose` | boolean | no | `false` | Log the rendered command and cwd to stdout. Useful for debugging template substitution. | -| `keep_temp_files` | boolean | no | `false` | Preserve `{PROMPT_FILE}` / `{OUTPUT_FILE}` after the run — handy while iterating on your command. | -| `healthcheck` | object | no | — | Pre-run health check (HTTP or command); the eval aborts if it fails. | -| `workers` | number | no | — | Concurrent test-case executions against this target. | -| `provider_batching` | boolean | no | `false` | Run all cases in one command invocation — see [Batching](#batching). | -| `grader_target` | string | no | — | LLM target used by this target's LLM graders. Required if your evals use LLM-based graders. | +| `runtime` | string or object | no | `host` | Target runtime placement. | +| `config.command` | string | yes | — | Shell command template. | +| `config.timeout_seconds` | number | no | — | Kill the process if it runs longer than this. | +| `config.cwd` | string | no | eval dir | Working directory. Relative paths resolve against the eval file. | +| `config.files_format` | string | no | `{path}` | How each entry in `{FILES}` is formatted. Placeholders: `{path}`, `{basename}`. | +| `config.verbose` | boolean | no | `false` | Log the rendered command and cwd to stdout. Useful for debugging template substitution. | +| `config.keep_temp_files` | boolean | no | `false` | Preserve `{PROMPT_FILE}` / `{OUTPUT_FILE}` after the run — handy while iterating on your command. | +| `config.healthcheck` | object | no | — | Pre-run health check (HTTP or command); the eval aborts if it fails. | +| `config.batch_requests` | boolean | no | `false` | Run all cases in one command invocation — see [Batching](#batching). | ## Batching -For targets where spin-up cost dominates per-case work (e.g. loading a model, authenticating), set `provider_batching: true`. AgentV invokes the command *once*, hands it a JSONL stream of cases, and expects a JSONL response keyed by each case's `id`: +For targets where spin-up cost dominates per-case work (e.g. loading a model, authenticating), set `batch_requests: true`. AgentV invokes the command *once*, hands it a JSONL stream of cases, and expects a JSONL response keyed by each case's `id`: ```yaml targets: - - name: batched_agent + - id: batched-agent provider: cli - provider_batching: true - command: python agent.py --batch-in {PROMPT_FILE} --batch-out {OUTPUT_FILE} + runtime: host + config: + batch_requests: true + command: python agent.py --batch-in {PROMPT_FILE} --batch-out {OUTPUT_FILE} ``` `{PROMPT_FILE}` contains one JSON object per line with an `id` and the case's inputs; your command writes one line per case to `{OUTPUT_FILE}`, each carrying the matching `id` plus the same output shape as the non-batched case. @@ -107,17 +118,30 @@ A common question when building a new eval: **"if my grader scores my agent poor AgentV has no dedicated "oracle" feature because the `cli` provider already composes into one. Declare a second target that prints your known-good answer into `{OUTPUT_FILE}`, run the same eval against it, and assert a perfect score: ```yaml -# .agentv/targets.yaml +# .agentv/config.yaml targets: - - name: my_agent + - id: my-agent provider: cli - command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} - grader_target: azure-base + runtime: host + config: + command: python agent.py --prompt {PROMPT} --out {OUTPUT_FILE} - - name: oracle + - id: oracle provider: cli - command: cp fixtures/{EVAL_ID}.expected.txt {OUTPUT_FILE} - grader_target: azure-base + runtime: host + config: + command: cp fixtures/{EVAL_ID}.expected.txt {OUTPUT_FILE} + +graders: + - id: azure-grader + provider: azure + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + +defaults: + grader: azure-grader ``` ```bash @@ -126,7 +150,7 @@ targets: agentv eval my.EVAL.yaml --target oracle # Then run the real target. -agentv eval my.EVAL.yaml --target my_agent +agentv eval my.EVAL.yaml --target my-agent ``` A few practical notes: @@ -138,6 +162,11 @@ A few practical notes: The pattern needs no special config field, no directory convention, and no flag — it's just a second target that happens to know the answer. +Use this pattern instead of eval mock dry-run for grader validation. Mock +execution was removed because fake candidate answers produced misleading +quality failures; a reference target gives deterministic output while exercising +the real grader path. + ## Debugging When a `cli` target misbehaves: diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx index b936d406b..cbbbcabe5 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/coding-agents.mdx @@ -3,290 +3,342 @@ title: Coding Agents description: Evaluate coding agent targets sidebar: order: 3 -slug: docs/v4.42.4/targets/coding-agents -editUrl: false -pagefind: false --- -Coding agent targets evaluate AI coding assistants and CLI-based agents. These targets require a `grader_target` (also accepts `judge_target` for backward compatibility) to run LLM-based graders. +Coding agent targets evaluate assistants that run against a real workspace. They +use the target shape `id`, `provider`, `runtime`, and `config`. The target `id` +is AgentV's stable selection and artifact identity. `provider` names the +adapter or control boundary. `runtime` names where the target runs. Provider +settings live under `config`. -## Prompt format +Use `defaults.grader`, CLI `--grader` / `--grader-target`, or an +evaluator-specific target override for LLM-based grading. Grader selection is +separate from the coding-agent target, so target definitions do not carry a +grader field. -Agent providers receive a structured prompt document with two sections: a **preread block** listing files the agent must read, and the **user query** containing the eval input. - -### File handling +```yaml +targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high -When an eval test includes `type: file` inputs, agent providers do **not** receive the file content inline. Instead, they receive: +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini -1. A preread block with `file://` URIs pointing to absolute paths on disk -2. The user query with `` reference tags +defaults: + target: codex-local + grader: openai-grader -The agent is expected to read the files itself using its filesystem tools. +execution: + max_concurrency: 3 +``` -This differs from [LLM providers](/docs/v4.42.4/targets/llm-providers/), which receive file content embedded directly in the prompt as XML: +Process-backed coding-agent providers use `config.command` as a non-empty argv +array. The first element is the executable or shim, and the remaining elements +are arguments. -```xml - -// file content is inlined here - -``` +## Runtime placement -### Example prompt +| Runtime | Boundary | Best fit | +| --- | --- | --- | +| `host` | Runs the installed CLI or child runner on the current machine. | Local research, subscription OAuth, and evaluating the same profile an engineer uses manually. | +| `profile` | Runs a host process with isolated home/config/env such as `HOME`, `CODEX_HOME`, or temp dirs. | Cleaner local evals without container cost. | +| `sandbox` | Runs through a separate substrate such as Docker or a managed sandbox. | CI, reproducibility, untrusted tasks, and stronger filesystem containment. | -Given an eval with file inputs: +Sandbox mode does not inherit host credentials. For CI, prefer API keys or +explicit secrets injected through sandbox configuration. Subscription OAuth can +be evaluated only by intentionally mounting or seeding the profile directory +the agent needs, which trades portability for fidelity to the local agent setup. ```yaml -input: - - role: user - content: - - type: file - value: ./src/example.ts - - type: text - value: Review this code +targets: + - id: codex-clean-profile + provider: codex-cli + runtime: + mode: profile + codex_home: .agentv/profiles/codex-clean + tmp_dir: .agentv/tmp/codex-clean + config: + command: ["codex", "exec", "--json"] + model: gpt-5-codex + sandbox_mode: workspace-write + approval_policy: never ``` -The agent receives a prompt like: - +```yaml +targets: + - id: codex-ci-sandbox + provider: codex-cli + runtime: + mode: sandbox + engine: docker + image: ghcr.io/acme/codex-agent:sha256 + workdir: /workspace + mounts: + - source: ./workspace + target: /workspace + access: rw + - source: ./.agentv/results + target: /results + access: rw + secrets: + OPENAI_API_KEY: ${{ OPENAI_API_KEY }} + config: + command: ["codex", "exec", "--json"] + model: gpt-5-codex + timeout_seconds: 300 ``` -Read all input files: -* [example.ts](file:///abs/path/src/example.ts). -If any file is missing, fail with ERROR: missing-file and stop. -Then apply system_instructions on the user query below. +## Provider tradeoffs -[[ ## user_query ## ]] - -Review this code -``` +| Provider | Boundary | Transcript and isolation notes | +| --- | --- | --- | +| `codex-app-server` | Codex app-server subprocess. | Preferred Codex path for rich protocol events, session control, cancellation, and structured transcripts. | +| `codex-cli` | Codex CLI subprocess. | Best for simple local/CI process isolation and evaluating an installed Codex shim or profile. | +| `codex-sdk` | Codex SDK in an AgentV child runner. | Explicit SDK path. SDK crashes, malformed child output, and missing optional SDK packages are target execution errors. | +| `pi-rpc` | Pi launched in RPC mode over stdio. | Preferred rich Pi control boundary; AgentV launches the configured command with RPC mode when needed. | +| `pi-cli` | Pi CLI subprocess. | Simple process boundary; transcript richness depends on Pi CLI output. | +| `pi-sdk` | Pi SDK in an AgentV child runner. | Explicit SDK path for SDK-native events with child-process isolation. | +| `claude-cli` | Claude CLI subprocess. | Default Claude path; captures structured stream output when available. | +| `claude-sdk` | Claude Agent SDK in an AgentV child runner. | Explicit SDK path; useful when SDK-native events matter more than matching a local CLI invocation. | +| `copilot-cli` | Copilot CLI subprocess/protocol path. | Active Copilot eval run through the installed process. | +| `copilot-log` | Passive Copilot session-log reader. | Zero-cost transcript grading for existing sessions; it does not run a new agent. | +| `copilot-sdk` | Copilot SDK in an AgentV child runner. | Explicit SDK path with child-process isolation. | -The preread block instructs the agent to read input files before processing the query. If a `system_prompt` is configured on the target, it is passed separately via the provider SDK (not in the prompt document). +Every coding-agent provider returns a structured target execution envelope. +Run bundles preserve target id, provider kind, runtime mode, command argv, cwd, +stdout/stderr, transcripts or logs, final output when available, timing, +timeouts, exit codes, signals, and partial artifacts on failure. -## Claude +## Codex + +Use `codex-app-server` when you want rich protocol control: ```yaml targets: - - name: claude_agent - provider: claude - grader_target: azure-base + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high + model_verbosity: medium ``` -| Field | Required | Description | -|-------|----------|-------------| -| `executable` | No | CLI binary name or path (default: `claude`). Accepts a bare name looked up on PATH or an absolute/relative file path. | -| `cwd` | No | Working directory | -| `grader_target` | Yes | LLM target for evaluation | - -## Codex CLI +Use `codex-cli` when a simple CLI boundary is enough or when you want an +operator-specific shim: ```yaml targets: - - id: codex_target + - id: codex-eng provider: codex-cli runtime: host config: - command: ["codex-eng"] + command: ["codex-eng", "exec", "--json"] model: ${{ CODEX_MODEL }} - model_reasoning_effort: ${{ CODEX_REASONING_EFFORT }} ``` -| Field | Required | Description | -|-------|----------|-------------| -| `executable` | No | Codex binary or profile shim to run, such as `codex-eng` | -| `model` | No | Model to use | -| `model_reasoning_effort` | No | Codex SDK reasoning effort: `minimal`, `low`, `medium`, `high`, or `xhigh` | -| `cwd` | No | Working directory | -| `grader_target` | Yes | LLM target for evaluation | - -## Copilot CLI +Use `codex-sdk` only when you intentionally want the Codex SDK path: ```yaml targets: - - name: copilot - provider: copilot - model: gpt-5-mini - grader_target: azure-base + - id: codex-sdk-isolated + provider: codex-sdk + runtime: host + config: + model: gpt-5-codex ``` -| Field | Required | Description | -|-------|----------|-------------| -| `model` | No | Model to use (defaults to copilot's default) | -| `cwd` | No | Working directory | -| `subprovider` | No | OpenAI-compatible provider type for `copilot`, `copilot-cli`, or `copilot-sdk`, such as `openai` or `azure` | -| `base_url` | No | Provider base URL or Azure resource URL/name | -| `api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. | -| `bearer_token` | No | Provider bearer token. Prefer `${{ ENV_VAR }}` references. Takes precedence over `api_key` when set. | -| `api_version` | No | Provider API version, primarily for Azure endpoints | -| `api_format` | No | Provider API format, such as `responses` | -| `grader_target` | Yes | LLM target for evaluation | +Common Codex config fields include `command`, `model`, `reasoning_effort`, +`model_verbosity`, `base_url`, `api_key`, `api_format`, `sandbox_mode`, +`approval_policy`, `cwd`, `timeout_seconds`, `log_dir`, `stream_log`, and +`system_prompt`. -Route Copilot through an OpenAI-compatible endpoint: +## Pi + +Use `pi-rpc` for the rich stdio/RPC boundary: ```yaml targets: - - name: copilot-openai - provider: copilot-cli - subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - api_format: responses - grader_target: azure-base + - id: pi-rpc-local + provider: pi-rpc + runtime: host + config: + command: ["pi"] + model: gpt-5-codex + thinking: medium ``` -Values can come from environment variables through `${{ ... }}` interpolation. For `copilot-cli`, AgentV maps these flat fields to Copilot's documented provider environment variables before spawning `copilot`; omitted fields leave existing ambient `COPILOT_PROVIDER_*` values unchanged. - -## Pi Coding Agent +Use `pi-cli` for simple subprocess execution: ```yaml targets: - - name: pi_target - provider: pi-coding-agent - subprovider: openai-codex - model: gpt-5.5 - thinking: medium - grader_target: azure-base + - id: pi-cli-local + provider: pi-cli + runtime: + mode: profile + home: .agentv/profiles/pi-local + config: + command: ["pi"] + subprovider: openrouter + model: ${{ OPENROUTER_MODEL }} + api_key: ${{ OPENROUTER_API_KEY }} ``` -| Field | Required | Description | -|-------|----------|-------------| -| `subprovider` | No | Pi provider to use, such as `google`, `openai`, `openai-codex`, `azure`, `anthropic`, or `openrouter`. Defaults to Pi's default provider. | -| `model` | No | Model to use. For OpenAI subscription auth through Pi, use `subprovider: openai-codex` with a subscription model such as `gpt-5.5`. | -| `thinking` | No | Pi reasoning level: `off`, `minimal`, `low`, `medium`, `high`, or `xhigh`. Passed to the Pi SDK as `thinkingLevel`. | -| `tools` | No | Comma-separated Pi tool allowlist, such as `read,bash,edit,write`. | -| `api_key` | No | Provider API key. Prefer `${{ ENV_VAR }}` references. Omit for subscription auth handled by Pi. | -| `base_url` | No | Provider base URL or Azure resource URL/name. | -| `cwd` | No | Working directory | -| `timeout_seconds` | No | Per-case timeout | -| `grader_target` | Yes | LLM target for evaluation | - -For `provider: pi-coding-agent`, `base_url` is passed through the Pi SDK model -configuration. This works for OpenAI-compatible endpoints: +Use `pi-sdk` only when you intentionally want the SDK path: ```yaml targets: - - name: pi-sdk-openai - provider: pi-coding-agent - subprovider: openai - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} - grader_target: azure-base + - id: pi-sdk-isolated + provider: pi-sdk + runtime: host + config: + subprovider: openai-codex + model: gpt-5.5 + thinking: medium ``` -Use `provider: pi-cli` instead when you want AgentV to spawn the `pi` binary directly. It accepts the same Pi fields above plus: - -| Field | Required | Description | -|-------|----------|-------------| -| `executable` | No | Pi binary or shim to run. Defaults to `pi`. | -| `args` | No | Extra arguments appended before the prompt. | +Pi config fields include `command`, `subprovider`, `model`, `thinking`, +`tools`, `api_key`, `base_url`, `cwd`, `timeout_seconds`, `log_dir`, +`stream_log`, and `system_prompt`. With `pi-cli`, the built-in OpenAI provider +does not expose a CLI base-url option; use a Pi custom provider name or Pi's +Azure provider path for custom gateways. -Pi CLI has one important difference from the SDK path: the built-in `openai` -provider does not currently expose a CLI base-url option. With `provider: pi-cli` -and `subprovider: openai`, AgentV can pass the API key and model, but `base_url` -does not re-route the built-in OpenAI provider. For custom endpoints, either -configure a Pi custom provider in Pi's own `models.json` and reference that -provider name as `subprovider`, or use Pi's Azure provider path when your gateway -is compatible with Azure OpenAI Responses: +## Claude ```yaml targets: - - name: pi-cli-gateway - provider: pi-cli - subprovider: azure - base_url: ${{ OPENAI_ENDPOINT }} - api_key: ${{ OPENAI_API_KEY }} - model: ${{ OPENAI_MODEL }} - grader_target: azure-base + - id: claude-local + provider: claude-cli + runtime: host + config: + command: ["claude"] + model: claude-sonnet-4-20250514 + max_turns: 10 ``` -## VS Code +Use `claude-cli` when you want AgentV to spawn the same Claude CLI a user runs +locally. Use `claude-sdk` only when you intentionally want the Claude Agent SDK +path: ```yaml targets: - - name: vscode_dev - provider: vscode - grader_target: azure-base + - id: claude-sdk-isolated + provider: claude-sdk + runtime: host + config: + model: claude-sonnet-4-20250514 + max_turns: 10 ``` -| Field | Required | Description | -|-------|----------|-------------| -| `executable` | No | Path to VS Code binary. Supports `${{ ENV_VAR }}` syntax or literal paths. Defaults to `code` (or `code-insiders` for the insiders provider). | -| `grader_target` | Yes | LLM target for evaluation | +Claude config fields include `command`, `model`, `cwd`, `timeout_seconds`, +`max_turns`, `max_budget_usd`, `bypass_permissions`, `log_dir`, `stream_log`, +and `system_prompt`. -Using a custom executable path: +## Copilot ```yaml targets: - - name: vscode_dev - provider: vscode - executable: ${{ VSCODE_CMD }} - grader_target: azure-base + - id: copilot-local + provider: copilot-cli + runtime: host + config: + command: ["copilot"] + model: gpt-5-mini ``` -## VS Code Insiders +Route Copilot through an OpenAI-compatible endpoint: ```yaml targets: - - name: vscode_insiders - provider: vscode-insiders - grader_target: azure-base + - id: copilot-openai + provider: copilot-cli + runtime: host + config: + command: ["copilot"] + subprovider: openai + base_url: ${{ OPENAI_ENDPOINT }} + api_key: ${{ OPENAI_API_KEY }} + api_format: responses ``` -Same configuration as VS Code. +Read an existing Copilot session log without running a new agent: -## Custom CLI Agent +```yaml +targets: + - id: copilot-session-log + provider: copilot-log + runtime: host + config: + discover: latest +``` -Evaluate any command-line agent: +Use `copilot-sdk` only when you intentionally want the SDK path: ```yaml targets: - - name: local_agent - provider: cli - command: 'python agent.py --prompt-file {PROMPT_FILE} --output {OUTPUT_FILE}' - grader_target: azure-base + - id: copilot-sdk-isolated + provider: copilot-sdk + runtime: host + config: + model: gpt-5-mini ``` -| Field | Required | Description | -|-------|----------|-------------| -| `command` | Yes | Command to run. `{PROMPT}` is inline prompt text and `{PROMPT_FILE}` is a temp file path containing the prompt. | -| `cwd` | No | Working directory | -| `grader_target` | Yes | LLM target for evaluation | +Copilot config fields include `command`, `model`, `cwd`, `timeout_seconds`, +`subprovider`, `base_url`, `api_key`, `bearer_token`, `api_version`, +`api_format`, `log_dir`, `stream_log`, `system_prompt`, and session-log fields +such as `discover`, `session_id`, and `session_dir` for `copilot-log`. -## Mock Provider +## File inputs -For testing the evaluation harness without calling real providers: +Agent providers receive file inputs as paths, not inline file content. The +prompt includes a preread block with `file://` URIs pointing to absolute paths +on disk, then the user query references each file: ```yaml -targets: - - name: mock_target - provider: mock +input: + - role: user + content: + - type: file + value: ./src/example.ts + - type: text + value: Review this code ``` -## Known limitations +The agent receives a prompt like: + +```text +Read all input files: +* [example.ts](file:///abs/path/src/example.ts). -### VS Code +If any file is missing, fail with ERROR: missing-file and stop. +Then apply system_instructions on the user query below. -The VS Code provider uses a **subagent file-messaging architecture**. AgentV provisions pre-configured VS Code workspace directories (subagents), dispatches requests by writing prompt files, and the AI agent writes its response to a file. Lock files control concurrency. +[[ ## user_query ## ]] + +Review this code +``` -- **Per-target worker limit**: VS Code evals run with 1 worker per target because the provider requires window focus to dispatch requests. When multiple targets are configured (e.g., `vscode` + `copilot`), they run concurrently — the single-worker limit only applies within each VS Code target. Subagents are provisioned automatically if needed. -- **Windows only**: VS Code is not available on Linux CI. E2E testing must be done on a Windows machine. -- **`.code-workspace` support**: When your eval uses `workspace.template` with a `.code-workspace` file, the template folders are opened in the VS Code window alongside the subagent directory. +LLM providers receive file content inline instead; see +[LLM providers](/docs/targets/llm-providers). -### Copilot CLI +## Mock provider -- **MCP OAuth token expiration**: If your copilot CLI has MCP servers configured that use OAuth authentication, **expired tokens will block eval execution**. The copilot CLI attempts to re-authenticate via a browser OAuth flow, which cannot complete in non-interactive mode and causes the eval to hang indefinitely. Before running evals, either re-authenticate your MCP servers manually (`copilot` → `/mcp`) or remove MCP servers with expired tokens. See [copilot-cli#1797](https://github.com/github/copilot-cli/issues/1797) and [copilot-cli#1491](https://github.com/github/copilot-cli/issues/1491) for upstream tracking. -- **Windows shell shim vs process spawn**: On Windows, `copilot -h` may work in PowerShell while AgentV still fails with `spawn copilot ENOENT`. Shell commands can execute `copilot.ps1`/`copilot.bat`, but AgentV launches a subprocess that expects a directly spawnable executable path. If this occurs, set an explicit target executable (for example via env var): +For deterministic harness checks without a real provider: ```yaml targets: - - name: copilot - provider: copilot - executable: ${{ COPILOT_EXE }} - grader_target: azure-base + - id: mock-target + provider: mock + runtime: host + config: + response: ok ``` - -Use a native binary path for `COPILOT_EXE` (for example `copilot.exe` from `@github/copilot-win32-x64`). - -### Claude Code - -- **Run evals externally**: Run agentv evals from **outside** Claude Code. Running `agentv eval` with the `claude` target from within a Claude Code session can cause unintended behavior — the spawned Claude agent may interfere with the parent session. -- **`ANTHROPIC_API_KEY` overrides subscription auth**: Claude Code loads `.env` from the working directory on startup. If your `.env` contains `ANTHROPIC_API_KEY`, the spawned Claude Code process will use that API key instead of your Claude subscription (Max/Pro). If the API key has insufficient credits, evals will fail with "Credit balance is too low". To use subscription auth, remove `ANTHROPIC_API_KEY` from your `.env` file. diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx index a8fb01afb..c66fc5a6c 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/configuration.mdx @@ -3,33 +3,176 @@ title: Targets Configuration description: Configure execution targets for providers and agents sidebar: order: 1 -slug: docs/v4.42.4/targets/configuration -editUrl: false -pagefind: false --- -Targets define which agent or LLM provider to evaluate. They are configured in `.agentv/targets.yaml` to decouple eval files from provider details. +Targets define which agent or LLM provider to evaluate. AgentV uses one +composable config graph across project manifests and eval files: + +- `.agentv/config.yaml` is the project-local discovery and composition root. It + can hold targets, graders, tests, defaults, execution policy, results + settings, and repo-local project policy. +- `$AGENTV_HOME/config.yaml` is the user/operator config. Use it for defaults + that apply across projects, project registry data, default result locations, + and provider defaults that should not be copied into each repo. +- `eval.yaml` is a focused, shareable slice of the same graph. Use it for a + suite-specific target, grader, tests, evaluator settings, or run controls + that should travel with the eval. + +Any supported top-level field can stay inline or become a direct field +reference such as `targets: file://targets.yaml`. Both forms normalize to the +same config graph. ## Structure ```yaml targets: - - name: azure-base - provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - - - name: vscode_dev - provider: vscode - grader_target: azure-base - - - name: local_agent - provider: cli - command: 'python agent.py --prompt {PROMPT}' - grader_target: azure-base + - id: local-openai + provider: openai + runtime: host + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} + + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini + +defaults: + target: codex-local + grader: openai-grader +``` + +Use `id` for the stable AgentV target identity. `provider` selects the adapter +or control boundary. `runtime` describes where the provider runs; use `host` as +the shorthand for the current machine, or object form when you need +`mode: host | profile | sandbox` plus runtime-specific settings. Provider +settings belong under `config`. Process-backed coding-agent providers use +`config.command` as a non-empty argv array. + +## Runtime Modes + +Use `runtime: host` when you want AgentV to run the target exactly as it is +installed on the current machine. This is the best fit for local research, +subscription-auth workflows, and evaluating the same CLI profile an engineer +uses manually. + +Use `runtime.mode: profile` when the target still runs as a host process but +should use an isolated home/config directory, such as a dedicated `CODEX_HOME` +or `HOME`. + +Use `runtime.mode: sandbox` when the target should run inside a separate +execution substrate. The built-in sandbox runner currently supports Docker for +`provider: cli`; provider-specific coding-agent adapters such as `codex-cli`, +`claude-cli`, `copilot-cli`, and `pi-cli` return a structured unsupported target +error until their transcript parsers are wired through sandbox-aware runners. + +```yaml +targets: + - id: codex-sandbox + provider: codex-cli + runtime: + mode: sandbox + engine: docker + image: ghcr.io/acme/codex-agent:sha256 + workdir: /workspace + network: none + mounts: + - source: ./workspace + target: /workspace + access: rw + - source: ./.agentv/results + target: /results + access: rw + env: + AGENTV_RESULT_DIR: /results + secrets: + OPENAI_API_KEY: ${{ OPENAI_API_KEY }} + config: + command: ["codex", "exec", "--json"] + timeout_seconds: 300 ``` +Sandbox mode does not inherit host credentials by default. Mount only the +workspace, results, cache, or credential paths the target needs, and pass only +the environment variables and secrets listed under `runtime.env` and +`runtime.secrets`. Install the target CLI by using an image that already +contains it or by adding explicit setup under `runtime.setup`; locate the CLI +with `config.command`. + +For CI, API-key or explicitly injected secret auth is the most reproducible +path. Subscription OAuth can work in a sandbox only when you intentionally mount +or seed the relevant profile directory into the sandbox. That makes the run less +portable than API-key CI and should be reserved for workflows where matching a +local subscription profile is the point of the evaluation. + +Inline and decomposed forms are equivalent. This single-file config: + +```yaml +targets: + - id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + +graders: + - id: openai-grader + provider: openai + config: + model: gpt-5-mini + +tests: + - id: smoke + input: Fix the failing test. + +defaults: + target: codex-local + grader: openai-grader +``` + +can be decomposed like this: + +```yaml +targets: file://targets.yaml +graders: file://graders.yaml +tests: file://tests.yaml +defaults: file://defaults.yaml +``` + +Referenced field files contain the field value directly. `targets.yaml` contains +a bare array, not an object wrapped in `targets:`: + +```yaml +# .agentv/targets.yaml +- id: codex-local + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex +``` + +```yaml +# .agentv/defaults.yaml +target: codex-local +grader: openai-grader +``` + +File refs are optional. Use them when a config field is large, reused, or owned +by a separate team; keep fields inline when that is easier to read. + ## Environment Variables Use `${{ VARIABLE_NAME }}` syntax to reference values from your environment. AgentV reads @@ -38,10 +181,12 @@ eval directory hierarchy when present: ```yaml targets: - - name: my_target + - id: my-target provider: anthropic - api_key: ${{ ANTHROPIC_API_KEY }} - model: ${{ ANTHROPIC_MODEL }} + runtime: host + config: + api_key: ${{ ANTHROPIC_API_KEY }} + model: ${{ ANTHROPIC_MODEL }} ``` This keeps secrets out of version-controlled files and avoids requiring a CI step that rewrites @@ -54,92 +199,97 @@ already-exported secrets into `.env`. | `azure` | LLM | Azure OpenAI | | `anthropic` | LLM | Anthropic Claude API | | `gemini` | LLM | Google Gemini | -| `claude` | Agent | Claude Agent SDK | -| `codex` | Agent | Codex CLI | -| `pi-coding-agent` | Agent | Pi Coding Agent | +| `claude-cli` | Agent | Claude CLI subprocess | +| `claude-sdk` | Agent | Claude Agent SDK in an isolated child runner | +| `codex-cli` | Agent | Codex CLI subprocess | +| `codex-app-server` | Agent | Codex app-server subprocess | +| `codex-sdk` | Agent | Codex SDK in an isolated child runner | +| `copilot-cli` | Agent | Copilot CLI subprocess | +| `copilot-log` | Agent | Passive Copilot CLI session log reader | +| `copilot-sdk` | Agent | Copilot SDK in an isolated child runner | +| `pi-sdk` | Agent | Pi SDK in an isolated child runner | +| `pi-cli` | Agent | Pi CLI subprocess | +| `pi-rpc` | Agent | Pi RPC subprocess over stdio | | `vscode` | Agent | VS Code with Copilot | | `vscode-insiders` | Agent | VS Code Insiders | -| `cli` | Agent | Any CLI command — see [CLI Provider](/docs/v4.42.4/targets/cli-provider/) | -| `mock` | Testing | Mock provider for dry runs | +| `cli` | Agent | Any CLI command — see [CLI Provider](/docs/targets/cli-provider) | +| `mock` | Testing | Explicit mock target for examples and tests | ## Referencing Targets in Evals -Set the default target at the top level or override per case: +Select the system under test with `defaults.target`, top-level `target`, or CLI +`--target`, depending on the command flow. Test cases do not choose targets; +split target-specific cases into separate eval suites, select them with +tags/filters, or run the same eval with different `--target` values. ```yaml -# Top-level default -execution: - target: azure-base +target: local-openai tests: - id: test-1 - # Uses azure-base - - id: test-2 - execution: - target: vscode_dev # Override for this case ``` -## Grader Target - -Agent targets that need LLM-based evaluation specify a `grader_target` (also accepts `judge_target` for backward compatibility) — the LLM used to run LLM grader graders: +The string is a configured target `id`. Use object form when an eval needs a +local target variant: ```yaml -targets: - - id: codex_target - provider: codex-cli - runtime: host - config: - command: ["codex"] +target: + id: codex-high-reasoning + provider: codex-app-server + runtime: host + config: + command: ["codex", "app-server"] + model: gpt-5-codex + reasoning_effort: high ``` -### Workspace Lifecycle Hooks +Use `defaults.grader` for the project default grader. A specific evaluator can +still choose its own grader target when the evaluator supports that override. + +### Lifecycle Extensions -Run commands and reset/cleanup policies at different lifecycle points using `workspace.hooks`. This can be defined at the suite level (applies to all tests) or per test (overrides suite-level). +Run non-provisioning setup at Promptfoo-compatible lifecycle points using +top-level `extensions`. The harness materializes `workspace.template` and +`workspace.repos` first, then runs `beforeAll` extensions. Use extensions for +dependency installs, builds, fixture generation, and agent-rule staging. Use +target hooks for runner-specific setup. Keep repo identity and checkout pins in +`workspace.repos`; extensions must not become the default repo acquisition path. ```yaml +extensions: + - file://scripts/workspace.mjs:beforeAll + - file://scripts/workspace.mjs:beforeEach + - file://scripts/workspace.mjs:afterEach + - file://scripts/workspace.mjs:afterAll + - id: agentv:agent-rules + hook: beforeAll + skills: agent-rules/skills + rules: agent-rules/AGENTS.md + workspace: template: ./workspace-templates/my-project hooks: - before_all: - command: ["bun", "run", "setup.ts"] - timeout_ms: 120000 - cwd: ./scripts after_each: - command: ["bun", "run", "reset.ts"] - timeout_ms: 5000 reset: fast - after_all: - command: ["bun", "run", "cleanup.ts"] - timeout_ms: 30000 ``` | Field | Description | |-------|-------------| | `template` | Directory to copy as workspace | -| `hooks.before_all` | Runs once after workspace creation, before the first test | -| `hooks.after_all` | Runs once after the last test, before cleanup | -| `hooks.before_each` | Runs before each test | -| `hooks.after_each` | Runs after each test (supports both `command` and `reset`) | - -Each hook config accepts: +| `extensions[]` | `file://...:beforeAll`, `beforeEach`, `afterEach`, `afterAll`, or `agentv:agent-rules` | +| `hooks.after_each.reset` | Reset mode: `none`, `fast`, `strict` | -| Field | Description | -|-------|-------------| -| `command` | Command array (e.g., `["bun", "run", "setup.ts"]`) | -| `reset` | Reset mode: `none`, `fast`, `strict` | -| `timeout_ms` | Timeout in milliseconds (default: 60000 for setup hooks, 30000 for teardown hooks) | -| `cwd` | Working directory (relative paths resolved against eval file directory) | - -**Lifecycle order:** template copy → repo materialization → workspace `hooks.before_all` → target `hooks.before_all` → git baseline → (`hooks.before_each` → target `hooks.before_each` → agent runs → file changes captured → target `hooks.after_each` → `hooks.after_each`) × N tests → target `hooks.after_all` → `hooks.after_all` → cleanup +**Lifecycle order:** template copy → repo materialization → `extensions.beforeAll` → target `hooks.before_all` → git baseline → (`extensions.beforeEach` → target `hooks.before_each` → agent runs → file changes captured → target `hooks.after_each` → `extensions.afterEach` → `workspace.hooks.after_each.reset`) × N tests → target `hooks.after_all` → `extensions.afterAll` → cleanup **Shared workspace:** The workspace is created once and shared across all tests in a suite. Use `hooks.after_each.reset` to reset state between tests (e.g., `fast`/`strict`). **Error handling:** -- `hooks.before_all` / `hooks.before_each` command failure aborts the test with an error result -- `hooks.after_all` / `hooks.after_each` command failure is non-fatal (warning only) +- `beforeAll` / `beforeEach` extension failure aborts the affected run with an error result +- `afterAll` / `afterEach` extension failure is non-fatal -**Script context:** All scripts receive a JSON object on stdin with case context: +**File hook context:** Exported functions receive a JSON-compatible object with +case context: ```json { @@ -147,11 +297,13 @@ Each hook config accepts: "test_id": "case-01", "eval_run_id": "run-123", "case_input": "Fix the bug", - "case_metadata": { "repo": "sympy/sympy", "base_commit": "abc123" } + "case_metadata": { "repo": "sympy/sympy", "source_commit": "abc123" } } ``` -**Suite vs per-test:** When both are defined, test-level fields replace suite-level fields. See [Per-Test Workspace Config](/docs/v4.42.4/evaluation/eval-cases/#per-case-workspace-config) for examples. +`workspace.hooks` remains the reset-policy home for `after_each.reset`. Legacy +command hooks still parse for existing local suites, but new portable evals +should use `extensions` for executable setup. ### Repository Lifecycle @@ -167,34 +319,27 @@ workspace: hooks: after_each: reset: fast # none | fast | strict - isolation: shared # shared (default) | per_test - mode: pooled # pooled | temp | static - path: /tmp/my-ws # workspace path for mode=static + scope: suite # suite (default) | attempt ``` -`repo` declares the repository identity. Acquisition is harness-owned: AgentV first looks for matching registered projects and configured mirrors, then uses its git cache, then falls back to remote clone. See [Workspace Architecture](/docs/v4.42.4/guides/workspace-architecture/#repo-provenance-vs-acquisition) for the resolver order and `git_cache.mirrors` config. +`repo` declares the repository identity. Acquisition is harness-owned: AgentV first applies configured `repo_resolvers`, then uses the built-in git path of registered projects, configured mirrors, AgentV's git cache, and remote clone. See [Workspace Architecture](/docs/guides/workspace-architecture/#acquisition-resolver) for the resolver order, command resolver protocol, and `git_cache.mirrors` config. | Field | Description | |-------|-------------| | `repos[].path` | Directory within the workspace to clone into | | `repos[].repo` | Repository identity: full clone URL or GitHub `org/name` shorthand | | `repos[].commit` | Branch, tag, or SHA to check out (default: `HEAD`) | -| `repos[].base_commit` | Alias for `commit`, useful for SWE-bench-style datasets | | `repos[].ancestor` | Walk N commits back from the checked-out ref (e.g., `1` for parent) | | `repos[].sparse` | Sparse checkout paths | | `hooks.after_each.reset` | Reset policy after each test: `none`, `fast`, `strict` | -| `isolation` | `shared` reuses one workspace; `per_test` creates a fresh copy per test | -| `mode` | Workspace mode: `pooled`, `temp`, `static` | -| `path` | Workspace path for `mode=static`. When empty or missing, the workspace is auto-materialised (template copied + repos cloned). Populated directories are reused as-is. | +| `scope` | `suite` reuses one harness-managed workspace for the suite; `attempt` creates a clean workspace for each resolved execution attempt | | `hooks.enabled` | Boolean (default: `true`). Set `false` to skip all lifecycle hooks. | -**Pooling:** `mode: pooled` (or default shared repo mode) reuses pool slots between runs. Use `mode: temp` to disable pooling for fresh clone/checkouts each run. +Use `scope: attempt` when mutating agents need clean filesystem state for every prompt-target-test-repeat execution. Use `scope: suite` when the suite intentionally shares state across tests. -**Static auto-materialisation:** When `mode: static` and `path` points to an empty or missing directory, AgentV automatically copies the template and clones repos into it. If the directory already exists and is populated, it is reused as-is. +**Existing local workspaces:** do not commit local paths in eval YAML. Use `--workspace-path /path/to/workspace` for a one-off run, or put `execution.workspace_path` in `.agentv/config.local.yaml`. -Pool management commands: -- `agentv workspace list` — list all pool entries with size and repo info -- `agentv workspace clean` — remove all pool entries +Workspace command: - `agentv workspace deps ` — scan eval files and output a JSON manifest of required git repos (for CI pre-cloning) **Common patterns:** @@ -218,12 +363,12 @@ workspace: after_each: reset: fast -# GitHub shorthand with a base_commit alias +# GitHub shorthand with a pinned commit workspace: repos: - path: ./repo repo: org/repo - base_commit: abc123def + commit: abc123def ``` ### Cleanup Behavior @@ -244,31 +389,23 @@ Use `cwd` on a target to run in an existing directory (shared across tests). If Eval files can define per-target hooks that run setup/teardown scripts to customize the workspace for each target variant. This enables comparing different harness configurations (e.g., baseline vs with-plugins) in a single eval file. -Targets do not declare `repos`. Repositories belong to the shared eval workspace so every target runs in the same world; target hooks customize the harness under evaluation. Use hooks for per-target setup such as copying skills, enabling wrappers, or changing provider-local config. +Targets do not declare `repos`. Repositories belong to the shared eval workspace so every target runs in the same world; target hooks customize the harness under evaluation. Use hooks for per-target setup such as enabling wrappers or changing provider-local config. Keep installs, builds, fixture generation, and case setup in top-level lifecycle `extensions`. -Target hooks are defined in the eval file's `execution.targets` array using object form: +Target hooks can be scoped to an eval-local target object: ```yaml -execution: - targets: - - baseline # string shorthand (no hooks) - - name: with-skills # object form with hooks - use_target: default - hooks: - before_each: - command: ["setup-plugins.sh", "skills"] - - name: with-guidelines - use_target: default - hooks: - before_each: - command: ["sh", "-c", "cp guidelines.md {{workspace_path}}/.claude/"] +target: + extends: default + hooks: + before_each: + command: ["setup-plugins.sh", "skills"] ``` ### Hook execution order Target hooks run after workspace hooks on setup, before workspace hooks on teardown: -1. Workspace `before_all` +1. Extension `beforeAll` 2. **Target `before_all`** 3. For each test: - Workspace `before_each` diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/custom-providers.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/custom-providers.mdx index 6c5c2394e..7e810a233 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/custom-providers.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/custom-providers.mdx @@ -3,9 +3,6 @@ title: Custom Providers (SDK) description: Implement native TypeScript providers using the ProviderRegistry API sidebar: order: 6 -slug: docs/v4.42.4/targets/custom-providers -editUrl: false -pagefind: false --- Custom providers let you implement evaluation targets in TypeScript instead of shelling out to a CLI command. This is useful when you want to call an HTTP API, use an SDK, or implement custom logic that goes beyond what the CLI provider supports. @@ -178,11 +175,24 @@ registry.register('http-agent', (target: ResolvedTarget) => { Then reference it in your targets file: ```yaml -# .agentv/targets.yaml +# .agentv/config.yaml targets: - - name: my_http_agent + - id: my-http-agent provider: http-agent - grader_target: azure-base + runtime: host + config: + base_url: http://localhost:8080 + +graders: + - id: azure-grader + provider: azure + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + +defaults: + grader: azure-grader ``` :::note @@ -211,9 +221,11 @@ Use `provider: cli` when: ```yaml targets: - - name: python_agent + - id: python-agent provider: cli - command: 'python agent.py --prompt-file {PROMPT_FILE} --output {OUTPUT_FILE}' + runtime: host + config: + command: 'python agent.py --prompt-file {PROMPT_FILE} --output {OUTPUT_FILE}' ``` ### When to use native providers diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/llm-providers.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/llm-providers.mdx index db90b3c08..0a0df092b 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/llm-providers.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/llm-providers.mdx @@ -3,9 +3,6 @@ title: LLM Providers description: Direct LLM API provider targets sidebar: order: 2 -slug: docs/v4.42.4/targets/llm-providers -editUrl: false -pagefind: false --- LLM provider targets call language model APIs directly. These are used both as evaluation targets and as grader targets for scoring. @@ -14,10 +11,12 @@ LLM provider targets call language model APIs directly. These are used both as e ```yaml targets: - - name: openai-target + - id: openai-target provider: openai - api_key: ${{ OPENAI_API_KEY }} - model: gpt-4o + runtime: host + config: + api_key: ${{ OPENAI_API_KEY }} + model: gpt-4o ``` | Field | Required | Description | @@ -41,30 +40,57 @@ Most users should leave this unset. The default `chat` format is universally sup ```yaml # OpenAI-compatible endpoint (default chat format works) targets: - - name: github-models + - id: github-models provider: openai - api_format: chat - base_url: https://models.github.ai/inference/v1 - api_key: ${{ GH_MODELS_TOKEN }} - model: ${{ GH_MODELS_MODEL }} + runtime: host + config: + api_format: chat + base_url: https://models.github.ai/inference/v1 + api_key: ${{ GH_MODELS_TOKEN }} + model: ${{ GH_MODELS_MODEL }} # Opt in to Responses API for api.openai.com - - name: openai-responses + - id: openai-responses provider: openai - api_format: responses - api_key: ${{ OPENAI_API_KEY }} - model: gpt-4o + runtime: host + config: + api_format: responses + api_key: ${{ OPENAI_API_KEY }} + model: gpt-4o ``` +### Local OpenAI-compatible endpoints + +For smoke tests and dogfood runs against a local OpenAI-compatible proxy, keep +the endpoint, model, and placeholder key in environment variables: + +```yaml +graders: + - id: local-openai-grader + provider: openai + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} +``` + +If the local proxy does not require authentication, set +`LOCAL_OPENAI_PROXY_API_KEY` to a non-secret placeholder such as +`dummy-local-key`; do not commit literal keys or machine-local model choices to +shared eval files. + ## Azure OpenAI ```yaml targets: - - name: azure-base + - id: azure-base provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} + runtime: host + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} ``` | Field | Required | Description | @@ -82,12 +108,14 @@ If your Azure deployment only exposes `/chat/completions` (older deployments, ce ```yaml targets: - - name: azure-chat + - id: azure-chat provider: openai - base_url: https://.openai.azure.com/openai/deployments/ - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: - api_format: chat + runtime: host + config: + base_url: https://.openai.azure.com/openai/deployments/ + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: + api_format: chat ``` The `api_format` field was previously available on `provider: azure` but has been removed — Azure targets always go through the Responses API. @@ -96,10 +124,12 @@ The `api_format` field was previously available on `provider: azure` but has bee ```yaml targets: - - name: claude_target + - id: claude-target provider: anthropic - api_key: ${{ ANTHROPIC_API_KEY }} - model: claude-sonnet-4-20250514 + runtime: host + config: + api_key: ${{ ANTHROPIC_API_KEY }} + model: claude-sonnet-4-20250514 ``` | Field | Required | Description | @@ -111,10 +141,12 @@ targets: ```yaml targets: - - name: gemini_target + - id: gemini-target provider: gemini - api_key: ${{ GEMINI_API_KEY }} - model: gemini-2.0-flash + runtime: host + config: + api_key: ${{ GEMINI_API_KEY }} + model: gemini-2.0-flash ``` | Field | Required | Description | diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/retry.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/retry.mdx index 1963b9a43..725a5ded5 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/retry.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/retry.mdx @@ -3,9 +3,6 @@ title: Retry Configuration description: Configure automatic retry with exponential backoff sidebar: order: 5 -slug: docs/v4.42.4/targets/retry -editUrl: false -pagefind: false --- Configure automatic retry with exponential backoff for transient failures. @@ -16,16 +13,18 @@ Add retry fields to any target: ```yaml targets: - - name: azure-base + - id: azure-base provider: azure - endpoint: ${{ AZURE_OPENAI_ENDPOINT }} - api_key: ${{ AZURE_OPENAI_API_KEY }} - model: ${{ AZURE_DEPLOYMENT_NAME }} - max_retries: 5 - retry_initial_delay_ms: 2000 - retry_max_delay_ms: 120000 - retry_backoff_factor: 2 - retry_status_codes: [500, 408, 429, 502, 503, 504] + runtime: host + config: + endpoint: ${{ AZURE_OPENAI_ENDPOINT }} + api_key: ${{ AZURE_OPENAI_API_KEY }} + model: ${{ AZURE_DEPLOYMENT_NAME }} + max_retries: 5 + retry_initial_delay_ms: 2000 + retry_max_delay_ms: 120000 + retry_backoff_factor: 2 + retry_status_codes: [500, 408, 429, 502, 503, 504] ``` ## Fields diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/dashboard.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/dashboard.mdx index 02554447a..f76ddd593 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/dashboard.mdx @@ -3,9 +3,6 @@ title: Dashboard description: Visual dashboard for reviewing evaluation results sidebar: order: 6 -slug: docs/v4.42.4/tools/dashboard -editUrl: false -pagefind: false --- import { Image } from 'astro:assets'; @@ -34,7 +31,7 @@ The `dashboard` command launches a web-based dashboard for browsing evaluation r agentv dashboard ``` -Dashboard auto-discovers run workspaces from `.agentv/results/runs/` in the current directory and opens at `http://localhost:3117`. +Dashboard auto-discovers v2 run workspaces from `.agentv/results//` in the current directory and opens at `http://localhost:3117`. Experiment is read from `summary.json` or row metadata, not from the path. To open a different project, pass the project root with `--dir`: @@ -42,7 +39,22 @@ To open a different project, pass the project root with `--dir`: agentv dashboard --dir /path/to/project ``` -Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/runs/` tree, plus an external results repository or run directory configured under `results:` in YAML. For one-off inspection of a copied run bundle, use `agentv results report `. +Dashboard does not accept a run workspace directory or `index.jsonl` manifest as a direct source. It reads one configured run source per project: the project's `.agentv/results/` tree, plus an external results repository or run directory configured under `results:` in YAML. For one-off inspection of a copied run bundle, use `agentv results report `. + +## Data boundary + +Dashboard is the supported zero-infra inspection path for AgentV-owned runs, +trace sidecars, transcripts, sessions, and Git-backed result artifacts. It does +not require Phoenix, the `px` CLI, a Phoenix database, or a hosted Dashboard +service. + +If a trace artifact includes safe `external_trace` metadata for spans that were +already emitted to Phoenix by Codex, Arize, or another hook, Dashboard may show +that external reference as an **Open in Phoenix** link. Dashboard does not +proxy Phoenix GraphQL/REST or embed Phoenix session, trace, or span views. +AgentV still treats the local/Git-backed run artifacts as canonical and does +not export or project completed runs, transcripts, datasets, experiments, or +indexes into Phoenix. ## Options @@ -61,7 +73,6 @@ Dashboard does not accept a run workspace directory or `index.jsonl` manifest as - **Experiments** — group and compare runs by experiment name - **Targets** — group runs by target (model/agent) - **Run Detail** — drill into a run to see per-test results, scores, and grader output -- **Human Review** — add feedback annotations to individual test results - **Analytics** — two modes: an aggregated experiment × target matrix, and a per-run view for selecting individual runs to compare side-by-side with optional retroactive tags. Includes a collapsible charts section with baseline comparison analytics - **Remote Results** — sync and browse runs pushed from other machines or CI (see [Remote Results](#remote-results)) @@ -76,26 +87,19 @@ dashboard: Legacy `studio.threshold`, `studio.pass_threshold`, and root-level `pass_threshold` values are still read for existing projects. When Dashboard saves settings, it writes the canonical `dashboard.threshold` field and preserves unrelated config. -## White label - -Dashboard shows AgentV by default. Override the displayed name with `dashboard.app_name` in project-local `.agentv/config.yaml`: - -```yaml -dashboard: - app_name: ai evals -``` - -You can also set the same field globally in `$AGENTV_HOME/config.yaml` or `~/.agentv/config.yaml`. Project-local config takes precedence over the global value. - ## Run Detail -Click any run to see a breakdown by suite, per-test scores, target, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `index.jsonl`—including per-result task bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. +Click any run to see a breakdown by suite, per-test scores, target, duration, and cost. The source label (`local` or `remote`) tells you where the run came from. Files and source views resolve against the generated run artifacts referenced by `index.jsonl`—including per-result test bundles when present—so Dashboard does not require authors to create a separate source-side bundle structure. + +In the per-test results table, click a test ID to open its checks, transcript, source, and files in a row detail panel while the table, filters, and scroll position stay in place. Use **Full page** from the panel when you want the standalone eval detail route. AgentV Dashboard run detail showing 100% pass rate across 5 tests with scores and duration ## Run management -In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. +In Recent Runs, select local completed runs to combine partial runs or delete stale run workspaces. Combine creates a new local run workspace and leaves the source runs in place. If all selected runs are from one experiment, the combined run records that experiment label, including `default`; if selected runs span experiments, Dashboard asks for a new experiment name before creating the combined run. Delete removes the selected local run workspace directory, including sidecars such as `tags.json`; remote runs are read-only. + +When you launch an eval from Dashboard, set the experiment and initial tags before the run starts. The selected experiment is recorded with the new run, and tags are written to that run workspace's `tags.json` sidecar; existing runs are not changed. The same deletion primitive is available from the CLI: @@ -147,9 +151,9 @@ Select 2+ rows with the checkboxes and click the sticky **Compare N** action to ### Retroactive tags -Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the run workspace, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to remove every tag (deletes the sidecar). +Click any row's **Tags** cell to tag a run after the fact. Each run can carry multiple free-form tags (max 20, up to 60 characters each); local tags are stored in a `tags.json` sidecar next to `index.jsonl` in the run folder, so they're mutable, non-destructive, and won't touch your eval YAML or run manifest. The chip editor supports Enter/comma to commit a new tag, Backspace to remove the last chip, and **Clear all** to record an empty tag state. The sidecar includes a `tag_revision`; if a stale browser tab submits tags after the run's tags changed, Dashboard rejects the write and asks you to refresh before retrying. -Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `.agentv/results/metadata/runs/.../tags.json` in the configured results repo clone. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. +Remote run payloads stay immutable, but their tags are editable. Dashboard writes remote tag changes as metadata overlays under `metadata/runs//tags.json` in the configured results repo clone/branch. That overlay path is a remote-results implementation detail, not part of the local `.agentv/results//` layout. Remote tag overlays use the same `tag_revision` stale-write check as local tags. Until those overlays are synced, the run and project show a dirty state; **Sync Project** commits and pushes them when it is safe to do so. Use tags to annotate ad-hoc variants, experiment cross-cuts, or status flags you didn't plan for up front — `baseline`, `v2-prompt`, `slow`, `after-retry-fix`, `regression`, etc. Unlike `experiment` — which groups runs and is baked into the JSONL at eval-run time — tags are mutable, multi-valued, and never touch the original run data. @@ -167,7 +171,7 @@ Below the aggregated matrix, a collapsible **Analytics** section provides visual The section includes the following visualizations: -- **Normalized Gain (g)** — horizontal bar chart showing how much of the remaining headroom each experiment × target captured relative to the baseline. Bars are colour-coded green (positive gain), red (regression), or grey (null / no headroom). See [Normalized Gain](/docs/v4.42.4/tools/compare/#normalized-gain-g) for the formula. +- **Normalized Gain (g)** — horizontal bar chart showing how much of the remaining headroom each experiment × target captured relative to the baseline. Bars are colour-coded green (positive gain), red (regression), or grey (null / no headroom). See [Normalized Gain](/docs/tools/compare#normalized-gain-g) for the formula. - **Tag × Target Heatmap** — pass-rate grid across tags and targets, colour-coded by performance (emerald for high, amber for medium, red for low). - **Negative Delta Table** — filtered list of experiment × target pairs that scored worse than the baseline, sorted by largest regression. - **Score Distribution** — histogram showing the variance of scores across all test cases, binned by 10% intervals. @@ -195,15 +199,14 @@ folder path, and select a directory that contains `.agentv/`. Each path must contain a `.agentv/` directory. Registered projects are stored under `projects:` in `$AGENTV_HOME/config.yaml`, or `~/.agentv/config.yaml` when `AGENTV_HOME` is unset. -To register a remote repo and keep it synced automatically, add `repo_url` and `ref` to the entry in `$AGENTV_HOME/config.yaml`. `repo_url` is the Git remote URL AgentV passes to `git clone`, so it can be HTTPS or SSH: +To register a remote repo and keep it synced automatically, add the source repo fields directly to the entry in `$AGENTV_HOME/config.yaml`. `repo` is the Git remote slug or URL AgentV passes to `git clone`, so it can be an `owner/name` slug, HTTPS, or SSH. `branch` is the branch or ref to check out, and `path` is the local checkout path: ```yaml projects: - id: my-evals - name: My Evals - repo_url: https://github.com/example/my-evals.git + repo: https://github.com/example/my-evals.git path: /srv/agentv/my-evals - ref: main + branch: main ``` On each Dashboard startup, AgentV clones the repo if the path is empty (`git clone --depth 1`) or pulls the latest if a clone already exists (`git pull --ff-only`). You can also trigger a sync manually from the Dashboard UI's **Sync** button. @@ -235,6 +238,8 @@ AGENTV_HOME=/tmp/agentv-home-a agentv dashboard --port 3117 AGENTV_HOME=/tmp/agentv-home-b agentv dashboard --port 3118 ``` +For local Docker development, prefer building from the latest AgentV checkout and running `bun` inside that image instead of depending on a globally installed or npm-installed `agentv` binary. A setup script should mount a writable `AGENTV_HOME`, project checkouts, and any results repo paths, publish the dashboard port on `0.0.0.0` when Tailscale access is needed, then start the dashboard with the local build, for example `bun apps/cli/dist/cli.js dashboard`. Keep this Docker path separate from Kubernetes or Helm deployment assets unless those assets are actually needed to model the local runtime. + The landing page shows a card for each project with run count, pass rate, and last run time. AgentV Dashboard projects dashboard showing project cards with pass rates @@ -251,7 +256,7 @@ IDs are derived from the directory name (e.g., `/home/user/repos/my-evals` becom ## Remote Results -Dashboard can display runs pushed to a remote git repository by other machines or CI alongside your local runs. Each run in the list carries a source badge: **local** (green) or **remote** (amber). For in-progress eval durability before final publish, AgentV writes [WIP checkpoints](/docs/v4.42.4/tools/wip-checkpoints/) to `agentv/wip/...` branches; Dashboard lists them only after they are recovered locally or published to the normal results branch. +Dashboard can display runs pushed to a remote git repository by other machines or CI alongside your local runs. Each run in the list carries a source badge: **local** (green) or **remote** (amber). For in-progress eval durability before final publish, AgentV writes [WIP checkpoints](/docs/tools/wip-checkpoints/) to `agentv/wip/...` branches; Dashboard lists them only after they are recovered locally or published to the normal results branch. ### Configuration @@ -260,56 +265,49 @@ For a registered project, put results repo settings on that project's entry in ` ```yaml projects: - id: agentv - name: AgentV - repo_url: https://github.com/EntityProcess/agentv.git + repo: https://github.com/EntityProcess/agentv.git path: /home/entity/projects/EntityProcess/agentv - ref: main + branch: main results: - repo_path: . + repo: https://github.com/EntityProcess/agentv.git + path: /home/entity/projects/EntityProcess/agentv branch: agentv/results/v1 - remote: origin - sync: - auto_push: false - require_push: false + auto_push: false ``` -`results.repo_path: .` stores completed run artifacts on a dedicated branch of the source repository without checking out that branch in the source worktree. The name is intentionally `repo_path`, not `repo`: in AgentV, `workspace.repos[].repo` means a portable repository identity such as a clone URL or `org/name`, while `results.repo_path` means an existing local Git checkout whose object database and refs AgentV should write to. The branch defaults to `agentv/results/v1` for `repo_path` configs. AgentV creates the branch automatically on first publish and commits only `.agentv/results/**` paths into it. `sync.auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. `sync.require_push: true` is for CI workflows where a push failure should fail the command after local artifacts are written. +`results.repo` is the Git remote slug or URL used when AgentV creates a fresh results checkout, and the intended remote URL for portable project config. `results.path` is the local Git checkout AgentV writes result commits into; pointing it at the source repository checkout stores completed run artifacts on a dedicated branch of that repository. AgentV does not add or rewrite remotes inside an existing checkout; the checkout's existing `origin` must already point at the repository you want to fetch and push. When `results.repo` is omitted, `results.path` means an existing local Git checkout whose object database and refs AgentV should write to, and the branch defaults to `agentv/results/v1`. AgentV creates the branch automatically on first publish and commits only AgentV result paths into it. `auto_push: false` keeps the result commit local; set it to `true` to push the branch best-effort after each completed run. For CI workflows where a push failure should fail the command after local artifacts are written, invoke the run with `agentv eval run --results-require-push`. The default conflict behavior is block-and-ask, because AgentV never force-pushes result branches. Non-fast-forward result branch pushes are auto-merged with artifact-aware Git merge drivers and pushed as a fast-forward, so the canonical results branch is never force-pushed or rewritten. Genuine overlay conflicts route to a timestamped temp branch plus a GitHub compare link for a human merge instead. -For a separate results repository, use `repo_url` and an optional managed clone `path`: +For a separate results repository, set `results.repo` and an optional managed clone `results.path`: ```yaml projects: - id: agentv - name: AgentV path: /home/entity/projects/EntityProcess/agentv results: - repo_url: git@github.com:EntityProcess/agentv-examples-eval-results.git - branch: agentv/results/v1 + repo: git@github.com:EntityProcess/agentv-examples-eval-results.git path: /home/entity/projects/EntityProcess/agentv-examples-eval-results - sync: - auto_push: true + branch: agentv/results/v1 + auto_push: true ``` -`results.repo_url` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. `results.path` is the filesystem location of the local clone AgentV manages for that remote results repo. Use `repo_path` only when the results store is an already-existing local checkout such as `.`. +`results.repo` is the Git remote slug or URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured. When `results.repo` is set and `results.path` is missing or empty, AgentV creates that filesystem location with `git clone`. If `results.path` already points at a Git checkout, AgentV treats that checkout's remotes as user-owned state: it fetches and pushes using the existing configured remote name (`origin` by default), but it does not run `git remote add` or `git remote set-url`. Omit `results.repo` only when `results.path` points at an already-existing local checkout. You can also set a top-level global fallback in the same file. This is used when the current project is not registered or its registry entry has no `results` block: ```yaml results: - repo_path: . + repo: https://github.com/EntityProcess/agentv.git + path: /home/entity/projects/EntityProcess/agentv branch: agentv/results/v1 - remote: origin - sync: - auto_push: false - require_push: false + auto_push: false ``` -Project-local `.agentv/config.yaml` is for portable eval defaults such as `execution`, `eval_patterns`, and `dashboard`. Do not put `projects` in project-local config; AgentV warns and ignores it there. `results_by_project` is deprecated; use `projects[].results` in `$AGENTV_HOME/config.yaml`. +Project-local `.agentv/config.yaml` is for portable eval defaults such as `execution`, `eval_patterns`, and `dashboard`. Do not put `projects` in project-local config; AgentV warns and ignores it there. Put per-project results settings in `projects[].results` in `$AGENTV_HOME/config.yaml`. -The project `repo_url` and the `results` block sync different repositories: +The project `repo` and the `results` block sync different repositories: -- `projects[].repo_url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. -- `projects[].results.repo_url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone. +- `projects[].repo` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current. +- `projects[].results.repo` is the git-backed results store remote URL. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in the local checkout at `projects[].results.path`. #### Migration from the legacy project schema @@ -318,7 +316,6 @@ Before: ```yaml projects: - id: agentv - name: AgentV path: /home/entity/projects/EntityProcess/agentv source: url: https://github.com/EntityProcess/agentv @@ -335,33 +332,25 @@ After: ```yaml projects: - id: agentv - name: AgentV - repo_url: https://github.com/EntityProcess/agentv.git + repo: https://github.com/EntityProcess/agentv.git path: /home/entity/projects/EntityProcess/agentv - ref: main + branch: main results: - repo_url: https://github.com/EntityProcess/agentv-eval-results.git - branch: agentv/results/v1 + repo: https://github.com/EntityProcess/agentv-eval-results.git path: /home/entity/projects/EntityProcess/agentv-eval-results - sync: - auto_push: true + branch: agentv/results/v1 + auto_push: true ``` -Legacy project fields (`source`, `repository`, `results.mode`, `results.repo`, `results.repository`, `results.local_path`, and `results.auto_push`) fail validation with migration guidance. +Both the source repo and the `results` block use one flat shape: `repo` (slug or Git URL), `path` (local checkout), `branch`, and, for results, `auto_push`. Removed fields (`source`, `repository`, the nested `repo:`/`results.repo:` objects, `repo_url`, `repo_path`, `ref`, `results.remote`, `results.repository`, `results.local_path`, `results.sync`, `results.branch_prefix`, and `results.push_conflict_policy`) fail validation. Use project-level **Sync Project** as the results exchange workflow. It handles pulled remote runs, locally edited metadata, dirty state, and blocked conflict feedback in one project-scoped action. There is no separate `agentv results remote status` or `agentv results remote sync` command. The `agentv results` CLI stays focused on local run workspaces; manual remote exchange is Dashboard/API-only, with eval auto-export covering the common CI/publisher path. -Each run writes to a unique timestamped directory, so concurrent pushes from multiple machines are safe. AgentV creates a missing storage branch automatically and pushes with a non-fast-forward retry. `branch_prefix` remains only the prefix for temporary result/PR branch names; it is not the storage branch. - -### What happens to existing local runs? - -Existing runs already present under `.agentv/results/runs/` stay exactly where they are and continue to appear in Dashboard as **local** runs. - -Adding a `results` block does **not** backfill those historical runs into the results branch automatically. Result publishing only affects runs created after the results repo is configured. `sync.auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. +Each run writes to a unique run-id directory, so concurrent pushes from multiple machines are safe. AgentV creates a missing storage branch automatically and pushes with a non-fast-forward retry. Temporary result/PR branch names use a fixed prefix; they are not the storage branch. -If you want older local-only runs in the remote repo, rerun them or copy the run directories into the managed clone manually before syncing the project. +Adding a `results` block does not backfill local run workspaces into the results branch automatically. Result publishing affects runs created after the results repo is configured. `auto_push` controls network push and best-effort WIP checkpoints for in-progress `agentv eval` runs. ### Authentication @@ -378,7 +367,7 @@ Automation can use the same API that Dashboard uses: Single-project sessions also expose `GET /api/remote/status` and `POST /api/remote/sync`. -In the default multi-project flow, open a project card first, then use **Sync Project** in that project's toolbar. The toolbar shows the project display name, sync state, last synced time, configured repo, and remote run count. Statuses include clean, unavailable, behind, ahead, dirty, diverged, conflicted, and syncing. +In the default multi-project flow, open a project card first, then use **Sync Project** in that project's toolbar. The toolbar shows the project display name, sync state, last synced time, configured repo, and remote run count. Statuses include clean, unavailable, behind, ahead, dirty, diverged, conflicted, needs human merge, and syncing. Use the **All Sources / Local Only / Remote Only** filter to narrow the run list by origin. @@ -393,8 +382,10 @@ After sync, newly fetched remote runs appear in the list with a **remote** sourc **Sync Project** fetches the results repo and only changes the clone when Git says it is safe: - A clean clone that is behind the remote is fast-forwarded. -- Safe uncommitted changes under `.agentv/results/**`, such as remote tag metadata overlays, are committed and pushed when `sync.auto_push: true`. -- A local results repo that is ahead is pushed when `sync.auto_push: true` and the committed paths are all under `.agentv/results/**`. -- Dirty non-results files, dirty metadata plus remote changes, diverged history, unresolved conflicts, missing upstream branches, non-results commits ahead, and rejected pushes are blocked instead of reset. +- Safe uncommitted changes under the configured results repo's owned result and metadata paths, such as remote tag overlays under `metadata/runs/**`, are committed and pushed when `auto_push: true`. +- A local results repo that is ahead is pushed when `auto_push: true` and the committed paths are all under `.agentv/results/**`. +- Dirty non-results files, dirty metadata plus remote changes, unresolved conflicts, missing upstream branches, non-results commits ahead, and rejected pushes are blocked instead of reset. +- Non-fast-forward result branch pushes never force-push. AgentV runs a bounded fetch → merge → push loop that absorbs concurrent remote writes with a real merge commit using artifact-aware Git merge drivers (union for the append-only `index.jsonl`, a JSON-union driver for tag overlays), so the common append-mostly case auto-merges and pushes as a fast-forward. When Dashboard sync absorbs concurrent remote changes this way, the success feedback includes **Merged remote (auto)**. +- When a genuine overlay conflict cannot be auto-merged, AgentV does not touch the canonical branch. It pushes the local work to a fresh timestamped `agentv/results-sync/--` branch and reports `needs_human_merge` with a `pending_merge` block (temp branch, target branch, and a GitHub compare URL when the remote is on GitHub). The toolbar shows a **Pending merge** card: open the link to merge the branch into the canonical target on GitHub (GitHub's pull request is the conflict surface — AgentV builds no merge UI), then click **I merged it — resync**. That resumes canonical sync by fast-forward-pulling the merged target. A premature click is a safe no-op — local work stays intact and the next sync re-creates a temp branch. When sync is blocked, Dashboard keeps the local clone intact and shows the `block_reason`, `dirty_paths` or `conflicted_paths`, `git_status`, and a compact `git_diff_summary` so you can resolve the results repo manually before syncing again. diff --git a/examples/features/readme-quickstart/README.md b/examples/features/readme-quickstart/README.md index b8bba8be7..6866c6f46 100644 --- a/examples/features/readme-quickstart/README.md +++ b/examples/features/readme-quickstart/README.md @@ -24,6 +24,5 @@ Run it against a local OpenAI-compatible endpoint: LOCAL_OPENAI_PROXY_BASE_URL=http://127.0.0.1:10531/v1 \ LOCAL_OPENAI_PROXY_API_KEY=dummy-local-key \ LOCAL_OPENAI_PROXY_MODEL=gpt-5.3-codex-spark \ -bun apps/cli/src/cli.ts eval examples/features/readme-quickstart/evals/my-eval.eval.yaml \ - --targets examples/features/readme-quickstart/targets.yaml +bun apps/cli/src/cli.ts eval examples/features/readme-quickstart/evals/my-eval.eval.yaml ``` diff --git a/examples/features/readme-quickstart/evals/my-eval.eval.yaml b/examples/features/readme-quickstart/evals/my-eval.eval.yaml index f91ee6cc0..86a1700d6 100644 --- a/examples/features/readme-quickstart/evals/my-eval.eval.yaml +++ b/examples/features/readme-quickstart/evals/my-eval.eval.yaml @@ -2,7 +2,7 @@ description: Code generation quality tags: experiment: with-skills target: local-openai -evaluate_options: +execution: max_concurrency: 1 default_test: file://./default-test.yaml diff --git a/examples/features/readme-quickstart/targets.yaml b/examples/features/readme-quickstart/targets.yaml index 1a7d8cbd2..3299f646a 100644 --- a/examples/features/readme-quickstart/targets.yaml +++ b/examples/features/readme-quickstart/targets.yaml @@ -1,8 +1,9 @@ targets: - - label: local-openai + - id: local-openai provider: openai - api_format: chat - base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} - api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} - model: ${{ LOCAL_OPENAI_PROXY_MODEL }} - + runtime: host + config: + api_format: chat + base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }} + api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }} + model: ${{ LOCAL_OPENAI_PROXY_MODEL }} From 4c6a58b3b0b02ae6934dc060319d06b6007ba97c Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 20:36:56 +0200 Subject: [PATCH 09/10] test: align runtime provider fixtures with config contract --- .../core/test/evaluation/cache-config.test.ts | 2 +- .../evaluation/providers/codex-sdk.test.ts | 30 +++++------ .../evaluation/providers/copilot-sdk.test.ts | 4 +- .../evaluation/providers/pi-runtime.test.ts | 51 ++++++++++++------- 4 files changed, 50 insertions(+), 37 deletions(-) diff --git a/packages/core/test/evaluation/cache-config.test.ts b/packages/core/test/evaluation/cache-config.test.ts index 3c0e56588..6641940da 100644 --- a/packages/core/test/evaluation/cache-config.test.ts +++ b/packages/core/test/evaluation/cache-config.test.ts @@ -11,6 +11,6 @@ describe('extractCacheConfig', () => { it('rejects authored execution blocks', () => { const suite: JsonObject = { execution: { target: 'default' } }; - expect(() => extractCacheConfig(suite)).toThrow(/Top-level 'execution'/); + expect(() => extractCacheConfig(suite)).toThrow(/Top-level 'execution\.target'/); }); }); diff --git a/packages/core/test/evaluation/providers/codex-sdk.test.ts b/packages/core/test/evaluation/providers/codex-sdk.test.ts index b117a7886..058ae1597 100644 --- a/packages/core/test/evaluation/providers/codex-sdk.test.ts +++ b/packages/core/test/evaluation/providers/codex-sdk.test.ts @@ -86,7 +86,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); const request: ProviderRequest = { question: 'Write hello world', @@ -121,7 +121,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); const provider = new CodexProvider('test-target', { - executable: 'codex', + command: ['codex'], model: 'o4-mini', }); @@ -155,7 +155,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); const provider = new CodexProvider('test-target', { - executable: 'codex', + command: ['codex'], model: 'gpt-5.3-codex-spark', modelReasoningEffort: 'medium', modelVerbosity: 'medium', @@ -190,7 +190,7 @@ describe('CodexProvider (SDK)', () => { expect(threadOptions.approvalPolicy).toBe('never'); }); - it('passes executable config to Codex constructor as codexPathOverride', async () => { + it('passes command executable to Codex constructor as codexPathOverride', async () => { const thread = createMockThread({ events: [ { @@ -213,7 +213,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); const provider = new CodexProvider('test-target', { - executable: 'codex-eng', + command: ['codex-eng'], }); await provider.invoke({ question: 'Test' }); @@ -243,7 +243,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); const provider = new CodexProvider('test-target', { - executable: 'codex', + command: ['codex'], cwd: '/tmp/test-workspace', }); @@ -275,7 +275,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); const provider = new CodexProvider('test-target', { - executable: 'codex', + command: ['codex'], modelReasoningEffort: 'low', }); @@ -307,7 +307,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); const provider = new CodexProvider('test-target', { - executable: 'codex', + command: ['codex'], timeoutMs: 100, }); @@ -333,7 +333,7 @@ describe('CodexProvider (SDK)', () => { mock.module('@openai/codex-sdk', () => sdkMock); const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); const controller = new AbortController(); controller.abort(); @@ -361,7 +361,7 @@ describe('CodexProvider (SDK)', () => { mock.module('@openai/codex-sdk', () => sdkMock); const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); const response = await provider.invoke({ question: 'Tokens' }); @@ -401,7 +401,7 @@ describe('CodexProvider (SDK)', () => { mock.module('@openai/codex-sdk', () => sdkMock); const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); const response = await provider.invoke({ question: 'List files' }); @@ -444,7 +444,7 @@ describe('CodexProvider (SDK)', () => { mock.module('@openai/codex-sdk', () => sdkMock); const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); const response = await provider.invoke({ question: 'Update file' }); @@ -473,7 +473,7 @@ describe('CodexProvider (SDK)', () => { mock.module('@openai/codex-sdk', () => sdkMock); const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); const response = await provider.invoke({ question: 'Timing' }); expect(response.startTime).toBeDefined(); @@ -497,7 +497,7 @@ describe('CodexProvider (SDK)', () => { mock.module('@openai/codex-sdk', () => sdkMock); const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); await expect(provider.invoke({ question: 'Fail' })).rejects.toThrow(/turn failed/i); }); @@ -524,7 +524,7 @@ describe('CodexProvider (SDK)', () => { const { CodexProvider } = await import('../../../src/evaluation/providers/codex.js'); - const provider = new CodexProvider('test-target', { executable: 'codex' }); + const provider = new CodexProvider('test-target', { command: ['codex'] }); await provider.invoke({ question: 'First' }); await provider.invoke({ question: 'Second' }); diff --git a/packages/core/test/evaluation/providers/copilot-sdk.test.ts b/packages/core/test/evaluation/providers/copilot-sdk.test.ts index 10e7f5ba6..a784486fd 100644 --- a/packages/core/test/evaluation/providers/copilot-sdk.test.ts +++ b/packages/core/test/evaluation/providers/copilot-sdk.test.ts @@ -297,7 +297,7 @@ describe('CopilotSdkProvider', () => { mock.module('@github/copilot-sdk', () => sdkMock); const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); - const provider = new CopilotSdkProvider('test-target', {}); + const provider = new CopilotSdkProvider('test-target', { cliPath: 'copilot-native' }); await provider.invoke({ question: 'Test' }); @@ -394,7 +394,7 @@ describe('CopilotSdkProvider', () => { })); const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js'); - const provider = new CopilotSdkProvider('test-target', {}); + const provider = new CopilotSdkProvider('test-target', { cliPath: 'copilot-native' }); await provider.invoke({ question: 'First' }); await provider.invoke({ question: 'Second' }); diff --git a/packages/core/test/evaluation/providers/pi-runtime.test.ts b/packages/core/test/evaluation/providers/pi-runtime.test.ts index 961d49a5c..51e7598c2 100644 --- a/packages/core/test/evaluation/providers/pi-runtime.test.ts +++ b/packages/core/test/evaluation/providers/pi-runtime.test.ts @@ -1,4 +1,7 @@ import { describe, expect, it, mock } from 'bun:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { createProvider, @@ -15,6 +18,7 @@ import { extractLastAssistantContent } from '../../../src/evaluation/providers/t describe('Pi coding-agent runtime providers', () => { it('passes config.command argv and profile runtime env to pi-cli subprocesses', async () => { + const workspace = await mkdtemp(path.join(tmpdir(), 'agentv-pi-cli-test-')); let captured: PiProcessRunOptions | undefined; const runner = mock(async (options: PiProcessRunOptions) => { captured = options; @@ -29,35 +33,44 @@ describe('Pi coding-agent runtime providers', () => { }); const provider = new PiCliProvider('pi-cli-target', baseCliConfig(), runner); - const response = await provider.invoke({ question: 'hello', cwd: '/tmp/workspace' }); - - expect(extractLastAssistantContent(response.output)).toBe('cli ok'); - expect(captured?.command.slice(0, 3)).toEqual(['pi-shim', '--profile', 'clean']); - expect(captured?.command).toContain('--mode'); - expect(captured?.command).toContain('json'); - expect(captured?.env.HOME).toBe('/tmp/pi-profile'); - expect(captured?.env.PI_TEST_FLAG).toBe('enabled'); - expect(response.targetExecution?.status).toBe('success'); - expect(response.targetExecution?.runtimeMode).toBe('profile'); - expect(response.targetExecution?.command?.argv?.slice(0, 3)).toEqual([ - 'pi-shim', - '--profile', - 'clean', - ]); + try { + const response = await provider.invoke({ question: 'hello', cwd: workspace }); + + expect(extractLastAssistantContent(response.output)).toBe('cli ok'); + expect(captured?.command.slice(0, 3)).toEqual(['pi-shim', '--profile', 'clean']); + expect(captured?.command).toContain('--mode'); + expect(captured?.command).toContain('json'); + expect(captured?.env.HOME).toBe('/tmp/pi-profile'); + expect(captured?.env.PI_TEST_FLAG).toBe('enabled'); + expect(response.targetExecution?.status).toBe('success'); + expect(response.targetExecution?.runtimeMode).toBe('profile'); + expect(response.targetExecution?.command?.argv?.slice(0, 3)).toEqual([ + 'pi-shim', + '--profile', + 'clean', + ]); + } finally { + await rm(workspace, { recursive: true, force: true }); + } }); it('returns structured pi-cli malformed-output errors', async () => { + const workspace = await mkdtemp(path.join(tmpdir(), 'agentv-pi-cli-test-')); const provider = new PiCliProvider('pi-cli-target', baseCliConfig(), async () => ({ stdout: 'not json\n', stderr: '', exitCode: 0, })); - const response = await provider.invoke({ question: 'hello', cwd: '/tmp/workspace' }); + try { + const response = await provider.invoke({ question: 'hello', cwd: workspace }); - expect(response.targetExecution?.status).toBe('error'); - expect(response.targetExecution?.errorKind).toBe('malformed_output'); - expect(extractLastAssistantContent(response.output)).toMatch(/malformed output/i); + expect(response.targetExecution?.status).toBe('error'); + expect(response.targetExecution?.errorKind).toBe('malformed_output'); + expect(extractLastAssistantContent(response.output)).toMatch(/malformed output/i); + } finally { + await rm(workspace, { recursive: true, force: true }); + } }); it('runs pi-rpc over process stdio and returns fake RPC success', async () => { From 10f716226ee3b0e3c011100a1b9aa52931de53ff Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 3 Jul 2026 20:44:34 +0200 Subject: [PATCH 10/10] test: use explicit codex sdk provider in CLI fixture --- apps/cli/test/eval.integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 9ad40075c..862147855 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -48,7 +48,7 @@ targets: - label: cli-target provider: mock - label: codex-target - provider: codex + provider: codex-sdk model: gpt-5-default `; await writeFile(targetsPath, targetsContent, 'utf8');