diff --git a/apps/web/src/content/docs/docs/next/evaluation/batch-cli.mdx b/apps/web/src/content/docs/docs/next/evaluation/batch-cli.mdx deleted file mode 100644 index a1c114108..000000000 --- a/apps/web/src/content/docs/docs/next/evaluation/batch-cli.mdx +++ /dev/null @@ -1,277 +0,0 @@ ---- -title: Batch CLI Evaluation -description: Evaluate external tools that process all tests in a single invocation -sidebar: - order: 5 -slug: docs/evaluation/batch-cli ---- - -Batch CLI evaluation handles tools that process multiple inputs at once — bulk classifiers, screening engines, or any runner that reads all tests and outputs results in one pass. - -## Overview - -Use batch CLI evaluation when: - -- An external tool processes multiple inputs in a single invocation (e.g., AML screening, bulk classification) -- The runner reads the eval YAML directly to extract all tests -- Output is JSONL with records keyed by test `id` -- Each test has its own grader to validate its corresponding output record - -## Execution Flow - -1. **AgentV** invokes the batch runner once, passing `--eval ` and `--output ` -2. **Batch runner** reads the eval YAML, extracts all tests, processes them, and writes JSONL output keyed by `id` -3. **AgentV** parses the JSONL and routes each record to its matching test by `id` -4. **Per-test graders** validate the output for each test independently - -## Eval File Structure - -```yaml -description: Batch CLI demo using structured input -providers: - - batch_cli - -prompts: - - "{{ input }}" - -tests: - - id: case-001 - criteria: |- - Batch runner returns JSON with decision=CLEAR. - vars: - input: - - role: system - content: You are a batch processor. - - role: user - content: - request: - type: screening_check - jurisdiction: AU - row: - id: case-001 - name: Example A - amount: 5000 - expected_output: - - role: assistant - content: - decision: CLEAR - - assert: - - name: decision-check - type: script - command: [bun, run, ./scripts/check-output.ts] - cwd: . - - - id: case-002 - criteria: |- - Batch runner returns JSON with decision=REVIEW. - vars: - input: - - role: system - content: You are a batch processor. - - role: user - content: - request: - type: screening_check - jurisdiction: AU - row: - id: case-002 - name: Example B - amount: 25000 - expected_output: - - role: assistant - content: - decision: REVIEW - - assert: - - name: decision-check - type: script - command: [bun, run, ./scripts/check-output.ts] - cwd: . -``` - -## Batch Runner Contract - -The batch runner reads the eval YAML directly and processes all tests in one invocation. - -### Input - -The runner receives the eval file path via `--eval` and an output path via `--output`: - -```bash -bun run batch-runner.ts --eval ./my-eval.yaml --output ./output.jsonl -``` - -### Output - -JSONL where each line is a JSON object with an `id` matching a test: - -```json -{"id": "case-001", "text": "{\"decision\": \"CLEAR\", ...}"} -{"id": "case-002", "text": "{\"decision\": \"REVIEW\", ...}"} -``` - -The `id` field must match the test `id` for AgentV to route output to the correct grader. - -### Output with Trajectory Assertions - -To enable `trajectory:*` assertions, include `output` with `tool_calls`: - -```json -{ - "id": "case-001", - "text": "{\"decision\": \"CLEAR\", ...}", - "output": [ - { - "role": "assistant", - "tool_calls": [ - { - "tool": "screening_check", - "input": { "origin_country": "NZ", "amount": 5000 }, - "output": { "decision": "CLEAR", "reasons": [] } - } - ] - }, - { - "role": "assistant", - "content": { "decision": "CLEAR" } - } - ] -} -``` - -AgentV extracts tool calls directly from `output[].tool_calls[]` for trajectory assertions. - -## Grader Implementation - -Each test has its own grader that validates the batch runner output. The grader receives the standard `script` input via stdin. - -**Input (stdin):** -```json -{ - "output": "{\"id\":\"case-001\",\"decision\":\"CLEAR\",...}", - "expected_output": [{"role": "assistant", "content": {"decision": "CLEAR"}}], - "input": [...] -} -``` - -**Output (stdout):** -```json -{ - "pass": true, - "score": 1.0, - "reason": "Batch runner decision matches expected.", - "checks": [ - { "text": "decision matches: CLEAR", "pass": true, "reason": "Expected and actual decisions are CLEAR." } - ] -} -``` - -### Example Grader - -```typescript -import fs from 'node:fs'; - -type EvalInput = { - output?: string; - expected_output?: Array<{ role: string; content: unknown }>; -}; - -function main() { - const stdin = fs.readFileSync(0, 'utf8'); - const input = JSON.parse(stdin) as EvalInput; - - const expectedDecision = findExpectedDecision(input.expected_output); - - let candidateDecision: string | undefined; - try { - const parsed = JSON.parse(input.output ?? ''); - candidateDecision = parsed.decision; - } catch { - candidateDecision = undefined; - } - - const checks: Array<{ text: string; pass: boolean; reason: string }> = []; - - if (expectedDecision === candidateDecision) { - checks.push({ text: `decision matches: ${expectedDecision}`, pass: true, reason: 'Expected and actual decisions match.' }); - } else { - checks.push({ text: `mismatch: expected=${expectedDecision} actual=${candidateDecision}`, pass: false, reason: 'Expected and actual decisions differ.' }); - } - - const passed = checks.every((check) => check.pass); - - process.stdout.write(JSON.stringify({ - pass: passed, - score: passed ? 1 : 0, - reason: passed - ? 'Batch runner output matches expected.' - : 'Batch runner output did not match expected.', - checks, - })); -} - -function findExpectedDecision(messages?: Array<{ role: string; content: unknown }>) { - if (!messages) return undefined; - for (const msg of messages) { - if (typeof msg.content === 'object' && msg.content !== null) { - return (msg.content as Record).decision as string; - } - } - return undefined; -} - -main(); -``` - -## Structured Content - -Use structured objects in `vars.expected_output` to define expected output fields for easy validation: - -```yaml -vars: - expected_output: - - role: assistant - content: - decision: CLEAR - confidence: high - reasons: [] -``` - -The grader extracts these fields and compares them against the parsed candidate output. - -## Target Configuration - -Configure the batch CLI provider in your targets file or eval file: - -```yaml -# In agentv-providers.yaml or eval file -providers: - batch_cli: - provider: cli - command: bun run ./scripts/batch-runner.ts --eval {EVAL_FILE} --output {OUTPUT_FILE} - batch_requests: true -``` - -Key settings: - -| Setting | Description | -|---------|-------------| -| `provider: cli` | Use the CLI provider | -| `batch_requests: true` | Run once for all tests instead of per-test | -| `{EVAL_FILE}` | Placeholder replaced with the eval file path | -| `{OUTPUT_FILE}` | Placeholder replaced with the JSONL output path | - -## Best Practices - -1. **Use unique test IDs** -- the batch runner and AgentV use `id` to route outputs to the correct grader -2. **Structured input** -- put structured data in `user.content` for the runner to extract -3. **Structured expected_output** -- define expected output as objects for easy comparison -4. **Deterministic runners** -- batch runners should produce consistent output for reliable testing -5. **Healthcheck support** -- add a `--healthcheck` flag for runner validation: - ```typescript - if (args.includes('--healthcheck')) { - console.log('batch-runner: healthy'); - return; - } - ``` diff --git a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx index 4c5e199a6..24dbbbb0b 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/examples.mdx @@ -302,92 +302,6 @@ tests: value: "Matches the reference answer: {{ expected_output }}" ```` -## Batch CLI - -Evaluate external batch runners that process all tests in one invocation: - -```yaml -description: Batch CLI demo (AML screening) -providers: - - batch_cli - -prompts: - - "{{ input }}" - -tests: - - id: aml-001 - criteria: |- - Batch runner returns JSON with decision=CLEAR. - vars: - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-001 - customer_name: Example Customer A - origin_country: NZ - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 5000 - currency: USD - expected_output: - - role: assistant - content: - decision: CLEAR - - assert: - - name: decision-check - type: script - command: [bun, run, ./scripts/check-batch-cli-output.ts] - cwd: . - - - id: aml-002 - criteria: |- - Batch runner returns JSON with decision=REVIEW. - vars: - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-002 - customer_name: Example Customer B - origin_country: IR - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 2000 - currency: USD - expected_output: - - role: assistant - content: - decision: REVIEW - - assert: - - name: decision-check - type: script - command: [bun, run, ./scripts/check-batch-cli-output.ts] - cwd: . -``` - -### Batch CLI Pattern Notes - -- `target: batch_cli` -- configure the CLI provider with `batch_requests: true` -- The batch runner reads the eval YAML via `--eval` flag and outputs JSONL keyed by `id` -- Put structured data in `user.content` as objects for the runner to extract -- Use `vars.expected_output` with object fields for structured expected output -- Each test has its own grader to validate its portion of the output - ## Shared Prompt Context Share a common prompt or system instruction across all tests with top-level diff --git a/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx b/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx index 227d87df6..2a63f3269 100644 --- a/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx +++ b/apps/web/src/content/docs/docs/next/targets/cli-provider.mdx @@ -8,7 +8,7 @@ slug: docs/targets/cli-provider The `cli` provider runs an arbitrary shell command per test case and captures its output as the target's response. It's the escape hatch that lets you evaluate *anything* that exposes a command-line entry point — your own agent, a third-party CLI, a stub that prints a fixed answer, a script that calls an in-house microservice, etc. -Because the contract is "we invoke a command and read a file," almost any useful composition pattern (sanity-checking your grader against a known-good answer, diffing two implementations, driving a batch mode) can be built on top without any new primitives. +Because the contract is "we invoke a command and read a file," almost any useful composition pattern (sanity-checking your grader against a known-good answer, diffing two implementations, calling an in-house queue) can be built on top without any new primitives. ## Minimal example @@ -94,23 +94,23 @@ echo "Hello, world!" > {OUTPUT_FILE} | `config.verbose` | boolean | no | `false` | Log the rendered command and cwd to stdout. Useful for debugging template substitution. | | `config.keep_temp_files` | boolean | no | `false` | Preserve `{PROMPT_FILE}` / `{OUTPUT_FILE}` after the run — handy while iterating on your command. | | `config.healthcheck` | object | no | — | Pre-run health check (HTTP or command); the eval aborts if it fails. | -| `config.batch_requests` | boolean | no | `false` | Run all cases in one command invocation — see [Batching](#batching). | -## Batching +## Throughput and queueing -For targets where spin-up cost dominates per-case work (e.g. loading a model, authenticating), set `batch_requests: true`. AgentV invokes the command *once*, hands it a JSONL stream of cases, and expects a JSONL response keyed by each case's `id`: +AgentV invokes a CLI provider once per eval case. Each invocation receives one rendered prompt or prompt file and must write one response to `{OUTPUT_FILE}`. + +If your underlying tool benefits from throughput batching, keep that batching behind the provider boundary. For example, the command can enqueue the case into a long-running service, call a local daemon that batches requests internally, or use the tool's native queueing API. The eval YAML should still describe a normal per-case CLI provider: ```yaml providers: - - id: batched-agent - provider: cli + - id: cli + label: my-agent runtime: host config: - batch_requests: true - command: python agent.py --batch-in {PROMPT_FILE} --batch-out {OUTPUT_FILE} + command: python agent.py --prompt-file {PROMPT_FILE} --out {OUTPUT_FILE} ``` -`{PROMPT_FILE}` contains one JSON object per line with an `id` and the case's inputs; your command writes one line per case to `{OUTPUT_FILE}`, each carrying the matching `id` plus the same output shape as the non-batched case. +The important boundary is that AgentV still receives one result per case. Internal queueing, caching, daemon reuse, or request batching belongs in the adapter or CLI implementation, not in eval-runner batch configuration. ## Pattern: Oracle validation (sanity-check your grader) diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/batch-cli.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/batch-cli.mdx deleted file mode 100644 index 8f8038e9a..000000000 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/batch-cli.mdx +++ /dev/null @@ -1,276 +0,0 @@ ---- -title: Batch CLI Evaluation -description: Evaluate external tools that process all tests in a single invocation -sidebar: - order: 5 -slug: docs/v4.42.4/evaluation/batch-cli -editUrl: false -pagefind: false ---- - -Batch CLI evaluation handles tools that process multiple inputs at once — bulk classifiers, screening engines, or any runner that reads all tests and outputs results in one pass. - -## Overview - -Use batch CLI evaluation when: - -- An external tool processes multiple inputs in a single invocation (e.g., AML screening, bulk classification) -- The runner reads the eval YAML directly to extract all tests -- Output is JSONL with records keyed by test `id` -- Each test has its own grader to validate its corresponding output record - -## Execution Flow - -1. **AgentV** invokes the batch runner once, passing `--eval ` and `--output ` -2. **Batch runner** reads the eval YAML, extracts all tests, processes them, and writes JSONL output keyed by `id` -3. **AgentV** parses the JSONL and routes each record to its matching test by `id` -4. **Per-test graders** validate the output for each test independently - -## Eval File Structure - -```yaml -description: Batch CLI demo using structured input -execution: - target: batch_cli - -tests: - - id: case-001 - criteria: |- - Batch runner returns JSON with decision=CLEAR. - - expected_output: - - role: assistant - content: - decision: CLEAR - - input: - - role: system - content: You are a batch processor. - - role: user - content: - request: - type: screening_check - jurisdiction: AU - row: - id: case-001 - name: Example A - amount: 5000 - - assertions: - - name: decision-check - type: code-grader - command: [bun, run, ./scripts/check-output.ts] - cwd: . - - - id: case-002 - criteria: |- - Batch runner returns JSON with decision=REVIEW. - - expected_output: - - role: assistant - content: - decision: REVIEW - - input: - - role: system - content: You are a batch processor. - - role: user - content: - request: - type: screening_check - jurisdiction: AU - row: - id: case-002 - name: Example B - amount: 25000 - - assertions: - - name: decision-check - type: code-grader - command: [bun, run, ./scripts/check-output.ts] - cwd: . -``` - -## Batch Runner Contract - -The batch runner reads the eval YAML directly and processes all tests in one invocation. - -### Input - -The runner receives the eval file path via `--eval` and an output path via `--output`: - -```bash -bun run batch-runner.ts --eval ./my-eval.yaml --output ./output.jsonl -``` - -### Output - -JSONL where each line is a JSON object with an `id` matching a test: - -```json -{"id": "case-001", "text": "{\"decision\": \"CLEAR\", ...}"} -{"id": "case-002", "text": "{\"decision\": \"REVIEW\", ...}"} -``` - -The `id` field must match the test `id` for AgentV to route output to the correct grader. - -### Output with Tool Trajectory - -To enable `tool_trajectory` evaluation, include `output` with `tool_calls`: - -```json -{ - "id": "case-001", - "text": "{\"decision\": \"CLEAR\", ...}", - "output": [ - { - "role": "assistant", - "tool_calls": [ - { - "tool": "screening_check", - "input": { "origin_country": "NZ", "amount": 5000 }, - "output": { "decision": "CLEAR", "reasons": [] } - } - ] - }, - { - "role": "assistant", - "content": { "decision": "CLEAR" } - } - ] -} -``` - -AgentV extracts tool calls directly from `output[].tool_calls[]` for `tool_trajectory` graders. - -## Grader Implementation - -Each test has its own grader that validates the batch runner output. The grader receives the standard `code_grader` input via stdin. - -**Input (stdin):** -```json -{ - "output": "{\"id\":\"case-001\",\"decision\":\"CLEAR\",...}", - "expected_output": [{"role": "assistant", "content": {"decision": "CLEAR"}}], - "input": [...] -} -``` - -**Output (stdout):** -```json -{ - "score": 1.0, - "assertions": [ - { "text": "decision matches: CLEAR", "passed": true } - ], - "reasoning": "Batch runner decision matches expected." -} -``` - -### Example Grader - -```typescript -import fs from 'node:fs'; - -type EvalInput = { - output?: string; - expected_output?: Array<{ role: string; content: unknown }>; -}; - -function main() { - const stdin = fs.readFileSync(0, 'utf8'); - const input = JSON.parse(stdin) as EvalInput; - - const expectedDecision = findExpectedDecision(input.expected_output); - - let candidateDecision: string | undefined; - try { - const parsed = JSON.parse(input.output ?? ''); - candidateDecision = parsed.decision; - } catch { - candidateDecision = undefined; - } - - const assertions: Array<{ text: string; passed: boolean }> = []; - - if (expectedDecision === candidateDecision) { - assertions.push({ text: `decision matches: ${expectedDecision}`, passed: true }); - } else { - assertions.push({ text: `mismatch: expected=${expectedDecision} actual=${candidateDecision}`, passed: false }); - } - - const passed = assertions.every(a => a.passed); - - process.stdout.write(JSON.stringify({ - score: passed ? 1 : 0, - assertions, - reasoning: passed - ? 'Batch runner output matches expected.' - : 'Batch runner output did not match expected.', - })); -} - -function findExpectedDecision(messages?: Array<{ role: string; content: unknown }>) { - if (!messages) return undefined; - for (const msg of messages) { - if (typeof msg.content === 'object' && msg.content !== null) { - return (msg.content as Record).decision as string; - } - } - return undefined; -} - -main(); -``` - -## Structured Content - -Use structured objects in `expected_output` to define expected output fields for easy validation: - -```yaml -expected_output: - - role: assistant - content: - decision: CLEAR - confidence: high - reasons: [] -``` - -The grader extracts these fields and compares them against the parsed candidate output. - -## Provider Configuration - -Configure the batch CLI provider in your providers file or eval file: - -```yaml -# In .agentv/providers.yaml or an eval file -providers: - - id: cli - label: batch-cli - config: - command: bun run ./scripts/batch-runner.ts --eval {EVAL_FILE} --output {OUTPUT_FILE} - provider_batching: true -``` - -Key settings: - -| Setting | Description | -|---------|-------------| -| `id: cli` | Use the CLI provider | -| `provider_batching: true` | Run once for all tests instead of per-test | -| `{EVAL_FILE}` | Placeholder replaced with the eval file path | -| `{OUTPUT_FILE}` | Placeholder replaced with the JSONL output path | - -## Best Practices - -1. **Use unique test IDs** -- the batch runner and AgentV use `id` to route outputs to the correct grader -2. **Structured input** -- put structured data in `user.content` for the runner to extract -3. **Structured expected_output** -- define expected output as objects for easy comparison -4. **Deterministic runners** -- batch runners should produce consistent output for reliable testing -5. **Healthcheck support** -- add a `--healthcheck` flag for runner validation: - ```typescript - if (args.includes('--healthcheck')) { - console.log('batch-runner: healthy'); - return; - } - ``` diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx index 8dd45bfb6..f6f2e28d1 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/examples.mdx @@ -281,91 +281,6 @@ tests: ``` ```` -## Batch CLI - -Evaluate external batch runners that process all tests in one invocation: - -```yaml -description: Batch CLI demo (AML screening) -execution: - target: batch_cli - -tests: - - id: aml-001 - criteria: |- - Batch runner returns JSON with decision=CLEAR. - - expected_output: - - role: assistant - content: - decision: CLEAR - - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-001 - customer_name: Example Customer A - origin_country: NZ - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 5000 - currency: USD - - assertions: - - name: decision-check - type: code-grader - command: [bun, run, ./scripts/check-batch-cli-output.ts] - cwd: . - - - id: aml-002 - criteria: |- - Batch runner returns JSON with decision=REVIEW. - - expected_output: - - role: assistant - content: - decision: REVIEW - - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-002 - customer_name: Example Customer B - origin_country: IR - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 2000 - currency: USD - - assertions: - - name: decision-check - type: code-grader - command: [bun, run, ./scripts/check-batch-cli-output.ts] - cwd: . -``` - -### Batch CLI Pattern Notes - -- `execution.target: batch_cli` -- configure CLI provider with `provider_batching: true` -- The batch runner reads the eval YAML via `--eval` flag and outputs JSONL keyed by `id` -- Put structured data in `user.content` as objects for the runner to extract -- Use `expected_output` with object fields for structured expected output -- Each test has its own grader to validate its portion of the output - ## Suite-level Input Share a common prompt or system instruction across all tests. Suite-level `input` messages are prepended to each test's input — like suite-level `assertions` for graders: diff --git a/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx b/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx index 669dcab85..7ded38b8b 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/targets/cli-provider.mdx @@ -10,7 +10,7 @@ pagefind: false The `cli` provider runs an arbitrary shell command per test case and captures its output as the provider response. It's the escape hatch that lets you evaluate *anything* that exposes a command-line entry point — your own agent, a third-party CLI, a stub that prints a fixed answer, a script that calls an in-house microservice, etc. -Because the contract is "we invoke a command and read a file," almost any useful composition pattern (sanity-checking your grader against a known-good answer, diffing two implementations, driving a batch mode) can be built on top without any new primitives. +Because the contract is "we invoke a command and read a file," almost any useful composition pattern (sanity-checking your grader against a known-good answer, diffing two implementations, calling an in-house queue) can be built on top without any new primitives. ## Minimal example @@ -96,23 +96,23 @@ echo "Hello, world!" > {OUTPUT_FILE} | `config.verbose` | boolean | no | `false` | Log the rendered command and cwd to stdout. Useful for debugging template substitution. | | `config.keep_temp_files` | boolean | no | `false` | Preserve `{PROMPT_FILE}` / `{OUTPUT_FILE}` after the run — handy while iterating on your command. | | `config.healthcheck` | object | no | — | Pre-run health check (HTTP or command); the eval aborts if it fails. | -| `config.batch_requests` | boolean | no | `false` | Run all cases in one command invocation — see [Batching](#batching). | -## Batching +## Throughput and queueing -For providers where spin-up cost dominates per-case work (e.g. loading a model, authenticating), set `batch_requests: true`. AgentV invokes the command *once*, hands it a JSONL stream of cases, and expects a JSONL response keyed by each case's `id`: +AgentV invokes a CLI provider once per eval case. Each invocation receives one rendered prompt or prompt file and must write one response to `{OUTPUT_FILE}`. + +If your underlying tool benefits from throughput batching, keep that batching behind the provider boundary. For example, the command can enqueue the case into a long-running service, call a local daemon that batches requests internally, or use the tool's native queueing API. The eval YAML should still describe a normal per-case CLI provider: ```yaml providers: - id: cli - label: batched-agent + label: my-agent runtime: host config: - batch_requests: true - command: python agent.py --batch-in {PROMPT_FILE} --batch-out {OUTPUT_FILE} + command: python agent.py --prompt-file {PROMPT_FILE} --out {OUTPUT_FILE} ``` -`{PROMPT_FILE}` contains one JSON object per line with an `id` and the case's inputs; your command writes one line per case to `{OUTPUT_FILE}`, each carrying the matching `id` plus the same output shape as the non-batched case. +The important boundary is that AgentV still receives one result per case. Internal queueing, caching, daemon reuse, or request batching belongs in the adapter or CLI implementation, not in eval-runner batch configuration. ## Pattern: Oracle validation (sanity-check your grader) diff --git a/apps/web/src/data/docs-v4.42.4-routes.json b/apps/web/src/data/docs-v4.42.4-routes.json index 511485838..def1d9c9e 100644 --- a/apps/web/src/data/docs-v4.42.4-routes.json +++ b/apps/web/src/data/docs-v4.42.4-routes.json @@ -1,6 +1,5 @@ [ "/docs/v4.42.4/", - "/docs/v4.42.4/evaluation/batch-cli/", "/docs/v4.42.4/evaluation/eval-cases/", "/docs/v4.42.4/evaluation/eval-files/", "/docs/v4.42.4/evaluation/examples/", diff --git a/examples/README.md b/examples/README.md index a249f6981..c5e88ed64 100644 --- a/examples/README.md +++ b/examples/README.md @@ -48,7 +48,6 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it - [weighted-graders](features/weighted-graders/) - Weighted graders - [execution-metrics](features/execution-metrics/) - Metrics tracking (tokens, cost, latency) - [script-grader-with-llm-calls](features/script-grader-with-llm-calls/) - script graders with provider proxy for LLM calls -- [batch-cli](features/batch-cli/) - Batch CLI evaluation - [provider-owned-batching](features/provider-owned-batching/) - CLI adapter-owned request batching behind normal per-case provider calls - [document-extraction](features/document-extraction/) - Document data extraction - [local-cli](features/local-cli/) - Local CLI providers diff --git a/examples/features/README.md b/examples/features/README.md index 5186cb0ab..dc60fffbd 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -107,7 +107,6 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [file-changes](file-changes/) | Capture workspace file changes made by the agent across test runs | | [file-changes-graders](file-changes-graders/) | Grade file diffs with rubrics and LLM graders | | [local-cli](local-cli/) | Define and invoke local CLI providers | -| [batch-cli](batch-cli/) | Run bulk evaluations from the CLI | --- @@ -138,7 +137,6 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [assert-extended](assert-extended/) | Deterministic assertions | | [basic](basic/) | Getting started | | [basic-jsonl](basic-jsonl/) | Getting started | -| [batch-cli](batch-cli/) | Workspace & providers | | [benchmark-tooling](benchmark-tooling/) | Benchmarking | | [script-grader-sdk](script-grader-sdk/) | Custom graders | | [script-grader-with-llm-calls](script-grader-with-llm-calls/) | Custom graders | diff --git a/examples/features/batch-cli/.agentv/providers.yaml b/examples/features/batch-cli/.agentv/providers.yaml deleted file mode 100644 index d299122e6..000000000 --- a/examples/features/batch-cli/.agentv/providers.yaml +++ /dev/null @@ -1,11 +0,0 @@ -providers: - - id: cli - batch_requests: true - verbose: true - command: bun run ./scripts/batch-cli-runner.ts --eval-input ./evals/suite.yaml - --csv-input ./AmlScreeningInput.csv --output {OUTPUT_FILE} - cwd: .. - healthcheck: - command: bun run ./scripts/batch-cli-runner.ts --healthcheck - cwd: .. - label: batch_cli diff --git a/examples/features/batch-cli/.gitignore b/examples/features/batch-cli/.gitignore deleted file mode 100644 index f9f26bd16..000000000 --- a/examples/features/batch-cli/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Generated by the batch runner during example runs -agentv-evalresult.jsonl -AmlScreeningInput.csv \ No newline at end of file diff --git a/examples/features/batch-cli/README.md b/examples/features/batch-cli/README.md deleted file mode 100644 index 4721c170d..000000000 --- a/examples/features/batch-cli/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Batch CLI example (AML screening, CSV → JSONL) - -This example demonstrates an **external batch runner** pattern for a (synthetic) AML screening use-case. - -## How it works - -1. **Ground truth**: `evals/suite.yaml` contains tests with `input` (structured object content) and `expected_output` (e.g., `content.decision`). - -2. **CSV conversion**: `batch-cli-runner.ts` imports functions from `build-csv-from-eval.ts` to convert `input` into CSV format. The CSV contains only inputs (customer data, transaction details) - no expected decisions. - -3. **Batch processing**: `batch-cli-runner.ts` reads the CSV and applies synthetic AML screening rules, writing **actual responses** as JSONL to a temporary file. Each JSONL record includes `output` with `tool_calls` for trace extraction. - -4. **Evaluation**: AgentV compares the actual JSONL output against the ground truth in `evals/suite.yaml` using graders like `script` and `trajectory:tool-used`. - -## Batch error handling (missing JSONL id) - -This example intentionally includes a test (`aml-004-not-exist`) that is **not written into the CSV input** by `scripts/build-csv-from-eval.ts`. - -That means the batch runner never emits a JSONL record for that `test_id`, and the CLI provider surfaces a provider-side error: - -- `error: "Batch output missing id 'aml-004-not-exist'"` - -AgentV then reports that test as failed (with `error` populated), while still evaluating the other items in the batch. - -## Tool Trajectory via output - -The batch runner outputs JSONL records with `output` containing `tool_calls`: - -```json -{ - "id": "aml-001", - "text": "{...}", - "output": [ - { - "role": "assistant", - "tool_calls": [ - { - "tool": "aml_screening", - "input": { "origin_country": "NZ", ... }, - "output": { "decision": "CLEAR", ... } - } - ] - } - ] -} -``` - -Trajectory assertions extract tool calls directly from `output[].tool_calls[]`. This is the primary format - no separate `trace` field is required. - -## Files - -- `batch-cli-demo.yaml` — Ground truth: tests with inputs and expected outputs -- `scripts/build-csv-from-eval.ts` — Utilities to convert YAML tests to CSV format (imported by batch-cli-runner.ts) -- `scripts/batch-cli-runner.ts` — Main batch runner: converts inputs to CSV, processes them, writes actual responses as JSONL -- `.agentv/providers.yaml` — Defines the `batch_cli` CLI target with request batching enabled - -## Run - -From the repo root: - -```bash -cd examples/features/batch-cli - -# Run AgentV against the batch CLI target -# NOTE: This requires the CLI provider to support batching + JSONL batch output. -bun agentv eval ./evals/suite.yaml --provider batch_cli -``` diff --git a/examples/features/batch-cli/evals/suite.baseline.jsonl b/examples/features/batch-cli/evals/suite.baseline.jsonl deleted file mode 100644 index 43d5caad9..000000000 --- a/examples/features/batch-cli/evals/suite.baseline.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -{"timestamp":"2026-02-21T04:00:35.967Z","test_id":"aml-001","suite":"dataset.eval","score":1,"target":"batch_cli","scores":[{"name":"decision-check","type":"script","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"expected.decision present: CLEAR","passed":true,"evidence":"Batch runner decision matches the expected decision."},{"text":"candidate.decision present: CLEAR","passed":true}]},{"name":"trajectory-check","type":"trajectory:tool-used","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"aml_screening: called 1 times (required \u22651)","passed":true}]}],"assertions":[{"text":"expected.decision present: CLEAR","passed":true,"evidence":"decision-check: Batch runner decision matches the expected decision."},{"text":"candidate.decision present: CLEAR","passed":true},{"text":"aml_screening: called 1 times (required \u22651)","passed":true}]} -{"timestamp":"2026-02-21T04:00:36.039Z","test_id":"aml-002","suite":"dataset.eval","score":1,"target":"batch_cli","scores":[{"name":"decision-check","type":"script","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"expected.decision present: REVIEW","passed":true,"evidence":"Batch runner decision matches the expected decision."},{"text":"candidate.decision present: REVIEW","passed":true}]},{"name":"trajectory-check","type":"trajectory:tool-used","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"aml_screening: called 1 times (required \u22651)","passed":true}]}],"assertions":[{"text":"expected.decision present: REVIEW","passed":true,"evidence":"decision-check: Batch runner decision matches the expected decision."},{"text":"candidate.decision present: REVIEW","passed":true},{"text":"aml_screening: called 1 times (required \u22651)","passed":true}]} -{"timestamp":"2026-02-21T04:00:36.110Z","test_id":"aml-003","suite":"dataset.eval","score":1,"target":"batch_cli","scores":[{"name":"decision-check","type":"script","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"expected.decision present: REVIEW","passed":true,"evidence":"Batch runner decision matches the expected decision."},{"text":"candidate.decision present: REVIEW","passed":true}]},{"name":"trajectory-check","type":"trajectory:tool-used","score":1,"weight":1,"verdict":"pass","assertions":[{"text":"aml_screening: called 1 times (required \u22651)","passed":true}]}],"assertions":[{"text":"expected.decision present: REVIEW","passed":true,"evidence":"decision-check: Batch runner decision matches the expected decision."},{"text":"candidate.decision present: REVIEW","passed":true},{"text":"aml_screening: called 1 times (required \u22651)","passed":true}]} -{"timestamp":"2026-02-21T04:00:36.181Z","test_id":"aml-004-not-exist","suite":"dataset.eval","score":0,"target":"batch_cli","scores":[{"name":"decision-check","type":"script","score":0,"weight":1,"verdict":"fail","assertions":[{"text":"expected.decision present: REVIEW","passed":true,"evidence":"Batch runner decision did not match expected decision."},{"text":"Candidate output is not valid JSON with a decision field","passed":false},{"text":"decision mismatch: expected=REVIEW actual=null","passed":false}]},{"name":"trajectory-check","type":"trajectory:tool-used","score":0,"weight":1,"verdict":"fail","assertions":[{"text":"aml_screening: called 0 times (required \u22651)","passed":false}]}],"error":"Batch output missing id 'aml-004-not-exist'","assertions":[{"text":"expected.decision present: REVIEW","passed":true,"evidence":"decision-check: Batch runner decision did not match expected decision."},{"text":"Candidate output is not valid JSON with a decision field","passed":false},{"text":"decision mismatch: expected=REVIEW actual=null","passed":false},{"text":"aml_screening: called 0 times (required \u22651)","passed":false}]} diff --git a/examples/features/batch-cli/evals/suite.yaml b/examples/features/batch-cli/evals/suite.yaml deleted file mode 100644 index 1d591d4c9..000000000 --- a/examples/features/batch-cli/evals/suite.yaml +++ /dev/null @@ -1,157 +0,0 @@ -name: batch-cli -description: Batch CLI demo (AML screening) using structured input → CSV → JSONL with trace extraction -providers: - - batch_cli -tags: - - agent -prompts: - - "{{ input }}" -tests: - - id: aml-001 - assert: - - metric: decision-check - type: script - command: - - bun - - run - - ../graders/check-batch-cli-output.ts - cwd: . - - metric: trajectory-check - type: trajectory:tool-used - value: - name: aml_screening - min: 1 - - Batch runner returns a JSON object with decision=CLEAR. - vars: - input: - - role: system - content: You are a deterministic AML screening batch checker. Do not use external network calls. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-001 - customer_name: Example Customer A - origin_country: NZ - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 5000 - currency: USD - expected_output: - - role: assistant - content: - decision: CLEAR - - id: aml-002 - assert: - - metric: decision-check - type: script - command: - - bun - - run - - ../graders/check-batch-cli-output.ts - cwd: . - - metric: trajectory-check - type: trajectory:tool-used - value: - name: aml_screening - min: 1 - - Batch runner returns a JSON object with decision=REVIEW. - vars: - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-002 - customer_name: Example Customer B - origin_country: IR - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 2000 - currency: USD - expected_output: - - role: assistant - content: - decision: REVIEW - - id: aml-003 - assert: - - metric: decision-check - type: script - command: - - bun - - run - - ../graders/check-batch-cli-output.ts - cwd: . - - metric: trajectory-check - type: trajectory:tool-used - value: - name: aml_screening - min: 1 - - Batch runner returns a JSON object with decision=REVIEW. - vars: - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-003 - customer_name: Example Customer C - origin_country: US - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 25000 - currency: USD - expected_output: - - role: assistant - content: - decision: REVIEW - - id: aml-004-not-exist - assert: - - metric: decision-check - type: script - command: - - bun - - run - - ../graders/check-batch-cli-output.ts - cwd: . - - metric: trajectory-check - type: trajectory:tool-used - value: - name: aml_screening - min: 1 - - Batch runner returns a JSON object with decision=REVIEW. - vars: - input: - - role: system - content: You are a deterministic AML screening batch checker. - - role: user - content: - request: - type: aml_screening_check - jurisdiction: AU - effective_date: 2025-01-01 - row: - id: aml-003 - customer_name: Example Customer C - origin_country: US - destination_country: AU - transaction_type: INTERNATIONAL_TRANSFER - amount: 25000 - currency: USD - expected_output: - - role: assistant - content: - decision: REVIEW diff --git a/examples/features/batch-cli/graders/check-batch-cli-output.ts b/examples/features/batch-cli/graders/check-batch-cli-output.ts deleted file mode 100644 index 6be490b2c..000000000 --- a/examples/features/batch-cli/graders/check-batch-cli-output.ts +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bun -/** - * Batch CLI Output Grader - script grader - * - * Validates that the batch CLI runner produces the expected decision - * by comparing candidate output against expected_output or input. - */ -import { defineScriptGrader } from 'agentv'; - -function isObject(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function findExpectedDecisionFromExpectedMessages( - expectedOutput: readonly Record[], -): string | undefined { - for (const msg of expectedOutput) { - if (!isObject(msg)) continue; - const content = msg.content; - if (!isObject(content)) continue; - - const decision = content.decision; - if (typeof decision === 'string' && decision.trim().length > 0) { - return decision.trim(); - } - } - return undefined; -} - -function findExpectedDecisionFromInputMessages( - input: readonly Record[], -): string | undefined { - for (const msg of input) { - if (!isObject(msg)) continue; - if (msg.role !== 'user') continue; - const content = msg.content; - if (!isObject(content)) continue; - - const expected = content.expected; - if (!isObject(expected)) continue; - - const decision = expected.decision; - if (typeof decision === 'string' && decision.trim().length > 0) { - return decision.trim(); - } - } - return undefined; -} - -function getMessageText( - messages: readonly { role: string; content?: unknown }[], - role = 'assistant', -): string { - for (let i = messages.length - 1; i >= 0; i--) { - const msg = messages[i]; - if (msg.role === role) { - if (typeof msg.content === 'string') return msg.content; - if (Array.isArray(msg.content)) { - return msg.content - .filter((b: { type?: string }) => b.type === 'text') - .map((b: { text?: string }) => b.text) - .join('\n'); - } - } - } - return ''; -} - -export default defineScriptGrader(({ expectedOutput, input, output }) => { - const outputText = getMessageText(output ?? []); - const expectedDecision = - findExpectedDecisionFromExpectedMessages(expectedOutput) ?? - findExpectedDecisionFromInputMessages(input); - - let candidateObj: unknown; - try { - candidateObj = JSON.parse(outputText); - } catch { - candidateObj = undefined; - } - - const candidateDecision = - isObject(candidateObj) && typeof candidateObj.decision === 'string' - ? candidateObj.decision - : undefined; - - const assertions: Array<{ text: string; passed: boolean; evidence?: string }> = []; - - if (!expectedDecision) { - assertions.push({ - text: 'Missing expected decision (expected_output[].content.decision)', - passed: false, - }); - } else { - assertions.push({ text: `expected.decision present: ${expectedDecision}`, passed: true }); - } - - if (!candidateDecision) { - assertions.push({ - text: 'Candidate output is not valid JSON with a decision field', - passed: false, - }); - } else { - assertions.push({ text: `candidate.decision present: ${candidateDecision}`, passed: true }); - } - - const ok = - typeof expectedDecision === 'string' && - typeof candidateDecision === 'string' && - expectedDecision === candidateDecision; - - if (!ok) { - assertions.push({ - text: `decision mismatch: expected=${expectedDecision ?? 'null'} actual=${candidateDecision ?? 'null'}`, - passed: false, - }); - } - - return { - score: ok ? 1 : 0, - assertions, - }; -}); diff --git a/examples/features/batch-cli/package.json b/examples/features/batch-cli/package.json deleted file mode 100644 index e13dedb9e..000000000 --- a/examples/features/batch-cli/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "agentv-example-batch-cli", - "private": true, - "type": "module", - "dependencies": { - "agentv": "file:../../../apps/cli" - } -} diff --git a/examples/features/batch-cli/scripts/batch-cli-runner.ts b/examples/features/batch-cli/scripts/batch-cli-runner.ts deleted file mode 100644 index 60172febe..000000000 --- a/examples/features/batch-cli/scripts/batch-cli-runner.ts +++ /dev/null @@ -1,204 +0,0 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { buildCsv, extractRowsFromEvalYaml } from './build-csv-from-eval.ts'; - -// Batch CLI runner that processes CSV files for AML screening and outputs JSONL with output for trace extraction. - -function getFlag(args: readonly string[], name: string): string | undefined { - const idx = args.indexOf(name); - if (idx === -1) return undefined; - return args[idx + 1]; -} - -function parseCsvLine(line: string): string[] { - // Minimal RFC4180-ish parser for this demo (supports quoted cells). - const out: string[] = []; - let current = ''; - let inQuotes = false; - - for (let i = 0; i < line.length; i++) { - const ch = line[i]; - - if (inQuotes) { - if (ch === '"') { - const next = line[i + 1]; - if (next === '"') { - current += '"'; - i++; - } else { - inQuotes = false; - } - } else { - current += ch; - } - continue; - } - - if (ch === ',') { - out.push(current); - current = ''; - continue; - } - - if (ch === '"') { - inQuotes = true; - continue; - } - - current += ch; - } - - out.push(current); - return out; -} - -async function main(): Promise { - const args = process.argv.slice(2); - - if (args.includes('--healthcheck')) { - console.log('batch-cli-runner: healthy'); - return; - } - - const evalPathRaw = getFlag(args, '--eval-input'); - const csvPathRaw = getFlag(args, '--csv-input'); - const outPathRaw = getFlag(args, '--output'); - if (!csvPathRaw || !outPathRaw) { - throw new Error( - 'Usage: bun run batch-cli-runner.ts --csv-input --output [--eval-input ]', - ); - } - - const csvPath = path.resolve(csvPathRaw); - const outPath = path.resolve(outPathRaw); - - if (evalPathRaw) { - const evalPath = path.resolve(evalPathRaw); - const yamlText = await fs.readFile(evalPath, 'utf8'); - const rows = extractRowsFromEvalYaml(yamlText); - if (rows.length === 0) { - throw new Error(`No rows extracted from eval file: ${evalPathRaw}`); - } - await fs.writeFile(csvPath, buildCsv(rows), 'utf8'); - } - - const csvText = await fs.readFile(csvPath, 'utf8'); - const lines = csvText - .split(/\r?\n/) - .map((l) => l.trimEnd()) - .filter((l) => l.length > 0); - - if (lines.length < 2) { - throw new Error(`CSV has no data rows: ${csvPath}`); - } - - const header = parseCsvLine(lines[0]); - const idx = (name: string): number => header.indexOf(name); - - const idIndex = idx('id'); - const originCountryIndex = idx('origin_country'); - const destinationCountryIndex = idx('destination_country'); - const amountIndex = idx('amount'); - const currencyIndex = idx('currency'); - - if (idIndex === -1) { - throw new Error("CSV missing required header column 'id'"); - } - - type OutputRecord = { - id: string; - text: string; - output: Array<{ - role: string; - tool_calls?: Array<{ - tool: string; - input?: unknown; - output?: unknown; - }>; - content?: unknown; - }>; - }; - - const records: OutputRecord[] = []; - - for (const line of lines.slice(1)) { - const cols = parseCsvLine(line); - const id = cols[idIndex] ?? ''; - if (!id) continue; - - const originCountry = originCountryIndex !== -1 ? (cols[originCountryIndex] ?? '') : ''; - const destinationCountry = - destinationCountryIndex !== -1 ? (cols[destinationCountryIndex] ?? '') : ''; - const amountRaw = amountIndex !== -1 ? (cols[amountIndex] ?? '') : ''; - const currency = currencyIndex !== -1 ? (cols[currencyIndex] ?? '') : ''; - - const amount = Number.parseFloat(amountRaw); - - // Deterministic demo rule (synthetic): - // - REVIEW if origin/destination is in a high-risk list OR amount >= 10,000. - // - otherwise CLEAR. - const highRiskCountries = new Set(['IR', 'KP']); - const isHighRiskCountry = - highRiskCountries.has(originCountry) || highRiskCountries.has(destinationCountry); - const isHighValue = Number.isFinite(amount) && amount >= 10_000; - const decision = isHighRiskCountry || isHighValue ? 'REVIEW' : 'CLEAR'; - - const reasons: string[] = []; - if (isHighRiskCountry) reasons.push('high_risk_country'); - if (isHighValue) reasons.push('high_value_amount'); - - const payload = { - id, - decision, - rule: 'aml_screening_synthetic', - reasons, - amount: Number.isFinite(amount) ? amount : amountRaw, - currency, - }; - - // Build output with tool_calls for trace extraction - // This demonstrates the new output format that AgentV can extract traces from - records.push({ - id, - text: JSON.stringify(payload), - output: [ - { - role: 'assistant', - tool_calls: [ - { - tool: 'aml_screening', - input: { - origin_country: originCountry, - destination_country: destinationCountry, - amount: Number.isFinite(amount) ? amount : amountRaw, - currency, - }, - output: { - decision, - reasons, - }, - }, - ], - }, - { - role: 'assistant', - content: payload, - }, - ], - }); - } - - const jsonl = `${records.map((r) => JSON.stringify(r)).join('\n')}\n`; - await fs.writeFile(outPath, jsonl, 'utf8'); - - // Also write a stable artifact in the working directory for convenience. - const stablePath = path.resolve(process.cwd(), 'agentv-evalresult.jsonl'); - await fs.writeFile(stablePath, jsonl, 'utf8'); -} - -main().catch((error) => { - const message = error instanceof Error ? (error.stack ?? error.message) : String(error); - console.error(message); - process.exitCode = 1; -}); diff --git a/examples/features/batch-cli/scripts/build-csv-from-eval.ts b/examples/features/batch-cli/scripts/build-csv-from-eval.ts deleted file mode 100644 index f1f770492..000000000 --- a/examples/features/batch-cli/scripts/build-csv-from-eval.ts +++ /dev/null @@ -1,159 +0,0 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { parse } from 'yaml'; - -function isObject(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function getArg(args: readonly string[], name: string): string | undefined { - const index = args.indexOf(name); - if (index === -1) return undefined; - return args[index + 1]; -} - -function csvEscape(value: string): string { - if (/[",\n\r]/.test(value)) { - return `"${value.replaceAll('"', '""')}"`; - } - return value; -} - -export type EvalRow = { - readonly id: string; - readonly customer_name: string; - readonly origin_country: string; - readonly destination_country: string; - readonly transaction_type: string; - readonly amount: string; - readonly currency: string; - readonly jurisdiction: string; - readonly effective_date: string; -}; - -function findFirstUserContentObject(input: unknown): Record | undefined { - if (!Array.isArray(input)) return undefined; - - for (const msg of input) { - if (!isObject(msg)) continue; - if ((msg as Record).role !== 'user') continue; - const content = (msg as Record).content; - if (!isObject(content)) continue; - return content as Record; - } - - return undefined; -} - -export function extractRowsFromEvalYaml(yamlText: string): readonly EvalRow[] { - const parsed = parse(yamlText) as unknown; - if (!isObject(parsed)) return []; - - const evalcases = - (parsed as Record).tests ?? (parsed as Record).cases; - if (!Array.isArray(evalcases)) return []; - - const rows: EvalRow[] = []; - for (const item of evalcases) { - if (!isObject(item)) continue; - - const id = typeof item.id === 'string' ? item.id : ''; - if (!id) continue; - if (id.includes('not-exist')) { - // Skip placeholder cases that should not reach the CSV artifact. - continue; - } - - const content = findFirstUserContentObject(item.input ?? item.input); - if (!content) continue; - - const request = isObject(content.request) ? (content.request as Record) : {}; - const row = isObject(content.row) ? (content.row as Record) : {}; - - rows.push({ - id, - customer_name: typeof row.customer_name === 'string' ? row.customer_name : '', - origin_country: typeof row.origin_country === 'string' ? row.origin_country : '', - destination_country: - typeof row.destination_country === 'string' ? row.destination_country : '', - transaction_type: typeof row.transaction_type === 'string' ? row.transaction_type : '', - amount: - typeof row.amount === 'string' || typeof row.amount === 'number' ? String(row.amount) : '', - currency: typeof row.currency === 'string' ? row.currency : '', - jurisdiction: typeof request.jurisdiction === 'string' ? request.jurisdiction : '', - effective_date: typeof request.effective_date === 'string' ? request.effective_date : '', - }); - } - - return rows; -} - -export function buildCsv(rows: readonly EvalRow[]): string { - const headers = [ - 'id', - 'customer_name', - 'origin_country', - 'destination_country', - 'transaction_type', - 'amount', - 'currency', - 'jurisdiction', - 'effective_date', - ] as const; - const lines: string[] = []; - lines.push(headers.join(',')); - for (const row of rows) { - lines.push( - [ - row.id, - row.customer_name, - row.origin_country, - row.destination_country, - row.transaction_type, - row.amount, - row.currency, - row.jurisdiction, - row.effective_date, - ] - .map((v) => csvEscape(v ?? '')) - .join(','), - ); - } - return `${lines.join('\n')}\n`; -} - -async function main(): Promise { - const args = process.argv.slice(2); - const evalPathRaw = getArg(args, '--eval'); - const outPathRaw = getArg(args, '--out'); - - if (!evalPathRaw || !outPathRaw) { - throw new Error('Usage: bun run build-csv-from-eval.ts --eval --out '); - } - - const evalPath = path.resolve(evalPathRaw); - const outPath = path.resolve(outPathRaw); - - const yamlText = await fs.readFile(evalPath, 'utf8'); - const rows = extractRowsFromEvalYaml(yamlText); - - if (rows.length === 0) { - throw new Error( - 'No rows extracted. Ensure tests have a user input with object content.request/content.row', - ); - } - - const csvContent = buildCsv(rows); - await fs.writeFile(outPath, csvContent, 'utf8'); - console.log(`Wrote CSV (${rows.length} rows): ${outPath}`); -} - -// Only run main when executed directly, not when imported -if (import.meta.main) { - main().catch((error) => { - const message = error instanceof Error ? error.message : String(error); - console.error(message); - process.exitCode = 1; - }); -} diff --git a/skills-data/agentv-eval-migrations/references/breaking-changes.md b/skills-data/agentv-eval-migrations/references/breaking-changes.md index 0c6c28271..4abe458aa 100644 --- a/skills-data/agentv-eval-migrations/references/breaking-changes.md +++ b/skills-data/agentv-eval-migrations/references/breaking-changes.md @@ -892,8 +892,10 @@ targets: - Remove `use_target`; current authored target definitions must resolve to concrete provider objects. - Keep supported AgentV target extensions such as `grader_target`, - `fallback_targets`, `workers`, and `batch_requests` as top-level fields on - target objects. + `fallback_targets`, and `workers` as top-level fields on target objects. +- Remove runner-level request batching from migrated configs. CLI providers are + invoked once per eval case; throughput batching belongs inside provider + adapters or CLIs without eval YAML batch configuration. ### Verification