From f6da5c7ea16b522778f8e4302916053d1c29ff82 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 23:57:18 +0200 Subject: [PATCH] refactor(sdk): hard-rename provider proxy client --- .../docs/docs/next/graders/script-graders.mdx | 28 +- examples/README.md | 6 +- examples/features/README.md | 24 +- .../script-grader-with-llm-calls/README.md | 42 +-- .../evals/contextual-precision.eval.yaml | 2 +- .../evals/contextual-recall.eval.yaml | 2 +- .../scripts/contextual-precision.ts | 16 +- .../scripts/contextual-recall.ts | 18 +- examples/features/sdk-python/README.md | 2 +- .../sdk-python/src/agentv_py/__init__.py | 4 +- .../sdk-python/src/agentv_py/grader.py | 16 +- .../trial-output-consistency/README.md | 6 +- .../graders/trial-consistency.ts | 12 +- .../src/evaluation/graders/script-grader.ts | 34 +-- .../src/evaluation/loaders/grader-parser.ts | 34 ++- .../evaluation/registry/builtin-graders.ts | 2 +- packages/core/src/evaluation/types.ts | 12 +- .../evaluation/validation/eval-file.schema.ts | 11 +- .../{target-proxy.ts => provider-proxy.ts} | 92 +++---- .../evaluation/loaders/grader-parser.test.ts | 40 +++ .../core/test/evaluation/token-usage.test.ts | 22 +- .../core/test/runtime/provider-proxy.test.ts | 243 +++++++++++++++++ .../core/test/runtime/target-proxy.test.ts | 241 ----------------- packages/sdk/src/index.ts | 18 +- packages/sdk/src/provider-client.ts | 247 +++++++++++++++++ packages/sdk/src/target-client.ts | 251 ------------------ ...client.test.ts => provider-client.test.ts} | 162 +++++------ skills-data/agentv-eval-writer/SKILL.md | 4 +- .../references/custom-evaluators.md | 10 +- .../references/python-helpers.md | 4 +- 30 files changed, 829 insertions(+), 776 deletions(-) rename packages/core/src/runtime/{target-proxy.ts => provider-proxy.ts} (75%) create mode 100644 packages/core/test/runtime/provider-proxy.test.ts delete mode 100644 packages/core/test/runtime/target-proxy.test.ts create mode 100644 packages/sdk/src/provider-client.ts delete mode 100644 packages/sdk/src/target-client.ts rename packages/sdk/test/{target-client.test.ts => provider-client.test.ts} (62%) diff --git a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx index cb655d283..612ae66c8 100644 --- a/apps/web/src/content/docs/docs/next/graders/script-graders.mdx +++ b/apps/web/src/content/docs/docs/next/graders/script-graders.mdx @@ -288,30 +288,30 @@ Prefer Vitest verifiers when the checks naturally fit `expect(...)`. Use `define **SDK exports:** `defineScriptGrader`, `defineVitestWorkspaceGrader`, `defineWorkspaceGrader`, `Message`, `ToolCall`, `Trace`, `TraceSummary`, `ScriptGraderInput`, `ScriptGraderResult`, `ScriptGraderCheck`, `Workspace`, `WorkspaceCheck` -## Target Access +## Provider Access -Script graders can call an LLM through a target proxy for metrics that require multiple LLM calls (contextual precision, semantic similarity, etc.). +Script graders can call an LLM through a provider proxy for metrics that require multiple LLM calls (contextual precision, semantic similarity, etc.). ### Configuration -Add a `target` block to the grader config: +Add a `provider` block to the grader config: ```yaml assert: - name: contextual-precision type: script command: [bun, scripts/contextual-precision.ts] - target: + provider: max_calls: 10 # Default: 50 ``` ### Usage -Use `createTargetClient` from the SDK: +Use `createProviderClient` from the SDK: ```typescript #!/usr/bin/env bun -import { createTargetClient, defineScriptGrader } from 'agentv'; +import { createProviderClient, defineScriptGrader } from 'agentv'; export default defineScriptGrader(async ({ input, output }) => { const inputText = input @@ -319,10 +319,10 @@ export default defineScriptGrader(async ({ input, output }) => { .map((message) => typeof message.content === 'string' ? message.content : '') .join('\n'); const outputText = output ?? ''; - const target = createTargetClient(); - if (!target) return { pass: false, score: 0, reason: 'Target not configured' }; + const provider = createProviderClient(); + if (!provider) return { pass: false, score: 0, reason: 'Provider proxy not configured' }; - const response = await target.invoke({ + const response = await provider.invoke({ question: `Is this relevant to: ${inputText}? Response: ${outputText}`, systemPrompt: 'Respond with JSON: { "relevant": true/false }' }); @@ -336,14 +336,14 @@ export default defineScriptGrader(async ({ input, output }) => { }); ``` -Use `target.invokeBatch(requests)` for multiple calls in parallel. +Use `provider.invokeBatch(requests)` for multiple calls in parallel. -**Environment variables** (set automatically when `target` is configured): +**Environment variables** (set automatically when `provider` is configured): | Variable | Description | |----------|-------------| -| `AGENTV_TARGET_PROXY_URL` | Local proxy URL | -| `AGENTV_TARGET_PROXY_TOKEN` | Bearer token for authentication | +| `AGENTV_PROVIDER_PROXY_URL` | Local proxy URL | +| `AGENTV_PROVIDER_PROXY_TOKEN` | Bearer token for authentication | ## Advanced Input Fields @@ -353,7 +353,7 @@ Beyond the basic fields (`input`, `output`, `expected_output`), script graders r |-------|------|-------------| | `input` | `Message[]` | Full resolved input message array | | `output` | `string \| null` | Final answer / scored result only | -| `messages` | `Message[]` | Transcript messages from the target execution | +| `messages` | `Message[]` | Transcript messages from the provider execution | | `expected_output` | `Message[]` | Expected/reference output messages | | `output_path` | `string` | Temp file containing large final answer JSON, when `output` is omitted | | `input_files` | `string[]` | Paths to input files referenced in the eval | diff --git a/examples/README.md b/examples/README.md index 53226e819..424071bad 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,10 +47,10 @@ Focused demonstrations of specific AgentV capabilities. Each example includes it - [assert-set](features/assert-set/) - Assertion grouping patterns - [weighted-graders](features/weighted-graders/) - Weighted graders - [execution-metrics](features/execution-metrics/) - Metrics tracking (tokens, cost, latency) -- [script-grader-with-llm-calls](features/script-grader-with-llm-calls/) - script graders with target proxy for LLM calls +- [script-grader-with-llm-calls](features/script-grader-with-llm-calls/) - script graders with provider proxy for LLM calls - [batch-cli](features/batch-cli/) - Batch CLI evaluation - [document-extraction](features/document-extraction/) - Document data extraction -- [local-cli](features/local-cli/) - Local CLI targets +- [local-cli](features/local-cli/) - Local CLI providers - [compare](features/compare/) - Baseline comparison - [deterministic-graders](features/deterministic-graders/) - Deterministic assertions (contains, regex, JSON validation) - [vitest-workspace-grader](features/vitest-workspace-grader/) - Vitest-style deterministic workspace verifiers @@ -92,7 +92,7 @@ example-name/ │ └── *.md # LLM grader prompts (optional) ├── scripts/ # Helper scripts (optional) ├── .agentv/ -│ └── providers.yaml # Target configuration (optional) +│ └── providers.yaml # Provider configuration (optional) ├── package.json # Dependencies (if using agentv) └── README.md # Example documentation ``` diff --git a/examples/features/README.md b/examples/features/README.md index ce1fdd9dc..5186cb0ab 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -40,7 +40,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | Example | Description | |---------|-------------| | [script-grader-sdk](script-grader-sdk/) | TypeScript script graders using `defineScriptGrader()` from `agentv` | -| [script-grader-with-llm-calls](script-grader-with-llm-calls/) | script graders that make LLM calls via a target proxy | +| [script-grader-with-llm-calls](script-grader-with-llm-calls/) | script graders that make LLM calls via a provider proxy | | [eval-assert-demo](eval-assert-demo/) | script graders runnable both in a suite and individually via `agentv eval assert` | | [functional-grading](functional-grading/) | Install dependencies, compile, and run tests against agent-generated code | @@ -51,7 +51,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the |---------|-------------| | [trajectory-assertions-simple](trajectory-assertions-simple/) | Validate expected tool calls with Promptfoo trajectory assertions | | [trajectory-assertions-advanced](trajectory-assertions-advanced/) | Promptfoo trajectory assertions with `expected_output` and per-call checks | -| [latency-assertions](latency-assertions/) | Tool sequence and argument checks for a latency-flavored mock target | +| [latency-assertions](latency-assertions/) | Tool sequence and argument checks for a latency-flavored mock provider | | [tool-evaluation-plugins](tool-evaluation-plugins/) | F1 precision/recall scoring for tool-call accuracy | | [trace-evaluation](trace-evaluation/) | Inspect agent internals: LLM call counts, tool executions, step durations | @@ -103,10 +103,10 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [workspace-setup-script](workspace-setup-script/) | Multi-step setup with a `beforeAll` lifecycle extension | | [workspace-multi-repo](workspace-multi-repo/) | Multi-repo environment using a VS Code `.code-workspace` file | | [workspace-shared-config](workspace-shared-config/) | Define an environment recipe once and reference it across eval files | -| [repo-lifecycle](repo-lifecycle/) | Clone a git repo into the workspace and target the agent at it | +| [repo-lifecycle](repo-lifecycle/) | Clone a git repo into the workspace and point the agent at it | | [file-changes](file-changes/) | Capture workspace file changes made by the agent across test runs | | [file-changes-graders](file-changes-graders/) | Grade file diffs with rubrics and LLM graders | -| [local-cli](local-cli/) | Define and invoke local CLI targets | +| [local-cli](local-cli/) | Define and invoke local CLI providers | | [batch-cli](batch-cli/) | Run bulk evaluations from the CLI | --- @@ -138,7 +138,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [assert-extended](assert-extended/) | Deterministic assertions | | [basic](basic/) | Getting started | | [basic-jsonl](basic-jsonl/) | Getting started | -| [batch-cli](batch-cli/) | Workspace & targets | +| [batch-cli](batch-cli/) | Workspace & providers | | [benchmark-tooling](benchmark-tooling/) | Benchmarking | | [script-grader-sdk](script-grader-sdk/) | Custom graders | | [script-grader-with-llm-calls](script-grader-with-llm-calls/) | Custom graders | @@ -152,17 +152,17 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [eval-assert-demo](eval-assert-demo/) | Custom graders | | [execution-metrics](execution-metrics/) | Cost, latency & tokens | | [external-datasets](external-datasets/) | Dataset & input | -| [file-changes](file-changes/) | Workspace & targets | -| [file-changes-graders](file-changes-graders/) | Workspace & targets | +| [file-changes](file-changes/) | Workspace & providers | +| [file-changes-graders](file-changes-graders/) | Workspace & providers | | [functional-grading](functional-grading/) | Custom graders | | [input-files-shorthand](input-files-shorthand/) | Dataset & input | | [latency-assertions](latency-assertions/) | Tool & agent evaluation | -| [local-cli](local-cli/) | Workspace & targets | +| [local-cli](local-cli/) | Workspace & providers | | [multi-turn-conversation](multi-turn-conversation/) | LLM grading | | [nlp-metrics](nlp-metrics/) | Deterministic assertions | | [file-transforms](file-transforms/) | LLM grading | | [prompt-template-sdk](prompt-template-sdk/) | TypeScript SDK | -| [repo-lifecycle](repo-lifecycle/) | Workspace & targets | +| [repo-lifecycle](repo-lifecycle/) | Workspace & providers | | [rubric](rubric/) | LLM grading | | [scenarios](scenarios/) | Dataset & prompt templates | | [sdk-config-file](sdk-config-file/) | TypeScript SDK | @@ -181,6 +181,6 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [trial-output-consistency](trial-output-consistency/) | Benchmarking | | [trials](trials/) | Benchmarking | | [weighted-graders](weighted-graders/) | LLM grading | -| [workspace-multi-repo](workspace-multi-repo/) | Workspace & targets | -| [workspace-setup-script](workspace-setup-script/) | Workspace & targets | -| [workspace-shared-config](workspace-shared-config/) | Workspace & targets | +| [workspace-multi-repo](workspace-multi-repo/) | Workspace & providers | +| [workspace-setup-script](workspace-setup-script/) | Workspace & providers | +| [workspace-shared-config](workspace-shared-config/) | Workspace & providers | diff --git a/examples/features/script-grader-with-llm-calls/README.md b/examples/features/script-grader-with-llm-calls/README.md index b69869301..e519f4e37 100644 --- a/examples/features/script-grader-with-llm-calls/README.md +++ b/examples/features/script-grader-with-llm-calls/README.md @@ -141,7 +141,7 @@ The current `extractRetrievalContext()` in `utils.ts` flattens for simplicity. F ## Security -The target proxy is designed with security in mind: +The provider proxy is designed with security in mind: - Binds to **loopback only** (127.0.0.1) - not accessible from network - Uses **bearer token authentication** - unique per execution - Enforces **max_calls limit** - prevents runaway costs @@ -149,29 +149,29 @@ The target proxy is designed with security in mind: ## Configuration -Enable target access by adding a `target` block to your `script_grader` grader: +Enable provider access by adding a `provider` block to your `script_grader` grader: ```yaml graders: - name: contextual_precision type: script command: [bun, run, scripts/contextual-precision.ts] - target: + provider: max_calls: 10 # At least N nodes to evaluate - name: contextual_recall type: script command: [bun, run, scripts/contextual-recall.ts] - target: + provider: max_calls: 15 # 1 for extraction + N statements for attribution ``` ## Usage in Code ```typescript -import { createTargetClient, defineScriptGrader } from 'agentv'; +import { createProviderClient, defineScriptGrader } from 'agentv'; export default defineScriptGrader(async ({ question, config }) => { - const target = createTargetClient(); + const provider = createProviderClient(); const retrievalContext = config?.retrieval_context ?? []; // Batch evaluation of all nodes @@ -180,7 +180,7 @@ export default defineScriptGrader(async ({ question, config }) => { systemPrompt: 'Respond with JSON: { "relevant": true/false }' })); - const responses = await target.invokeBatch(requests); + const responses = await provider.invokeBatch(requests); // Calculate weighted precision score... }); @@ -188,42 +188,42 @@ export default defineScriptGrader(async ({ question, config }) => { ## Querying Proxy Info -You can query information about the target proxy: +You can query information about the provider proxy: ```typescript -const info = await target.getInfo(); -console.log(`Target: ${info.targetName}`); +const info = await provider.getInfo(); +console.log(`Provider: ${info.providerLabel}`); console.log(`Calls: ${info.callCount}/${info.maxCalls}`); -console.log(`Available targets: ${info.availableTargets.join(', ')}`); +console.log(`Available providers: ${info.availableProviderLabels.join(', ')}`); ``` -## Target Override +## Provider Override -Use different targets for different purposes within the same grader: +Use different providers for different purposes within the same grader: ```typescript // Use a coding agent for complex tasks -const agentResponses = await target.invokeBatch( +const agentResponses = await provider.invokeBatch( nodes.map(node => ({ question: `Is this relevant? ${node}`, - target: 'pi' // Override default target + provider: 'pi' // Override default provider })) ); // Use a base LLM for simple evaluation -const response = await target.invoke({ +const response = await provider.invoke({ question: complexAnalysisPrompt, - target: 'gemini-llm' // Use different target + provider: 'gemini-llm' // Use different provider }); ``` ## Environment Variables -When `target` is configured, these environment variables are automatically set: -- `AGENTV_TARGET_PROXY_URL` - Local proxy URL (e.g., `http://127.0.0.1:45123`) -- `AGENTV_TARGET_PROXY_TOKEN` - Bearer token for authentication +When `provider` is configured, these environment variables are automatically set: +- `AGENTV_PROVIDER_PROXY_URL` - Local proxy URL (e.g., `http://127.0.0.1:45123`) +- `AGENTV_PROVIDER_PROXY_TOKEN` - Bearer token for authentication -The `createTargetClient()` function reads these automatically. +The `createProviderClient()` function reads these automatically. ## Running diff --git a/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml index f0e07bcd8..d480634de 100644 --- a/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml +++ b/examples/features/script-grader-with-llm-calls/evals/contextual-precision.eval.yaml @@ -5,7 +5,7 @@ assert: - bun - run - ../scripts/contextual-precision.ts - target: + provider: max_calls: 10 providers: - llm diff --git a/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml index 5d778b790..a4ff6c88e 100644 --- a/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml +++ b/examples/features/script-grader-with-llm-calls/evals/contextual-recall.eval.yaml @@ -5,7 +5,7 @@ assert: - bun - run - ../scripts/contextual-recall.ts - target: + provider: max_calls: 15 providers: - llm diff --git a/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts b/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts index 33f634838..03923d7e2 100644 --- a/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/contextual-precision.ts @@ -12,10 +12,10 @@ * Retrieval context is extracted from expected_output.tool_calls output, * which represents the expected agent behavior (calling a retrieval tool). * - * Requires `target: { max_calls: N }` in the grader YAML config, + * Requires `provider: { max_calls: N }` in the grader YAML config, * where N >= number of retrieval context nodes to evaluate. */ -import { createTargetClient, defineScriptGrader } from 'agentv'; +import { createProviderClient, defineScriptGrader } from 'agentv'; import { extractRetrievalContext } from './utils.js'; interface RelevanceResult { @@ -63,14 +63,14 @@ export default defineScriptGrader(async (input) => { }; } - const target = createTargetClient(); + const provider = createProviderClient(); - if (!target) { + if (!provider) { return { score: 0, assertions: [ { - text: 'Target not available - ensure `target` block is configured in grader YAML', + text: 'Provider proxy not available - ensure `provider` block is configured in grader YAML', passed: false, }, ], @@ -78,7 +78,7 @@ export default defineScriptGrader(async (input) => { } // Step 1: Use batch invocation to determine relevance of each node - // Demonstrates target override - uses gemini-llm regardless of default target + // Demonstrates provider override - uses gemini-llm regardless of default provider const requests = retrievalContext.map((node, index) => ({ question: `Determine if this retrieved context node is relevant to answering the question. @@ -95,10 +95,10 @@ Is this node relevant to answering the question? Respond with JSON only: }`, systemPrompt: 'You are a precise relevance grader for RAG systems. Determine if a retrieved node contains information useful for answering the given question. Output valid JSON only.', - target: 'gemini-llm', // Override: use gemini-llm for relevance checks + provider: 'gemini-llm', // Override: use gemini-llm for relevance checks })); - const responses = await target.invokeBatch(requests); + const responses = await provider.invokeBatch(requests); // Step 2: Parse relevance scores for each node const relevanceScores: boolean[] = []; diff --git a/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts b/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts index 41d89b5c6..d6364aae3 100644 --- a/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts +++ b/examples/features/script-grader-with-llm-calls/scripts/contextual-recall.ts @@ -16,10 +16,10 @@ * Retrieval context is extracted from expected_output.tool_calls output, * which represents the expected agent behavior (calling a retrieval tool). * - * Requires `target: { max_calls: N }` in the grader YAML config, + * Requires `provider: { max_calls: N }` in the grader YAML config, * where N >= 2 (one for statement extraction + one for attribution check). */ -import { createTargetClient, defineScriptGrader } from 'agentv'; +import { createProviderClient, defineScriptGrader } from 'agentv'; import { extractRetrievalContext } from './utils.js'; interface StatementExtractionResult { @@ -85,14 +85,14 @@ export default defineScriptGrader(async (input) => { }; } - const target = createTargetClient(); + const provider = createProviderClient(); - if (!target) { + if (!provider) { return { score: 0, assertions: [ { - text: 'Target not available - ensure `target` block is configured in grader YAML', + text: 'Provider proxy not available - ensure `provider` block is configured in grader YAML', passed: false, }, ], @@ -100,7 +100,7 @@ export default defineScriptGrader(async (input) => { } // Step 1: Extract statements from the criteria - const extractionResponse = await target.invoke({ + const extractionResponse = await provider.invoke({ question: `Extract all distinct factual statements or claims from the following expected answer. Each statement should be a self-contained claim that can be independently verified. @@ -115,7 +115,7 @@ Extract the statements and respond with JSON only: }`, systemPrompt: 'You are a precise statement extractor. Break down answers into distinct, verifiable claims. Output valid JSON only.', - target: 'gemini-llm', + provider: 'gemini-llm', }); let statements: string[] = []; @@ -168,10 +168,10 @@ Respond with JSON only: }`, systemPrompt: 'You are a precise attribution verifier. Determine if statements can be logically derived from or supported by the given context. A statement is attributable if the context contains information that directly supports or implies it. Output valid JSON only.', - target: 'gemini-llm', + provider: 'gemini-llm', })); - const attributionResponses = await target.invokeBatch(attributionRequests); + const attributionResponses = await provider.invokeBatch(attributionRequests); // Step 3: Parse attribution results const attributionResults: Array<{ diff --git a/examples/features/sdk-python/README.md b/examples/features/sdk-python/README.md index 935236c93..0d71e034a 100644 --- a/examples/features/sdk-python/README.md +++ b/examples/features/sdk-python/README.md @@ -11,7 +11,7 @@ It does **not** add a native Python runner. Evaluations still run through the Ag ## Layout -- `src/agentv_py/grader.py` - canonical script-grader helper and target proxy client +- `src/agentv_py/grader.py` - canonical script-grader helper and provider proxy client - `src/agentv_py/evals.py` - YAML/JSONL authoring helpers plus optional CLI invocation - `scripts/check_expected_output.py` - example Python script-grader - `scripts/build_eval.py` - example eval definition builder diff --git a/examples/features/sdk-python/src/agentv_py/__init__.py b/examples/features/sdk-python/src/agentv_py/__init__.py index 40da2fc8e..34fd1ed83 100644 --- a/examples/features/sdk-python/src/agentv_py/__init__.py +++ b/examples/features/sdk-python/src/agentv_py/__init__.py @@ -14,9 +14,9 @@ Assertion, CodeGraderContext, Check, + ProviderClient, ScriptGraderContext, ScriptGraderResult, - TargetClient, define_script_grader, emit_grader_result, load_grader_input, @@ -29,7 +29,7 @@ "ScriptGraderContext", "ScriptGraderResult", "CodeGraderContext", - "TargetClient", + "ProviderClient", "define_script_grader", "emit_grader_result", "load_grader_input", diff --git a/examples/features/sdk-python/src/agentv_py/grader.py b/examples/features/sdk-python/src/agentv_py/grader.py index 4d32aacd7..c6fa43286 100644 --- a/examples/features/sdk-python/src/agentv_py/grader.py +++ b/examples/features/sdk-python/src/agentv_py/grader.py @@ -184,22 +184,22 @@ def emit_grader_result(result: ScriptGraderResult) -> None: sys.stdout.write(f"{json.dumps(result.to_wire(), indent=2)}\n") -class TargetClient: - """Minimal stdlib client for AgentV's target proxy.""" +class ProviderClient: + """Minimal stdlib client for AgentV's provider proxy.""" def __init__(self, url: str, token: str): self._url = url.rstrip("/") self._token = token @classmethod - def from_env(cls) -> "TargetClient | None": - url = os.environ.get("AGENTV_TARGET_PROXY_URL") - token = os.environ.get("AGENTV_TARGET_PROXY_TOKEN") + def from_env(cls) -> "ProviderClient | None": + url = os.environ.get("AGENTV_PROVIDER_PROXY_URL") + token = os.environ.get("AGENTV_PROVIDER_PROXY_TOKEN") if not url: return None if not token: raise RuntimeError( - "AGENTV_TARGET_PROXY_URL is set but AGENTV_TARGET_PROXY_TOKEN is missing" + "AGENTV_PROVIDER_PROXY_URL is set but AGENTV_PROVIDER_PROXY_TOKEN is missing" ) return cls(url, token) @@ -230,7 +230,7 @@ def invoke( system_prompt: str | None = None, eval_case_id: str | None = None, attempt: int | None = None, - target: str | None = None, + provider: str | None = None, ) -> Any: return self._request( "POST", @@ -240,7 +240,7 @@ def invoke( "systemPrompt": system_prompt, "evalCaseId": eval_case_id, "attempt": attempt, - "target": target, + "provider": provider, }, ) diff --git a/examples/features/trial-output-consistency/README.md b/examples/features/trial-output-consistency/README.md index 27917d41f..97d65300e 100644 --- a/examples/features/trial-output-consistency/README.md +++ b/examples/features/trial-output-consistency/README.md @@ -23,7 +23,7 @@ When an agent is run multiple times on the same input (trials), outputs may vary | Method | When Used | Accuracy | |--------|-----------|----------| | **Embedding** | Target client available, `fallback` not set | High — captures semantic similarity | -| **Token-overlap** | No target or `fallback: token` | Moderate — bag-of-words cosine | +| **Token-overlap** | No provider or `fallback: token` | Moderate — bag-of-words cosine | ## Edge Cases @@ -64,8 +64,8 @@ assert: # Validate the eval file bun agentv validate examples/features/trial-output-consistency/evals/suite.yaml -# Run a specific test with a configured target -bun agentv eval examples/features/trial-output-consistency/evals/suite.yaml --test-id high-consistency --provider +# Run a specific test with a configured provider +bun agentv eval examples/features/trial-output-consistency/evals/suite.yaml --test-id high-consistency --provider ``` ## Extending diff --git a/examples/features/trial-output-consistency/graders/trial-consistency.ts b/examples/features/trial-output-consistency/graders/trial-consistency.ts index 501512a76..3976edcc2 100644 --- a/examples/features/trial-output-consistency/graders/trial-consistency.ts +++ b/examples/features/trial-output-consistency/graders/trial-consistency.ts @@ -3,7 +3,7 @@ * Trial Output Consistency Grader * * Computes consistency across repeated trial outputs using embedding similarity. - * Uses the Vercel AI SDK for embeddings via AgentV's target client, with a + * Uses the Vercel AI SDK for embeddings via AgentV's provider client, with a * token-overlap cosine similarity fallback when embeddings are unavailable. * * Config: @@ -15,7 +15,7 @@ * 1 trial → score 1.0 (perfect consistency by definition) * 2+ trials → average pairwise cosine similarity */ -import { createTargetClient, defineScriptGrader, z } from 'agentv'; +import { createProviderClient, defineScriptGrader, z } from 'agentv'; const ConfigSchema = z.object({ trialOutputs: z.array(z.string()), @@ -64,11 +64,11 @@ function tokenVectors(texts: string[]): number[][] { return tfs.map((tf) => vocab.map((w) => tf.get(w) ?? 0)); } -// ── Embedding via target client ───────────────────────────────────────── +// ── Embedding via provider client ───────────────────────────────────────── async function getEmbeddings(texts: string[]): Promise { - const target = createTargetClient(); - if (!target) return null; + const provider = createProviderClient(); + if (!provider) return null; try { const requests = texts.map((text) => ({ @@ -76,7 +76,7 @@ async function getEmbeddings(texts: string[]): Promise { systemPrompt: 'Return ONLY a JSON array of 64 floating-point numbers representing a semantic embedding of the user message. No explanation.', })); - const responses = await target.invokeBatch(requests); + const responses = await provider.invokeBatch(requests); const embeddings: number[][] = []; for (const r of responses) { const raw = r.rawText ?? ''; diff --git a/packages/core/src/evaluation/graders/script-grader.ts b/packages/core/src/evaluation/graders/script-grader.ts index cf8446584..d8070a2b2 100644 --- a/packages/core/src/evaluation/graders/script-grader.ts +++ b/packages/core/src/evaluation/graders/script-grader.ts @@ -5,16 +5,16 @@ import { dirname, join } from 'node:path'; import { execFileWithStdin, execShellWithStdin } from '../../runtime/exec.js'; import { DEFAULT_MAX_CALLS, - type TargetProxyUsageMetadata, - createTargetProxy, -} from '../../runtime/target-proxy.js'; + type ProviderProxyUsageMetadata, + createProviderProxy, +} from '../../runtime/provider-proxy.js'; import { serializeSnakeCaseBoundaryPayload } from '../case-conversion.js'; import { type ContentImage, isContentArray } from '../content.js'; import type { AssertionEntry, GraderCheckResult, JsonObject, - TargetAccessConfig, + ProviderAccessConfig, } from '../types.js'; import { getRepoCheckoutTargets } from '../workspace/repo-checkout.js'; import { clampScore, isNonEmptyString, parseJsonSafe, scoreToVerdict } from './scoring.js'; @@ -191,8 +191,8 @@ export interface ScriptGraderOptions { readonly agentTimeoutMs?: number; /** Pass-through configuration from YAML (any unrecognized properties) */ readonly config?: Record; - /** Target access config - when present, enables target invocation */ - readonly target?: TargetAccessConfig; + /** Provider access config - when present, enables provider invocation */ + readonly provider?: ProviderAccessConfig; } export class ScriptGrader implements Grader { @@ -202,14 +202,14 @@ export class ScriptGrader implements Grader { private readonly cwd?: string; private readonly agentTimeoutMs?: number; private readonly config?: Record; - private readonly target?: TargetAccessConfig; + private readonly provider?: ProviderAccessConfig; constructor(options: ScriptGraderOptions) { this.command = options.command; this.cwd = options.cwd; this.agentTimeoutMs = options.agentTimeoutMs; this.config = options.config; - this.target = options.target; + this.provider = options.provider; } async evaluate(context: EvaluationContext): Promise { @@ -289,22 +289,22 @@ export class ScriptGrader implements Grader { const inputPayload = JSON.stringify(serializeSnakeCaseBoundaryPayload(payload), null, 2); - // Set up target proxy if configured and grader provider is available + // Set up provider proxy if configured and grader provider is available let proxyEnv: Record | undefined; let proxyShutdown: (() => Promise) | undefined; - let getProxyUsage: (() => TargetProxyUsageMetadata) | undefined; + let getProxyUsage: (() => ProviderProxyUsageMetadata) | undefined; - if (this.target !== undefined && context.graderProvider) { - const maxCalls = this.target.max_calls ?? DEFAULT_MAX_CALLS; - const proxy = await createTargetProxy({ + if (this.provider !== undefined && context.graderProvider) { + const maxCalls = this.provider.max_calls ?? DEFAULT_MAX_CALLS; + const proxy = await createProviderProxy({ defaultProvider: context.graderProvider, - targetResolver: context.targetResolver, - availableTargets: context.availableTargets, + providerResolver: context.targetResolver, + availableProviderLabels: context.availableTargets, maxCalls, }); proxyEnv = { - AGENTV_TARGET_PROXY_URL: proxy.url, - AGENTV_TARGET_PROXY_TOKEN: proxy.token, + AGENTV_PROVIDER_PROXY_URL: proxy.url, + AGENTV_PROVIDER_PROXY_TOKEN: proxy.token, }; proxyShutdown = proxy.shutdown; getProxyUsage = proxy.getUsageMetadata; diff --git a/packages/core/src/evaluation/loaders/grader-parser.ts b/packages/core/src/evaluation/loaders/grader-parser.ts index 1ecbeaf32..e189902ee 100644 --- a/packages/core/src/evaluation/loaders/grader-parser.ts +++ b/packages/core/src/evaluation/loaders/grader-parser.ts @@ -874,27 +874,33 @@ async function parseGraderList( resolvedCwd = searchRoots[0]; } - // Parse optional target config (enables target proxy access) - const rawTarget = rawEvaluator.target; - let targetConfig: import('../types.js').TargetAccessConfig | undefined; - if (rawTarget !== undefined) { - if (isJsonObject(rawTarget)) { - const maxCalls = rawTarget.max_calls; + if (rawEvaluator.target !== undefined) { + throw new Error( + `Script evaluator field 'target' has been removed in '${evalId}' for evaluator '${name}'. Use script evaluator 'provider' instead.`, + ); + } + + // Parse optional provider config (enables provider proxy access) + const rawProvider = rawEvaluator.provider; + let providerConfig: import('../types.js').ProviderAccessConfig | undefined; + if (rawProvider !== undefined) { + if (isJsonObject(rawProvider)) { + const maxCalls = rawProvider.max_calls; if (maxCalls !== undefined && (typeof maxCalls !== 'number' || maxCalls < 0)) { logWarning( - `Invalid target.max_calls for evaluator '${name}' in '${evalId}': must be a non-negative number`, + `Invalid provider.max_calls for evaluator '${name}' in '${evalId}': must be a non-negative number`, ); } else { - targetConfig = { + providerConfig = { ...(typeof maxCalls === 'number' ? { max_calls: maxCalls } : {}), }; } - } else if (rawTarget === true) { - // Support shorthand: `target: true` to enable with defaults - targetConfig = {}; + } else if (rawProvider === true) { + // Support shorthand: `provider: true` to enable with defaults + providerConfig = {}; } else { logWarning( - `Invalid target config for evaluator '${name}' in '${evalId}': expected object or true`, + `Invalid provider config for evaluator '${name}' in '${evalId}': expected object or true`, ); } } @@ -913,7 +919,7 @@ async function parseGraderList( 'command', 'cwd', 'weight', - 'target', + 'provider', 'config', 'preprocessors', 'required', @@ -948,7 +954,7 @@ async function parseGraderList( ...(inheritedInternalPreprocessors ? { preprocessors: inheritedInternalPreprocessors } : {}), - ...(targetConfig !== undefined ? { target: targetConfig } : {}), + ...(providerConfig !== undefined ? { provider: providerConfig } : {}), }); continue; } diff --git a/packages/core/src/evaluation/registry/builtin-graders.ts b/packages/core/src/evaluation/registry/builtin-graders.ts index 260212bce..5c8cf0e47 100644 --- a/packages/core/src/evaluation/registry/builtin-graders.ts +++ b/packages/core/src/evaluation/registry/builtin-graders.ts @@ -206,7 +206,7 @@ export const scriptFactory: GraderFactoryFn = (config, context) => { cwd: c.resolvedCwd ?? c.cwd, agentTimeoutMs: context.agentTimeoutMs, config: c.config, - target: c.target, + provider: c.provider, }); }; diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 0a442c15c..94e551600 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -222,12 +222,12 @@ export function isGraderKind(value: unknown): value is GraderKind { } /** - * Configuration for enabling target access in script evaluators. + * Configuration for enabling provider access in script evaluators. * When present, the runtime will start a local proxy server that allows - * the script to invoke configured targets without direct credential access. + * the script to invoke configured providers without direct credential access. */ -export type TargetAccessConfig = { - /** Maximum number of target invocations allowed per execution (default: 50) */ +export type ProviderAccessConfig = { + /** Maximum number of provider invocations allowed per execution (default: 50) */ readonly max_calls?: number; }; @@ -427,8 +427,8 @@ export type ScriptGraderConfig = { readonly negate?: boolean; /** Pass-through configuration for the script (any unrecognized YAML properties) */ readonly config?: JsonObject; - /** When present, enables target access via local proxy */ - readonly target?: TargetAccessConfig; + /** When present, enables provider access via local proxy */ + readonly provider?: ProviderAccessConfig; /** Optional content preprocessors inherited from suite/evaluator config */ readonly preprocessors?: readonly ContentPreprocessorConfig[]; }; diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 19ce22079..f6e895258 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -191,7 +191,7 @@ const ScriptGraderSchema = EvaluatorCommonSchema.extend({ type: z.literal('script'), command: z.union([z.string(), z.array(z.string())]), cwd: z.string().optional(), - target: z.union([z.boolean(), z.object({ max_calls: z.number().optional() })]).optional(), + provider: z.union([z.boolean(), z.object({ max_calls: z.number().optional() })]).optional(), config: z.record(z.unknown()).optional(), }); @@ -362,7 +362,14 @@ const AssertionObjectSchema = JsonObjectSchema.superRefine((value, ctx) => { message: 'postprocess has been removed. Use transform instead.', }); } - if (typeof value.target === 'string') { + if (value.type === 'script' && value.target !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['target'], + message: + "Script evaluator field 'target' has been removed. Use script evaluator 'provider' instead.", + }); + } else if (typeof value.target === 'string') { ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['target'], diff --git a/packages/core/src/runtime/target-proxy.ts b/packages/core/src/runtime/provider-proxy.ts similarity index 75% rename from packages/core/src/runtime/target-proxy.ts rename to packages/core/src/runtime/provider-proxy.ts index 81da74c2e..f97c8e611 100644 --- a/packages/core/src/runtime/target-proxy.ts +++ b/packages/core/src/runtime/provider-proxy.ts @@ -1,5 +1,5 @@ /** - * Local HTTP proxy server for target invocations from script graders. + * Local HTTP proxy server for provider invocations from script graders. * * Security properties: * - Binds to loopback only (127.0.0.1) @@ -18,19 +18,19 @@ import type { TokenUsage } from '../evaluation/trace.js'; /** * Request body for /invoke endpoint */ -export interface TargetProxyInvokeRequest { +export interface ProviderProxyInvokeRequest { readonly evalCaseId: string; readonly attempt: number; readonly question: string; readonly systemPrompt?: string; - /** Optional target override - use a different target for this invocation */ - readonly target?: string; + /** Optional provider override - use a different provider for this invocation */ + readonly provider?: string; } /** * Response body for /invoke endpoint */ -export interface TargetProxyInvokeResponse { +export interface ProviderProxyInvokeResponse { readonly output: readonly unknown[]; readonly rawText?: string; readonly tokenUsage?: TokenUsage; @@ -39,7 +39,7 @@ export interface TargetProxyInvokeResponse { /** * Proxy usage metadata recorded after execution */ -export interface TargetProxyUsageMetadata { +export interface ProviderProxyUsageMetadata { readonly callCount: number; readonly maxCalls: number; readonly tokenUsage?: TokenUsage; @@ -48,48 +48,50 @@ export interface TargetProxyUsageMetadata { /** * Response body for /info endpoint */ -export interface TargetProxyInfoResponse { - readonly targetName: string; +export interface ProviderProxyInfoResponse { + readonly providerLabel: string; readonly maxCalls: number; readonly callCount: number; - readonly availableTargets: readonly string[]; + readonly availableProviderLabels: readonly string[]; } /** - * Function to resolve a target name to a provider + * Function to resolve a provider label to a provider */ -export type TargetResolver = (targetName: string) => Provider | undefined; +export type ProviderResolver = (providerLabel: string) => Provider | undefined; /** - * Options for creating a target proxy + * Options for creating a provider proxy */ -export interface TargetProxyOptions { +export interface ProviderProxyOptions { readonly defaultProvider: Provider; - /** Optional resolver for target override - if not provided, only default target is available */ - readonly targetResolver?: TargetResolver; - /** Names of all available targets (for /info endpoint) */ - readonly availableTargets?: readonly string[]; + /** Optional resolver for provider override - if not provided, only default provider is available */ + readonly providerResolver?: ProviderResolver; + /** Labels of all available providers (for /info endpoint) */ + readonly availableProviderLabels?: readonly string[]; readonly maxCalls: number; } /** - * Active target proxy instance + * Active provider proxy instance */ -export interface TargetProxyInstance { +export interface ProviderProxyInstance { readonly url: string; readonly token: string; readonly shutdown: () => Promise; - readonly getUsageMetadata: () => TargetProxyUsageMetadata; + readonly getUsageMetadata: () => ProviderProxyUsageMetadata; } /** Default max calls if not specified */ export const DEFAULT_MAX_CALLS = 50; /** - * Create and start a target proxy server. + * Create and start a provider proxy server. */ -export async function createTargetProxy(options: TargetProxyOptions): Promise { - const { defaultProvider, targetResolver, availableTargets, maxCalls } = options; +export async function createProviderProxy( + options: ProviderProxyOptions, +): Promise { + const { defaultProvider, providerResolver, availableProviderLabels, maxCalls } = options; // Generate unique token for this invocation const token = randomBytes(32).toString('hex'); @@ -99,20 +101,20 @@ export async function createTargetProxy(options: TargetProxyOptions): Promise { try { const body = await readBody(req); - const { requests } = JSON.parse(body) as { requests: TargetProxyInvokeRequest[] }; + const { requests } = JSON.parse(body) as { requests: ProviderProxyInvokeRequest[] }; if (!Array.isArray(requests)) { sendJson(res, 400, { error: 'Missing required field: requests (array)' }); @@ -247,7 +249,7 @@ export async function createTargetProxy(options: TargetProxyOptions): Promise { expect(config.config).toEqual({ threshold: 0.9, algorithm: 'levenshtein' }); }); + it('parses script provider proxy config', async () => { + const rawEvalCase = { + assert: [ + { + metric: 'provider-backed-script', + type: 'script', + command: ['bun', 'run', './test_script.ts'], + provider: { max_calls: 2 }, + }, + ], + }; + + const evaluators = await parseGraders(rawEvalCase, undefined, [tempDir], 'test-case'); + + expect(evaluators).toHaveLength(1); + const config = evaluators?.[0] as ScriptGraderConfig; + expect(config.provider).toEqual({ max_calls: 2 }); + expect(config.config).toBeUndefined(); + }); + + it('rejects removed script provider proxy target config', async () => { + await expect( + parseGraders( + { + assert: [ + { + metric: 'legacy-provider-access', + type: 'script', + command: ['bun', 'run', './test_script.ts'], + target: { max_calls: 2 }, + }, + ], + }, + undefined, + [tempDir], + 'test-case', + ), + ).rejects.toThrow(/Script evaluator field 'target' has been removed.*provider/); + }); + it('converts string commands into argv using a shell', async () => { const rawEvalCase = { assert: [ diff --git a/packages/core/test/evaluation/token-usage.test.ts b/packages/core/test/evaluation/token-usage.test.ts index f5f627cda..477aba5b9 100644 --- a/packages/core/test/evaluation/token-usage.test.ts +++ b/packages/core/test/evaluation/token-usage.test.ts @@ -1,6 +1,6 @@ /** * Tests for token usage tracking across the evaluation pipeline. - * Covers: AI SDK mapResponse, target proxy accumulation, orchestrator passthrough. + * Covers: AI SDK mapResponse, provider proxy accumulation, orchestrator passthrough. */ import { describe, expect, it } from 'bun:test'; @@ -58,8 +58,8 @@ describe('token usage type contracts', () => { }); }); -// ─── Target proxy token usage accumulation ───────────────────────────── -describe('target proxy token usage accumulation', () => { +// ─── Provider proxy token usage accumulation ─────────────────────────── +describe('provider proxy token usage accumulation', () => { function createMockProviderWithUsage( targetName: string, tokenUsage: { input: number; output: number }, @@ -76,10 +76,10 @@ describe('target proxy token usage accumulation', () => { } it('accumulates tokenUsage across multiple invoke calls', async () => { - const { createTargetProxy } = await import('../../src/runtime/target-proxy.js'); + const { createProviderProxy } = await import('../../src/runtime/provider-proxy.js'); const provider = createMockProviderWithUsage('test', { input: 100, output: 50 }); - const proxy = await createTargetProxy({ defaultProvider: provider, maxCalls: 10 }); + const proxy = await createProviderProxy({ defaultProvider: provider, maxCalls: 10 }); try { const headers = { @@ -105,10 +105,10 @@ describe('target proxy token usage accumulation', () => { }); it('returns per-call tokenUsage in invoke response', async () => { - const { createTargetProxy } = await import('../../src/runtime/target-proxy.js'); + const { createProviderProxy } = await import('../../src/runtime/provider-proxy.js'); const provider = createMockProviderWithUsage('test', { input: 42, output: 17 }); - const proxy = await createTargetProxy({ defaultProvider: provider, maxCalls: 10 }); + const proxy = await createProviderProxy({ defaultProvider: provider, maxCalls: 10 }); try { const response = await fetch(`${proxy.url}/invoke`, { @@ -128,7 +128,7 @@ describe('target proxy token usage accumulation', () => { }); it('returns undefined tokenUsage when provider reports none', async () => { - const { createTargetProxy } = await import('../../src/runtime/target-proxy.js'); + const { createProviderProxy } = await import('../../src/runtime/provider-proxy.js'); const provider: Provider = { id: 'no-usage', @@ -138,7 +138,7 @@ describe('target proxy token usage accumulation', () => { output: [{ role: 'assistant', content: 'response' }], }), }; - const proxy = await createTargetProxy({ defaultProvider: provider, maxCalls: 10 }); + const proxy = await createProviderProxy({ defaultProvider: provider, maxCalls: 10 }); try { await fetch(`${proxy.url}/invoke`, { @@ -159,10 +159,10 @@ describe('target proxy token usage accumulation', () => { }); it('accumulates tokenUsage in batch requests', async () => { - const { createTargetProxy } = await import('../../src/runtime/target-proxy.js'); + const { createProviderProxy } = await import('../../src/runtime/provider-proxy.js'); const provider = createMockProviderWithUsage('test', { input: 10, output: 5 }); - const proxy = await createTargetProxy({ defaultProvider: provider, maxCalls: 10 }); + const proxy = await createProviderProxy({ defaultProvider: provider, maxCalls: 10 }); try { const response = await fetch(`${proxy.url}/invokeBatch`, { diff --git a/packages/core/test/runtime/provider-proxy.test.ts b/packages/core/test/runtime/provider-proxy.test.ts new file mode 100644 index 000000000..5f278c9c5 --- /dev/null +++ b/packages/core/test/runtime/provider-proxy.test.ts @@ -0,0 +1,243 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; + +import type { + Provider, + ProviderRequest, + ProviderResponse, +} from '../../src/evaluation/providers/types.js'; +import { + type ProviderProxyInfoResponse, + type ProviderProxyInvokeResponse, + createProviderProxy, +} from '../../src/runtime/provider-proxy.js'; + +function createMockProvider(providerLabel: string): Provider { + return { + id: providerLabel, + kind: 'mock', + targetName: providerLabel, + invoke: async (request: ProviderRequest): Promise => ({ + output: [ + { role: 'assistant', content: `Response from ${providerLabel}: ${request.question}` }, + ], + }), + }; +} + +describe('createProviderProxy', () => { + let shutdown: (() => Promise) | undefined; + + afterEach(async () => { + if (shutdown) { + await shutdown(); + shutdown = undefined; + } + }); + + describe('/info endpoint', () => { + it('returns proxy info with default provider', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/info`, { + headers: { Authorization: `Bearer ${proxy.token}` }, + }); + + expect(response.ok).toBe(true); + const info = (await response.json()) as ProviderProxyInfoResponse; + expect(info.providerLabel).toBe('default-provider'); + expect(info.maxCalls).toBe(50); + expect(info.callCount).toBe(0); + expect(info.availableProviderLabels).toEqual(['default-provider']); + }); + + it('returns all available providers when providerResolver is provided', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + providerResolver: (name) => + name === 'alt-provider' ? createMockProvider('alt-provider') : undefined, + availableProviderLabels: ['default-provider', 'alt-provider'], + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/info`, { + headers: { Authorization: `Bearer ${proxy.token}` }, + }); + + expect(response.ok).toBe(true); + const info = (await response.json()) as ProviderProxyInfoResponse; + expect(info.availableProviderLabels).toEqual(['default-provider', 'alt-provider']); + }); + + it('updates callCount after invocations', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + // Make an invoke call + await fetch(`${proxy.url}/invoke`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${proxy.token}`, + }, + body: JSON.stringify({ question: 'test' }), + }); + + const response = await fetch(`${proxy.url}/info`, { + headers: { Authorization: `Bearer ${proxy.token}` }, + }); + + const info = (await response.json()) as ProviderProxyInfoResponse; + expect(info.callCount).toBe(1); + }); + + it('requires authentication', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/info`); + expect(response.status).toBe(401); + }); + }); + + describe('provider override', () => { + it('uses default provider when no provider is specified', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/invoke`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${proxy.token}`, + }, + body: JSON.stringify({ question: 'test' }), + }); + + expect(response.ok).toBe(true); + const result = (await response.json()) as ProviderProxyInvokeResponse; + expect(result.rawText).toContain('default-provider'); + }); + + it('uses specified provider when provider is provided', async () => { + const defaultProvider = createMockProvider('default-provider'); + const altProvider = createMockProvider('alt-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + providerResolver: (name) => (name === 'alt-provider' ? altProvider : undefined), + availableProviderLabels: ['default-provider', 'alt-provider'], + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/invoke`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${proxy.token}`, + }, + body: JSON.stringify({ question: 'test', provider: 'alt-provider' }), + }); + + expect(response.ok).toBe(true); + const result = (await response.json()) as ProviderProxyInvokeResponse; + expect(result.rawText).toContain('alt-provider'); + }); + + it('returns 400 for unknown provider', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + availableProviderLabels: ['default-provider'], + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/invoke`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${proxy.token}`, + }, + body: JSON.stringify({ question: 'test', provider: 'unknown-provider' }), + }); + + expect(response.status).toBe(400); + const body = (await response.json()) as { error: string }; + expect(body.error).toContain('Unknown provider'); + expect(body.error).toContain('default-provider'); + }); + + it('supports provider override in batch requests', async () => { + const defaultProvider = createMockProvider('default-provider'); + const altProvider = createMockProvider('alt-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + providerResolver: (name) => (name === 'alt-provider' ? altProvider : undefined), + availableProviderLabels: ['default-provider', 'alt-provider'], + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/invokeBatch`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${proxy.token}`, + }, + body: JSON.stringify({ + requests: [{ question: 'q1' }, { question: 'q2', provider: 'alt-provider' }], + }), + }); + + expect(response.ok).toBe(true); + const result = (await response.json()) as { responses: ProviderProxyInvokeResponse[] }; + expect(result.responses[0].rawText).toContain('default-provider'); + expect(result.responses[1].rawText).toContain('alt-provider'); + }); + + it('handles unknown provider in batch gracefully', async () => { + const defaultProvider = createMockProvider('default-provider'); + const proxy = await createProviderProxy({ + defaultProvider, + availableProviderLabels: ['default-provider'], + maxCalls: 50, + }); + shutdown = proxy.shutdown; + + const response = await fetch(`${proxy.url}/invokeBatch`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${proxy.token}`, + }, + body: JSON.stringify({ + requests: [{ question: 'q1' }, { question: 'q2', provider: 'unknown-provider' }], + }), + }); + + expect(response.ok).toBe(true); + const result = (await response.json()) as { responses: ProviderProxyInvokeResponse[] }; + expect(result.responses[0].rawText).toContain('default-provider'); + expect(result.responses[1].rawText).toContain('Unknown provider'); + }); + }); +}); diff --git a/packages/core/test/runtime/target-proxy.test.ts b/packages/core/test/runtime/target-proxy.test.ts deleted file mode 100644 index 1dc9edc5c..000000000 --- a/packages/core/test/runtime/target-proxy.test.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; - -import type { - Provider, - ProviderRequest, - ProviderResponse, -} from '../../src/evaluation/providers/types.js'; -import { - type TargetProxyInfoResponse, - type TargetProxyInvokeResponse, - createTargetProxy, -} from '../../src/runtime/target-proxy.js'; - -function createMockProvider(targetName: string): Provider { - return { - id: targetName, - kind: 'mock', - targetName, - invoke: async (request: ProviderRequest): Promise => ({ - output: [{ role: 'assistant', content: `Response from ${targetName}: ${request.question}` }], - }), - }; -} - -describe('createTargetProxy', () => { - let shutdown: (() => Promise) | undefined; - - afterEach(async () => { - if (shutdown) { - await shutdown(); - shutdown = undefined; - } - }); - - describe('/info endpoint', () => { - it('returns proxy info with default target', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/info`, { - headers: { Authorization: `Bearer ${proxy.token}` }, - }); - - expect(response.ok).toBe(true); - const info = (await response.json()) as TargetProxyInfoResponse; - expect(info.targetName).toBe('default-target'); - expect(info.maxCalls).toBe(50); - expect(info.callCount).toBe(0); - expect(info.availableTargets).toEqual(['default-target']); - }); - - it('returns all available targets when targetResolver is provided', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - targetResolver: (name) => - name === 'alt-target' ? createMockProvider('alt-target') : undefined, - availableTargets: ['default-target', 'alt-target'], - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/info`, { - headers: { Authorization: `Bearer ${proxy.token}` }, - }); - - expect(response.ok).toBe(true); - const info = (await response.json()) as TargetProxyInfoResponse; - expect(info.availableTargets).toEqual(['default-target', 'alt-target']); - }); - - it('updates callCount after invocations', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - // Make an invoke call - await fetch(`${proxy.url}/invoke`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${proxy.token}`, - }, - body: JSON.stringify({ question: 'test' }), - }); - - const response = await fetch(`${proxy.url}/info`, { - headers: { Authorization: `Bearer ${proxy.token}` }, - }); - - const info = (await response.json()) as TargetProxyInfoResponse; - expect(info.callCount).toBe(1); - }); - - it('requires authentication', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/info`); - expect(response.status).toBe(401); - }); - }); - - describe('target override', () => { - it('uses default target when no target specified', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/invoke`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${proxy.token}`, - }, - body: JSON.stringify({ question: 'test' }), - }); - - expect(response.ok).toBe(true); - const result = (await response.json()) as TargetProxyInvokeResponse; - expect(result.rawText).toContain('default-target'); - }); - - it('uses specified target when target is provided', async () => { - const defaultProvider = createMockProvider('default-target'); - const altProvider = createMockProvider('alt-target'); - const proxy = await createTargetProxy({ - defaultProvider, - targetResolver: (name) => (name === 'alt-target' ? altProvider : undefined), - availableTargets: ['default-target', 'alt-target'], - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/invoke`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${proxy.token}`, - }, - body: JSON.stringify({ question: 'test', target: 'alt-target' }), - }); - - expect(response.ok).toBe(true); - const result = (await response.json()) as TargetProxyInvokeResponse; - expect(result.rawText).toContain('alt-target'); - }); - - it('returns 400 for unknown target', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - availableTargets: ['default-target'], - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/invoke`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${proxy.token}`, - }, - body: JSON.stringify({ question: 'test', target: 'unknown-target' }), - }); - - expect(response.status).toBe(400); - const body = (await response.json()) as { error: string }; - expect(body.error).toContain('Unknown target'); - expect(body.error).toContain('default-target'); - }); - - it('supports target override in batch requests', async () => { - const defaultProvider = createMockProvider('default-target'); - const altProvider = createMockProvider('alt-target'); - const proxy = await createTargetProxy({ - defaultProvider, - targetResolver: (name) => (name === 'alt-target' ? altProvider : undefined), - availableTargets: ['default-target', 'alt-target'], - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/invokeBatch`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${proxy.token}`, - }, - body: JSON.stringify({ - requests: [{ question: 'q1' }, { question: 'q2', target: 'alt-target' }], - }), - }); - - expect(response.ok).toBe(true); - const result = (await response.json()) as { responses: TargetProxyInvokeResponse[] }; - expect(result.responses[0].rawText).toContain('default-target'); - expect(result.responses[1].rawText).toContain('alt-target'); - }); - - it('handles unknown target in batch gracefully', async () => { - const defaultProvider = createMockProvider('default-target'); - const proxy = await createTargetProxy({ - defaultProvider, - availableTargets: ['default-target'], - maxCalls: 50, - }); - shutdown = proxy.shutdown; - - const response = await fetch(`${proxy.url}/invokeBatch`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${proxy.token}`, - }, - body: JSON.stringify({ - requests: [{ question: 'q1' }, { question: 'q2', target: 'unknown-target' }], - }), - }); - - expect(response.ok).toBe(true); - const result = (await response.json()) as { responses: TargetProxyInvokeResponse[] }; - expect(result.responses[0].rawText).toContain('default-target'); - expect(result.responses[1].rawText).toContain('Unknown target'); - }); - }); -}); diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 7ec5deebe..1e1eb675a 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -205,16 +205,16 @@ export { type RegexGraderOptions, } from './graders.js'; -// Re-export target client +// Re-export provider client export { - createTargetClient, - TargetNotAvailableError, - TargetInvocationError, - type TargetClient, - type TargetInfo, - type TargetInvokeRequest, - type TargetInvokeResponse, -} from './target-client.js'; + createProviderClient, + ProviderNotAvailableError, + ProviderInvocationError, + type ProviderClient, + type ProviderInfo, + type ProviderInvokeRequest, + type ProviderInvokeResponse, +} from './provider-client.js'; // Re-export workspace grader helpers export { diff --git a/packages/sdk/src/provider-client.ts b/packages/sdk/src/provider-client.ts new file mode 100644 index 000000000..589c293b2 --- /dev/null +++ b/packages/sdk/src/provider-client.ts @@ -0,0 +1,247 @@ +/** + * Runtime-only client for invoking configured providers from script-grader scripts + * through AgentV's provider proxy. + * + * Environment variables (set automatically by AgentV when provider proxy access is enabled): + * - AGENTV_PROVIDER_PROXY_URL: The URL of the local proxy server + * - AGENTV_PROVIDER_PROXY_TOKEN: Bearer token for authentication + */ + +import type { TokenUsage } from './schemas.js'; + +/** + * Request to invoke the provider + */ +export interface ProviderInvokeRequest { + readonly question: string; + readonly systemPrompt?: string; + readonly evalCaseId?: string; + readonly attempt?: number; + /** Optional provider override - use a different provider for this invocation */ + readonly provider?: string; +} + +/** + * Response from a provider invocation + */ +export interface ProviderInvokeResponse { + readonly output: readonly unknown[]; + readonly rawText?: string; + readonly tokenUsage?: TokenUsage; +} + +/** + * Information about the provider proxy configuration + */ +export interface ProviderInfo { + /** Label of the default provider being used */ + readonly providerLabel: string; + /** Maximum number of calls allowed */ + readonly maxCalls: number; + /** Current number of calls made */ + readonly callCount: number; + /** Labels of all available providers */ + readonly availableProviderLabels: readonly string[]; +} + +/** + * Provider client for making provider invocations + */ +export interface ProviderClient { + /** + * Invoke the configured provider with a prompt. + * @param request - The question and optional system prompt + * @returns The provider's response with output messages and optional raw text + */ + invoke(request: ProviderInvokeRequest): Promise; + + /** + * Invoke the provider with multiple requests in sequence. + * Each request counts toward the max_calls limit. + * @param requests - Array of provider requests + * @returns Array of provider responses + */ + invokeBatch( + requests: readonly ProviderInvokeRequest[], + ): Promise; + + /** + * Get information about the provider proxy configuration. + * Returns the default provider label, max calls, current call count, and available providers. + */ + getInfo(): Promise; +} + +/** + * Error thrown when provider proxy is not available + */ +export class ProviderNotAvailableError extends Error { + constructor(message: string) { + super(message); + this.name = 'ProviderNotAvailableError'; + } +} + +/** + * Error thrown when provider invocation fails + */ +export class ProviderInvocationError extends Error { + readonly statusCode?: number; + + constructor(message: string, statusCode?: number) { + super(message); + this.name = 'ProviderInvocationError'; + this.statusCode = statusCode; + } +} + +/** + * Create a provider client from environment variables. + * + * This function reads the proxy URL and token from environment variables + * that are automatically set by AgentV when provider access is enabled on a + * `script` evaluator. + * + * @returns A provider client if environment variables are set, otherwise undefined + * @throws ProviderNotAvailableError if token is missing when URL is present + * + * @example + * ```typescript + * import { createProviderClient, defineScriptGrader } from 'agentv'; + * + * export default defineScriptGrader(async ({ input, criteria, output }) => { + * const provider = createProviderClient(); + * const question = input + * .filter((message) => message.role === 'user') + * .map((message) => typeof message.content === 'string' ? message.content : '') + * .join('\n'); + * + * if (!provider) { + * return { pass: false, score: 0.5, reason: 'Provider proxy not available' }; + * } + * + * const response = await provider.invoke({ + * question: `Is this answer correct? Question: ${question}, Expected: ${criteria}, Answer: ${output ?? ''}`, + * systemPrompt: 'You are an expert grader. Respond with JSON: { "correct": true/false }' + * }); + * + * const result = JSON.parse(response.rawText ?? '{}'); + * return { + * pass: result.correct === true, + * score: result.correct === true ? 1.0 : 0.0, + * reason: result.correct === true ? 'Provider judged the answer correct' : 'Provider judged the answer incorrect', + * }; + * }); + * ``` + */ +export function createProviderClient(): ProviderClient | undefined { + const proxyUrl = process.env.AGENTV_PROVIDER_PROXY_URL; + const proxyToken = process.env.AGENTV_PROVIDER_PROXY_TOKEN; + + if (!proxyUrl) { + return undefined; + } + + if (!proxyToken) { + throw new ProviderNotAvailableError( + 'AGENTV_PROVIDER_PROXY_URL is set but AGENTV_PROVIDER_PROXY_TOKEN is missing', + ); + } + + return createProviderClientInternal(proxyUrl, proxyToken); +} + +/** + * Internal: Create a provider client with explicit URL and token. + * Exported for testing only - use createProviderClient() in production. + */ +export function createProviderClientInternal(url: string, token: string): ProviderClient { + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${token}`, + }; + + return { + async invoke(request: ProviderInvokeRequest): Promise { + const response = await fetch(`${url}/invoke`, { + method: 'POST', + headers, + body: JSON.stringify({ + question: request.question, + systemPrompt: request.systemPrompt, + evalCaseId: request.evalCaseId, + attempt: request.attempt, + provider: request.provider, + }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + let errorMessage: string; + try { + const errorJson = JSON.parse(errorBody) as { error?: string }; + errorMessage = errorJson.error ?? `HTTP ${response.status}`; + } catch { + errorMessage = errorBody || `HTTP ${response.status}`; + } + throw new ProviderInvocationError(errorMessage, response.status); + } + + return (await response.json()) as ProviderInvokeResponse; + }, + + async invokeBatch( + requests: readonly ProviderInvokeRequest[], + ): Promise { + const response = await fetch(`${url}/invokeBatch`, { + method: 'POST', + headers, + body: JSON.stringify({ + requests: requests.map((r) => ({ + question: r.question, + systemPrompt: r.systemPrompt, + evalCaseId: r.evalCaseId, + attempt: r.attempt, + provider: r.provider, + })), + }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + let errorMessage: string; + try { + const errorJson = JSON.parse(errorBody) as { error?: string }; + errorMessage = errorJson.error ?? `HTTP ${response.status}`; + } catch { + errorMessage = errorBody || `HTTP ${response.status}`; + } + throw new ProviderInvocationError(errorMessage, response.status); + } + + const result = (await response.json()) as { responses: ProviderInvokeResponse[] }; + return result.responses; + }, + + async getInfo(): Promise { + const response = await fetch(`${url}/info`, { + method: 'GET', + headers, + }); + + if (!response.ok) { + const errorBody = await response.text(); + let errorMessage: string; + try { + const errorJson = JSON.parse(errorBody) as { error?: string }; + errorMessage = errorJson.error ?? `HTTP ${response.status}`; + } catch { + errorMessage = errorBody || `HTTP ${response.status}`; + } + throw new ProviderInvocationError(errorMessage, response.status); + } + + return (await response.json()) as ProviderInfo; + }, + }; +} diff --git a/packages/sdk/src/target-client.ts b/packages/sdk/src/target-client.ts deleted file mode 100644 index 1556f433c..000000000 --- a/packages/sdk/src/target-client.ts +++ /dev/null @@ -1,251 +0,0 @@ -/** - * Runtime-only client for invoking configured providers from script-grader scripts - * through AgentV's target proxy. - * - * This module keeps the historical target-proxy vocabulary because it talks to - * runtime internals. It is not TypeScript eval authoring syntax; eval configs - * should select systems under test and grader providers through `providers`, - * `defaults.provider`, `defaults.grader`, and assertion `provider`. - * - * Environment variables (set automatically by AgentV when `target` config is present): - * - AGENTV_TARGET_PROXY_URL: The URL of the local proxy server - * - AGENTV_TARGET_PROXY_TOKEN: Bearer token for authentication - */ - -import type { TokenUsage } from './schemas.js'; - -/** - * Request to invoke the target - */ -export interface TargetInvokeRequest { - readonly question: string; - readonly systemPrompt?: string; - readonly evalCaseId?: string; - readonly attempt?: number; - /** Optional target override - use a different target for this invocation */ - readonly target?: string; -} - -/** - * Response from a target invocation - */ -export interface TargetInvokeResponse { - readonly output: readonly unknown[]; - readonly rawText?: string; - readonly tokenUsage?: TokenUsage; -} - -/** - * Information about the target proxy configuration - */ -export interface TargetInfo { - /** Name of the default target being used */ - readonly targetName: string; - /** Maximum number of calls allowed */ - readonly maxCalls: number; - /** Current number of calls made */ - readonly callCount: number; - /** List of all available target names */ - readonly availableTargets: readonly string[]; -} - -/** - * Target client for making target invocations - */ -export interface TargetClient { - /** - * Invoke the configured target with a prompt. - * @param request - The question and optional system prompt - * @returns The target's response with output messages and optional raw text - */ - invoke(request: TargetInvokeRequest): Promise; - - /** - * Invoke the target with multiple requests in sequence. - * Each request counts toward the max_calls limit. - * @param requests - Array of target requests - * @returns Array of target responses - */ - invokeBatch(requests: readonly TargetInvokeRequest[]): Promise; - - /** - * Get information about the target proxy configuration. - * Returns the default target name, max calls, current call count, and available targets. - */ - getInfo(): Promise; -} - -/** - * Error thrown when target proxy is not available - */ -export class TargetNotAvailableError extends Error { - constructor(message: string) { - super(message); - this.name = 'TargetNotAvailableError'; - } -} - -/** - * Error thrown when target invocation fails - */ -export class TargetInvocationError extends Error { - readonly statusCode?: number; - - constructor(message: string, statusCode?: number) { - super(message); - this.name = 'TargetInvocationError'; - this.statusCode = statusCode; - } -} - -/** - * Create a target client from environment variables. - * - * This function reads the proxy URL and token from environment variables - * that are automatically set by AgentV when a `target` config block is present - * on a `script` evaluator. - * - * @returns A target client if environment variables are set, otherwise undefined - * @throws TargetNotAvailableError if token is missing when URL is present - * - * @example - * ```typescript - * import { createTargetClient, defineScriptGrader } from 'agentv'; - * - * export default defineScriptGrader(async ({ input, criteria, output }) => { - * const target = createTargetClient(); - * const question = input - * .filter((message) => message.role === 'user') - * .map((message) => typeof message.content === 'string' ? message.content : '') - * .join('\n'); - * - * if (!target) { - * // Target not available - no target config on this evaluator - * return { pass: false, score: 0.5, reason: 'Target not available' }; - * } - * - * const response = await target.invoke({ - * question: `Is this answer correct? Question: ${question}, Expected: ${criteria}, Answer: ${output ?? ''}`, - * systemPrompt: 'You are an expert grader. Respond with JSON: { "correct": true/false }' - * }); - * - * const result = JSON.parse(response.rawText ?? '{}'); - * return { - * pass: result.correct === true, - * score: result.correct === true ? 1.0 : 0.0, - * reason: result.correct === true ? 'Target judged the answer correct' : 'Target judged the answer incorrect', - * }; - * }); - * ``` - */ -export function createTargetClient(): TargetClient | undefined { - const proxyUrl = process.env.AGENTV_TARGET_PROXY_URL; - const proxyToken = process.env.AGENTV_TARGET_PROXY_TOKEN; - - if (!proxyUrl) { - return undefined; - } - - if (!proxyToken) { - throw new TargetNotAvailableError( - 'AGENTV_TARGET_PROXY_URL is set but AGENTV_TARGET_PROXY_TOKEN is missing', - ); - } - - return createTargetClientInternal(proxyUrl, proxyToken); -} - -/** - * Internal: Create a target client with explicit URL and token. - * Exported for testing only - use createTargetClient() in production. - */ -export function createTargetClientInternal(url: string, token: string): TargetClient { - const headers = { - 'Content-Type': 'application/json', - Authorization: `Bearer ${token}`, - }; - - return { - async invoke(request: TargetInvokeRequest): Promise { - const response = await fetch(`${url}/invoke`, { - method: 'POST', - headers, - body: JSON.stringify({ - question: request.question, - systemPrompt: request.systemPrompt, - evalCaseId: request.evalCaseId, - attempt: request.attempt, - target: request.target, - }), - }); - - if (!response.ok) { - const errorBody = await response.text(); - let errorMessage: string; - try { - const errorJson = JSON.parse(errorBody) as { error?: string }; - errorMessage = errorJson.error ?? `HTTP ${response.status}`; - } catch { - errorMessage = errorBody || `HTTP ${response.status}`; - } - throw new TargetInvocationError(errorMessage, response.status); - } - - return (await response.json()) as TargetInvokeResponse; - }, - - async invokeBatch( - requests: readonly TargetInvokeRequest[], - ): Promise { - const response = await fetch(`${url}/invokeBatch`, { - method: 'POST', - headers, - body: JSON.stringify({ - requests: requests.map((r) => ({ - question: r.question, - systemPrompt: r.systemPrompt, - evalCaseId: r.evalCaseId, - attempt: r.attempt, - target: r.target, - })), - }), - }); - - if (!response.ok) { - const errorBody = await response.text(); - let errorMessage: string; - try { - const errorJson = JSON.parse(errorBody) as { error?: string }; - errorMessage = errorJson.error ?? `HTTP ${response.status}`; - } catch { - errorMessage = errorBody || `HTTP ${response.status}`; - } - throw new TargetInvocationError(errorMessage, response.status); - } - - const result = (await response.json()) as { responses: TargetInvokeResponse[] }; - return result.responses; - }, - - async getInfo(): Promise { - const response = await fetch(`${url}/info`, { - method: 'GET', - headers, - }); - - if (!response.ok) { - const errorBody = await response.text(); - let errorMessage: string; - try { - const errorJson = JSON.parse(errorBody) as { error?: string }; - errorMessage = errorJson.error ?? `HTTP ${response.status}`; - } catch { - errorMessage = errorBody || `HTTP ${response.status}`; - } - throw new TargetInvocationError(errorMessage, response.status); - } - - return (await response.json()) as TargetInfo; - }, - }; -} diff --git a/packages/sdk/test/target-client.test.ts b/packages/sdk/test/provider-client.test.ts similarity index 62% rename from packages/sdk/test/target-client.test.ts rename to packages/sdk/test/provider-client.test.ts index c07e692a7..544c77846 100644 --- a/packages/sdk/test/target-client.test.ts +++ b/packages/sdk/test/provider-client.test.ts @@ -1,54 +1,54 @@ import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test'; import { - TargetInvocationError, - TargetNotAvailableError, - createTargetClient, - createTargetClientInternal, -} from '../src/target-client.js'; + ProviderInvocationError, + ProviderNotAvailableError, + createProviderClient, + createProviderClientInternal, +} from '../src/provider-client.js'; -describe('createTargetClient', () => { - const originalUrl = process.env.AGENTV_TARGET_PROXY_URL; - const originalToken = process.env.AGENTV_TARGET_PROXY_TOKEN; +describe('createProviderClient', () => { + const originalUrl = process.env.AGENTV_PROVIDER_PROXY_URL; + const originalToken = process.env.AGENTV_PROVIDER_PROXY_TOKEN; beforeEach(() => { - process.env.AGENTV_TARGET_PROXY_URL = ''; - process.env.AGENTV_TARGET_PROXY_TOKEN = ''; + process.env.AGENTV_PROVIDER_PROXY_URL = ''; + process.env.AGENTV_PROVIDER_PROXY_TOKEN = ''; }); afterEach(() => { if (originalUrl === undefined) { - process.env.AGENTV_TARGET_PROXY_URL = ''; + process.env.AGENTV_PROVIDER_PROXY_URL = ''; } else { - process.env.AGENTV_TARGET_PROXY_URL = originalUrl; + process.env.AGENTV_PROVIDER_PROXY_URL = originalUrl; } if (originalToken === undefined) { - process.env.AGENTV_TARGET_PROXY_TOKEN = ''; + process.env.AGENTV_PROVIDER_PROXY_TOKEN = ''; } else { - process.env.AGENTV_TARGET_PROXY_TOKEN = originalToken; + process.env.AGENTV_PROVIDER_PROXY_TOKEN = originalToken; } }); it('returns undefined when no env vars are set', () => { - const client = createTargetClient(); + const client = createProviderClient(); expect(client).toBeUndefined(); }); - it('throws TargetNotAvailableError when URL is set but token is missing', () => { - process.env.AGENTV_TARGET_PROXY_URL = 'http://127.0.0.1:3000'; + it('throws ProviderNotAvailableError when URL is set but token is missing', () => { + process.env.AGENTV_PROVIDER_PROXY_URL = 'http://127.0.0.1:3000'; - expect(() => createTargetClient()).toThrow(TargetNotAvailableError); - expect(() => createTargetClient()).toThrow( - 'AGENTV_TARGET_PROXY_URL is set but AGENTV_TARGET_PROXY_TOKEN is missing', + expect(() => createProviderClient()).toThrow(ProviderNotAvailableError); + expect(() => createProviderClient()).toThrow( + 'AGENTV_PROVIDER_PROXY_URL is set but AGENTV_PROVIDER_PROXY_TOKEN is missing', ); }); it('returns client when both env vars are set', () => { - process.env.AGENTV_TARGET_PROXY_URL = 'http://127.0.0.1:3000'; - process.env.AGENTV_TARGET_PROXY_TOKEN = 'test-token-123'; + process.env.AGENTV_PROVIDER_PROXY_URL = 'http://127.0.0.1:3000'; + process.env.AGENTV_PROVIDER_PROXY_TOKEN = 'test-token-123'; - const client = createTargetClient(); + const client = createProviderClient(); expect(client).toBeDefined(); expect(typeof client?.invoke).toBe('function'); expect(typeof client?.invokeBatch).toBe('function'); @@ -56,24 +56,24 @@ describe('createTargetClient', () => { }); }); -describe('createTargetClientInternal', () => { +describe('createProviderClientInternal', () => { it('creates client with invoke method', () => { - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); expect(typeof client.invoke).toBe('function'); }); it('creates client with invokeBatch method', () => { - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); expect(typeof client.invokeBatch).toBe('function'); }); it('creates client with getInfo method', () => { - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); expect(typeof client.getInfo).toBe('function'); }); }); -describe('TargetClient.invoke', () => { +describe('ProviderClient.invoke', () => { it('makes POST request with correct headers', async () => { const mockResponse = { output: [{ role: 'assistant', content: 'test response' }], @@ -85,7 +85,7 @@ describe('TargetClient.invoke', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'secret-token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'secret-token'); await client.invoke({ question: 'test question' }); expect(fetchSpy).toHaveBeenCalledWith( @@ -113,7 +113,7 @@ describe('TargetClient.invoke', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); const response = await client.invoke({ question: 'test' }); expect(response.output).toEqual([{ role: 'assistant', content: 'test' }]); @@ -122,23 +122,23 @@ describe('TargetClient.invoke', () => { fetchSpy.mockRestore(); }); - it('throws TargetInvocationError on non-ok response', async () => { + it('throws ProviderInvocationError on non-ok response', async () => { const fetchSpy = spyOn(global, 'fetch').mockResolvedValue({ ok: false, status: 429, text: () => Promise.resolve('{"error":"Max calls exceeded"}'), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); - let error: TargetInvocationError | undefined; + let error: ProviderInvocationError | undefined; try { await client.invoke({ question: 'test' }); } catch (e) { - error = e as TargetInvocationError; + error = e as ProviderInvocationError; } - expect(error).toBeInstanceOf(TargetInvocationError); + expect(error).toBeInstanceOf(ProviderInvocationError); expect(error?.message).toBe('Max calls exceeded'); expect(error?.statusCode).toBe(429); @@ -152,16 +152,16 @@ describe('TargetClient.invoke', () => { text: () => Promise.resolve('Internal Server Error'), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); - let error: TargetInvocationError | undefined; + let error: ProviderInvocationError | undefined; try { await client.invoke({ question: 'test' }); } catch (e) { - error = e as TargetInvocationError; + error = e as ProviderInvocationError; } - expect(error).toBeInstanceOf(TargetInvocationError); + expect(error).toBeInstanceOf(ProviderInvocationError); expect(error?.message).toBe('Internal Server Error'); expect(error?.statusCode).toBe(500); @@ -169,7 +169,7 @@ describe('TargetClient.invoke', () => { }); }); -describe('TargetClient.invokeBatch', () => { +describe('ProviderClient.invokeBatch', () => { it('makes POST request to /invokeBatch endpoint', async () => { const mockResponse = { responses: [ @@ -183,7 +183,7 @@ describe('TargetClient.invokeBatch', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); await client.invokeBatch([{ question: 'q1' }, { question: 'q2' }]); expect(fetchSpy).toHaveBeenCalledWith( @@ -209,7 +209,7 @@ describe('TargetClient.invokeBatch', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); const responses = await client.invokeBatch([{ question: 'q1' }, { question: 'q2' }]); expect(responses).toHaveLength(2); @@ -219,7 +219,7 @@ describe('TargetClient.invokeBatch', () => { fetchSpy.mockRestore(); }); - it('throws TargetInvocationError on batch limit exceeded', async () => { + it('throws ProviderInvocationError on batch limit exceeded', async () => { const fetchSpy = spyOn(global, 'fetch').mockResolvedValue({ ok: false, status: 429, @@ -229,16 +229,16 @@ describe('TargetClient.invokeBatch', () => { ), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); - let error: TargetInvocationError | undefined; + let error: ProviderInvocationError | undefined; try { await client.invokeBatch([{ question: 'q1' }]); } catch (e) { - error = e as TargetInvocationError; + error = e as ProviderInvocationError; } - expect(error).toBeInstanceOf(TargetInvocationError); + expect(error).toBeInstanceOf(ProviderInvocationError); expect(error?.message).toContain('Batch would exceed max calls'); expect(error?.statusCode).toBe(429); @@ -246,13 +246,13 @@ describe('TargetClient.invokeBatch', () => { }); }); -describe('TargetClient.getInfo', () => { +describe('ProviderClient.getInfo', () => { it('makes GET request to /info endpoint', async () => { const mockResponse = { - targetName: 'default-target', + providerLabel: 'default-provider', maxCalls: 50, callCount: 5, - availableTargets: ['default-target', 'alt-target'], + availableProviderLabels: ['default-provider', 'alt-provider'], }; const fetchSpy = spyOn(global, 'fetch').mockResolvedValue({ @@ -260,7 +260,7 @@ describe('TargetClient.getInfo', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'secret-token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'secret-token'); await client.getInfo(); expect(fetchSpy).toHaveBeenCalledWith( @@ -277,12 +277,12 @@ describe('TargetClient.getInfo', () => { fetchSpy.mockRestore(); }); - it('returns target info', async () => { + it('returns provider info', async () => { const mockResponse = { - targetName: 'default-target', + providerLabel: 'default-provider', maxCalls: 50, callCount: 5, - availableTargets: ['default-target', 'alt-target'], + availableProviderLabels: ['default-provider', 'alt-provider'], }; const fetchSpy = spyOn(global, 'fetch').mockResolvedValue({ @@ -290,34 +290,34 @@ describe('TargetClient.getInfo', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); const info = await client.getInfo(); - expect(info.targetName).toBe('default-target'); + expect(info.providerLabel).toBe('default-provider'); expect(info.maxCalls).toBe(50); expect(info.callCount).toBe(5); - expect(info.availableTargets).toEqual(['default-target', 'alt-target']); + expect(info.availableProviderLabels).toEqual(['default-provider', 'alt-provider']); fetchSpy.mockRestore(); }); - it('throws TargetInvocationError on non-ok response', async () => { + it('throws ProviderInvocationError on non-ok response', async () => { const fetchSpy = spyOn(global, 'fetch').mockResolvedValue({ ok: false, status: 401, text: () => Promise.resolve('{"error":"Unauthorized"}'), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); - let error: TargetInvocationError | undefined; + let error: ProviderInvocationError | undefined; try { await client.getInfo(); } catch (e) { - error = e as TargetInvocationError; + error = e as ProviderInvocationError; } - expect(error).toBeInstanceOf(TargetInvocationError); + expect(error).toBeInstanceOf(ProviderInvocationError); expect(error?.message).toBe('Unauthorized'); expect(error?.statusCode).toBe(401); @@ -325,8 +325,8 @@ describe('TargetClient.getInfo', () => { }); }); -describe('TargetClient.invoke with target override', () => { - it('includes target in request body when specified', async () => { +describe('ProviderClient.invoke with provider override', () => { + it('includes provider in request body when specified', async () => { const mockResponse = { output: [{ role: 'assistant', content: 'test response' }], rawText: 'test response', @@ -337,8 +337,8 @@ describe('TargetClient.invoke with target override', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); - await client.invoke({ question: 'test', target: 'alt-target' }); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); + await client.invoke({ question: 'test', provider: 'alt-provider' }); expect(fetchSpy).toHaveBeenCalledWith( 'http://127.0.0.1:3000/invoke', @@ -348,7 +348,7 @@ describe('TargetClient.invoke with target override', () => { systemPrompt: undefined, evalCaseId: undefined, attempt: undefined, - target: 'alt-target', + provider: 'alt-provider', }), }), ); @@ -357,8 +357,8 @@ describe('TargetClient.invoke with target override', () => { }); }); -describe('TargetClient.invokeBatch with target override', () => { - it('includes target in each request when specified', async () => { +describe('ProviderClient.invokeBatch with provider override', () => { + it('includes provider in each request when specified', async () => { const mockResponse = { responses: [ { output: [], rawText: 'response 1' }, @@ -371,8 +371,8 @@ describe('TargetClient.invokeBatch with target override', () => { json: () => Promise.resolve(mockResponse), } as Response); - const client = createTargetClientInternal('http://127.0.0.1:3000', 'token'); - await client.invokeBatch([{ question: 'q1' }, { question: 'q2', target: 'alt-target' }]); + const client = createProviderClientInternal('http://127.0.0.1:3000', 'token'); + await client.invokeBatch([{ question: 'q1' }, { question: 'q2', provider: 'alt-provider' }]); expect(fetchSpy).toHaveBeenCalledWith( 'http://127.0.0.1:3000/invokeBatch', @@ -384,14 +384,14 @@ describe('TargetClient.invokeBatch with target override', () => { systemPrompt: undefined, evalCaseId: undefined, attempt: undefined, - target: undefined, + provider: undefined, }, { question: 'q2', systemPrompt: undefined, evalCaseId: undefined, attempt: undefined, - target: 'alt-target', + provider: 'alt-provider', }, ], }), @@ -403,21 +403,21 @@ describe('TargetClient.invokeBatch with target override', () => { }); describe('Error classes', () => { - it('TargetNotAvailableError has correct name', () => { - const error = new TargetNotAvailableError('test'); - expect(error.name).toBe('TargetNotAvailableError'); + it('ProviderNotAvailableError has correct name', () => { + const error = new ProviderNotAvailableError('test'); + expect(error.name).toBe('ProviderNotAvailableError'); expect(error.message).toBe('test'); }); - it('TargetInvocationError has correct name and statusCode', () => { - const error = new TargetInvocationError('test', 429); - expect(error.name).toBe('TargetInvocationError'); + it('ProviderInvocationError has correct name and statusCode', () => { + const error = new ProviderInvocationError('test', 429); + expect(error.name).toBe('ProviderInvocationError'); expect(error.message).toBe('test'); expect(error.statusCode).toBe(429); }); - it('TargetInvocationError works without statusCode', () => { - const error = new TargetInvocationError('test'); + it('ProviderInvocationError works without statusCode', () => { + const error = new ProviderInvocationError('test'); expect(error.statusCode).toBeUndefined(); }); }); diff --git a/skills-data/agentv-eval-writer/SKILL.md b/skills-data/agentv-eval-writer/SKILL.md index 1bc7f1551..f3437516f 100644 --- a/skills-data/agentv-eval-writer/SKILL.md +++ b/skills-data/agentv-eval-writer/SKILL.md @@ -551,7 +551,7 @@ Configure via the `assert` array. Multiple graders produce a weighted average sc type: script command: [uv, run, validate.py] cwd: ./scripts # optional working directory - target: {} # optional: enable LLM target proxy (max_calls: 50) + provider: {} # optional: enable LLM provider proxy (max_calls: 50) ``` Contract: stdin JSON -> stdout JSON `{pass, score, reason, checks?: [{text, pass, score?, reason}]}` Raw stdin uses snake_case and includes: `input`, `expected_output`, `output` (final answer string), `messages`, `trace`, `trace_summary`, `token_usage`, `cost_usd`, `duration_ms`, `start_time`, `end_time`, `file_changes`, `workspace_path`, `config` @@ -825,7 +825,7 @@ import { graders } from '@agentv/sdk'; export function ragFaithfulness() { return graders.llmRubric(undefined, { name: 'rag-faithfulness', - target: 'grader-target', + provider: 'grader-provider', prompt: 'Grade whether the answer is supported by the retrieved context.', }); } diff --git a/skills-data/agentv-eval-writer/references/custom-evaluators.md b/skills-data/agentv-eval-writer/references/custom-evaluators.md index 997f90c91..7db44cafd 100644 --- a/skills-data/agentv-eval-writer/references/custom-evaluators.md +++ b/skills-data/agentv-eval-writer/references/custom-evaluators.md @@ -56,7 +56,7 @@ Promptfoo normally calls eval custom logic assertions and uses fixed assertion t ```typescript import { - createTargetClient, + createProviderClient, defineScriptGrader, defineEval, type EvalConfig, @@ -69,7 +69,7 @@ import { - `EvalConfig` - Public TypeScript eval authoring type for default-exported `*.eval.ts` and `*.eval.mts` files - `defineEval(definition)` - Optional thin helper over the same `EvalConfig` shape - `graders` - Helper catalog that returns ordinary AgentV `assert` entries -- `createTargetClient()` - Returns LLM proxy client (when `target: {}` configured) +- `createProviderClient()` - Returns LLM proxy client (when `provider: {}` configured) - `.invoke({question, systemPrompt})` - Single LLM call - `.invokeBatch(requests)` - Batch LLM calls - `definePromptTemplate(fn)` - Wraps prompt generation function @@ -88,7 +88,7 @@ import { graders, type EvalConfig } from '@agentv/sdk'; function ragFaithfulness() { return graders.llmRubric(undefined, { name: 'rag-faithfulness', - target: 'grader-target', + provider: 'grader-provider', prompt: 'Grade whether the answer is supported by the retrieved context.', }); } @@ -120,7 +120,7 @@ assert: value: source - name: rag-faithfulness type: llm-rubric - target: grader-target + provider: grader-provider prompt: Grade whether the answer is supported by the retrieved context. ``` @@ -181,6 +181,6 @@ Derived from test fields (users never author these directly): | `input` | Full resolved input array (JSON) | | `expected_output` | Full resolved expected array (JSON) | | `output` | Final answer / scored result string | -| `messages` | Transcript messages from target execution | +| `messages` | Transcript messages from provider execution | Markdown templates use `{{variable}}` syntax. TypeScript templates receive context object. diff --git a/skills-data/agentv-eval-writer/references/python-helpers.md b/skills-data/agentv-eval-writer/references/python-helpers.md index bffe46c5d..ec75c7e84 100644 --- a/skills-data/agentv-eval-writer/references/python-helpers.md +++ b/skills-data/agentv-eval-writer/references/python-helpers.md @@ -18,7 +18,7 @@ Use it when the user wants Python-based custom graders or wants to emit AgentV Y - `load_grader_input()` - `run_script_grader(handler)` - `define_script_grader(handler)` - - `TargetClient.from_env()` + - `ProviderClient.from_env()` - `agentv_py.evals` - `EvalDefinition` - `EvalTest` @@ -57,7 +57,7 @@ def rag_faithfulness(): return { "name": "rag-faithfulness", "type": "llm-rubric", - "target": "grader-target", + "provider": "grader-provider", "prompt": "Grade whether the answer is supported by the retrieved context.", }