From e6ed4b0938826d4659ff5a407bb061bb9500890c Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Wed, 8 Jul 2026 01:32:06 +0200 Subject: [PATCH] fix(core): hard-deprecate runner batch config --- apps/cli/src/commands/eval/task-bundle.ts | 1 - .../src/evaluation/loaders/config-graph.ts | 9 +- packages/core/src/evaluation/orchestrator.ts | 313 ------------------ .../src/evaluation/providers/claude-cli.ts | 2 - .../src/evaluation/providers/claude-sdk.ts | 2 - .../core/src/evaluation/providers/claude.ts | 2 - packages/core/src/evaluation/providers/cli.ts | 2 - .../src/evaluation/providers/codex-cli.ts | 2 - .../core/src/evaluation/providers/codex.ts | 2 - .../src/evaluation/providers/copilot-cli.ts | 2 - .../src/evaluation/providers/copilot-sdk.ts | 2 - .../core/src/evaluation/providers/pi-cli.ts | 2 - .../evaluation/providers/pi-coding-agent.ts | 2 - .../core/src/evaluation/providers/pi-rpc.ts | 2 - .../core/src/evaluation/providers/replay.ts | 2 - .../providers/sdk-child-provider.ts | 2 - .../core/src/evaluation/providers/targets.ts | 31 +- .../core/src/evaluation/providers/types.ts | 11 - .../evaluation/providers/vscode-provider.ts | 2 - .../validation/targets-validator.ts | 41 ++- .../core/test/evaluation/orchestrator.test.ts | 78 ++--- .../providers/pi-coding-agent.test.ts | 1 - .../test/evaluation/providers/targets.test.ts | 38 +-- .../validation/targets-validator.test.ts | 36 +- 24 files changed, 132 insertions(+), 455 deletions(-) diff --git a/apps/cli/src/commands/eval/task-bundle.ts b/apps/cli/src/commands/eval/task-bundle.ts index ad61611a4..eb034c220 100644 --- a/apps/cli/src/commands/eval/task-bundle.ts +++ b/apps/cli/src/commands/eval/task-bundle.ts @@ -64,7 +64,6 @@ const AUTHORING_TOP_LEVEL_TARGET_FIELDS = new Set([ 'grader_target', 'max_budget_usd', 'workers', - 'batch_requests', 'subagent_mode_allowed', 'max_retries', 'retry_initial_delay_ms', diff --git a/packages/core/src/evaluation/loaders/config-graph.ts b/packages/core/src/evaluation/loaders/config-graph.ts index 453277d2b..53d22041b 100644 --- a/packages/core/src/evaluation/loaders/config-graph.ts +++ b/packages/core/src/evaluation/loaders/config-graph.ts @@ -32,6 +32,8 @@ const SUPPORTED_FILE_REF_FIELDS = new Set([ ]); const RUNTIME_MODES = new Set(['host', 'profile', 'sandbox']); const AMBIGUOUS_PROVIDER_ALIASES = new Set(['codex', 'claude', 'copilot', 'pi']); +const REMOVED_RUNNER_BATCHING_MESSAGE = + 'runner-level batching was removed; providers should implement internal queueing/batching behind per-request invocation.'; const REMOVED_TARGET_FIELDS = new Map([ ['label', "target identity uses 'id'; remove 'label'."], ['name', "target identity uses 'id'; remove 'name'."], @@ -44,10 +46,9 @@ const REMOVED_TARGET_FIELDS = new Map([ ['install', 'install/setup steps belong in environment.setup, not under targets.'], ['grader_target', "grader selection belongs in 'defaults.grader' or evaluator config."], ['workers', "target-level 'workers' is not general run policy; use 'execution.max_concurrency'."], - [ - 'batch_requests', - "target-level 'batch_requests' is not part of the base config contract; keep provider batching under provider-specific config only when needed.", - ], + ['batch_requests', REMOVED_RUNNER_BATCHING_MESSAGE], + ['provider_batching', REMOVED_RUNNER_BATCHING_MESSAGE], + ['providerBatching', REMOVED_RUNNER_BATCHING_MESSAGE], [ 'subagent_mode_allowed', "target-level 'subagent_mode_allowed' is not part of the base config contract.", diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index 260f3495a..b435527d4 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -949,48 +949,7 @@ export async function runEvaluation( // Discover custom providers from .agentv/providers/ directory const providerRegistry = createBuiltinProviderRegistry(); await discoverProviders(providerRegistry, discoveryBaseDir); - const primaryProvider = getOrCreateProvider(target); - let providerSupportsBatch = - target.providerBatching === true && - primaryProvider.supportsBatch === true && - typeof primaryProvider.invokeBatch === 'function'; - let batchingDisabledByRuntimePolicy = false; - - // Disable batch mode when repeat count > 1 (batching is incompatible with separate attempts). - if (trials && trials.count > 1 && providerSupportsBatch) { - console.warn( - 'Warning: Batch mode is disabled when evaluate_options.repeat.count > 1. Using per-attempt dispatch.', - ); - providerSupportsBatch = false; - batchingDisabledByRuntimePolicy = true; - } - - const requiresWorkspaceDispatch = - workspacePath !== undefined || - legacyWorkspacePath !== undefined || - target.environment !== undefined || - filteredEvalCases.some( - (evalCase) => evalCase.workspace !== undefined || evalCase.environment !== undefined, - ); - if (providerSupportsBatch && requiresWorkspaceDispatch) { - if (verbose) { - console.warn('Warning: Batch mode is disabled for workspace- or environment-enabled evals.'); - } - providerSupportsBatch = false; - batchingDisabledByRuntimePolicy = true; - } - - if ( - target.providerBatching && - !providerSupportsBatch && - verbose && - !batchingDisabledByRuntimePolicy - ) { - console.warn( - `Request batching requested for target '${target.name}', but provider does not advertise batch support. Using per-attempt dispatch.`, - ); - } // Notify about total test count before starting if (onProgress && filteredEvalCases.length > 0) { @@ -1004,37 +963,6 @@ export async function runEvaluation( } } - if (providerSupportsBatch) { - try { - return await runBatchEvaluation({ - evalCases: filteredEvalCases, - provider: primaryProvider, - target, - evaluatorRegistry, - typeRegistry, - nowFn: now ?? (() => new Date()), - onProgress, - onResult, - verbose, - resolveGraderProvider, - agentTimeoutMs, - targetResolver, - availableTargets, - threshold: scoreThreshold, - replayRecording, - evalFilePath, - repoRoot: repoRootPath, - }); - } catch (error) { - if (verbose) { - const message = error instanceof Error ? error.message : String(error); - console.warn( - `Request batch execution failed, falling back to per-attempt dispatch: ${message}`, - ); - } - } - } - const resolvedRetainOnSuccess = retainOnSuccess ?? (keepWorkspaces ? 'keep' : 'cleanup'); const resolvedRetainOnFailure = retainOnFailure ?? (cleanupWorkspaces ? 'cleanup' : 'keep'); const workers = options.maxConcurrency ?? target.workers ?? 1; @@ -1630,247 +1558,6 @@ export async function runEvaluation( } } -async function runBatchEvaluation(options: { - readonly evalCases: readonly EvalTest[]; - readonly provider: Provider; - readonly target: ResolvedProviderBackend; - readonly evaluatorRegistry: Partial> & { - readonly 'llm-grader': Grader; - }; - readonly typeRegistry: import('./registry/grader-registry.js').GraderRegistry; - readonly nowFn: () => Date; - readonly onProgress?: (event: ProgressEvent) => MaybePromise; - readonly onResult?: (result: EvaluationResult) => MaybePromise; - readonly verbose?: boolean; - readonly resolveGraderProvider: ( - target: ResolvedProviderBackend, - ) => Promise; - readonly agentTimeoutMs?: number; - readonly targetResolver?: (name: string) => Provider | undefined; - readonly availableTargets?: readonly string[]; - readonly threshold?: number; - readonly replayRecording?: ReplayRecordingOptions; - readonly evalFilePath: string; - readonly repoRoot: string; -}): Promise { - const { - evalCases, - provider, - target, - evaluatorRegistry, - typeRegistry, - nowFn, - onProgress, - onResult, - verbose, - resolveGraderProvider, - agentTimeoutMs, - targetResolver, - availableTargets, - threshold: batchThreshold, - replayRecording, - evalFilePath, - repoRoot, - } = options; - - // Prepare prompt inputs up front so we can reuse them for grading. - const promptInputsList: PromptInputs[] = []; - const formattingMode = usesFileReferencePrompt(provider) ? 'agent' : 'lm'; - - for (const evalCase of evalCases) { - const promptInputs = await buildPromptInputs(evalCase, formattingMode); - promptInputsList.push(promptInputs); - } - - const batchRequests: ProviderRequest[] = evalCases.map((evalCase, index) => { - const promptInputs = promptInputsList[index]; - return { - question: promptInputs.question, - systemPrompt: promptInputs.systemMessage, - inputFiles: evalCase.file_paths, - evalCaseId: evalCase.id, - suite: evalCase.suite, - evalFilePath, - }; - }); - - const batchResponse = await provider.invokeBatch?.(batchRequests); - if (!Array.isArray(batchResponse)) { - throw new Error('Request batching failed: invokeBatch did not return an array'); - } - if (batchResponse.length !== evalCases.length) { - throw new Error( - `Request batching failed: expected ${evalCases.length} responses, received ${batchResponse.length}`, - ); - } - - if (onProgress) { - const startedAt = Date.now(); - for (let i = 0; i < evalCases.length; i++) { - await onProgress({ - workerId: 1, - testId: evalCases[i].id, - status: 'running', - startedAt, - }); - } - } - - const results: EvaluationResult[] = []; - for (let i = 0; i < evalCases.length; i++) { - const evalCase = evalCases[i]; - const promptInputs = promptInputsList[i]; - const providerResponse = batchResponse[i]; - await maybeRecordReplayFixture({ - replayRecording, - evalCase, - evalFilePath, - repoRoot, - target, - attempt: 0, - response: providerResponse, - nowFn, - }); - - // Extract output from batch response - const output = providerResponse.output; - const hasExecutionMetrics = - providerResponse.tokenUsage !== undefined || - providerResponse.costUsd !== undefined || - providerResponse.durationMs !== undefined; - - const computed = output - ? computeTraceSummary(output) - : hasExecutionMetrics - ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } - : undefined; - const merged = computed - ? mergeExecutionMetrics(computed, { - tokenUsage: providerResponse.tokenUsage, - costUsd: providerResponse.costUsd, - durationMs: providerResponse.durationMs, - startTime: providerResponse.startTime, - endTime: providerResponse.endTime, - }) - : undefined; - const trace = merged?.trace; - const costUsd = merged?.costUsd; - const durationMs = merged?.durationMs; - const tokenUsage = merged?.tokenUsage; - const startTime = merged?.startTime; - const endTime = merged?.endTime; - const rawProviderLogPath = extractProviderRawLogPath(providerResponse); - const providerResponseMetadata = buildProviderResponseTransformMetadata(providerResponse); - - // Extract candidate from last assistant message in output - const candidate = extractLastAssistantContent(output); - - const providerError = extractProviderError(providerResponse); - - let result: EvaluationResult; - try { - result = await evaluateCandidate({ - evalCase, - candidate, - target, - provider, - evaluators: evaluatorRegistry, - typeRegistry, - promptInputs, - nowFn, - attempt: 0, - sampleIndex: 0, - graderProvider: await resolveGraderProvider(target), - agentTimeoutMs, - output, - trace, - costUsd, - durationMs, - tokenUsage, - providerResponseMetadata, - startTime, - endTime, - rawProviderLogPath, - targetResolver, - availableTargets, - verbose, - threshold: evalCase.threshold ?? batchThreshold, - }); - - if (providerError) { - const failureReasonCode = targetExecutionFailureReasonCode(providerResponse); - result = { - ...result, - targetExecution: providerResponse.targetExecution, - trace: appendErrorEventToTrace(result.trace, providerError, { - failure_stage: 'agent', - failure_reason_code: failureReasonCode, - }), - error: providerError, - executionStatus: 'execution_error' as const, - failureStage: 'agent' as const, - failureReasonCode: failureReasonCode, - executionError: { message: providerError, stage: 'agent' as const }, - }; - } else if (providerResponse.targetExecution) { - result = { ...result, targetExecution: providerResponse.targetExecution }; - } - } catch (error) { - const errorResult = buildErrorResult( - evalCase, - target.name, - nowFn(), - error, - promptInputs, - provider, - 'evaluator', - 'evaluator_error', - verbose, - { sampleIndex: 0, retryIndex: 0 }, - ); - results.push(errorResult); - if (onResult) { - await onResult(errorResult); - } - if (onProgress) { - await onProgress({ - workerId: 1, - testId: evalCase.id, - status: 'failed', - completedAt: Date.now(), - error: error instanceof Error ? error.message : String(error), - score: errorResult.score, - executionStatus: errorResult.executionStatus, - evalRunDurationMs: errorResult.evalRun?.durationMs, - }); - } - continue; - } - - results.push(result); - if (onResult) { - await onResult(result); - } - - if (onProgress) { - await onProgress({ - workerId: 1, - testId: evalCase.id, - status: result.error ? 'failed' : 'completed', - startedAt: 0, - completedAt: Date.now(), - error: result.error, - score: result.score, - executionStatus: result.executionStatus, - durationMs: result.durationMs, - evalRunDurationMs: result.evalRun?.durationMs, - }); - } - } - - return results; -} - async function maybeRecordReplayFixture(options: { readonly replayRecording?: ReplayRecordingOptions; readonly evalCase: EvalTest; diff --git a/packages/core/src/evaluation/providers/claude-cli.ts b/packages/core/src/evaluation/providers/claude-cli.ts index 626f7c2df..3237a37ac 100644 --- a/packages/core/src/evaluation/providers/claude-cli.ts +++ b/packages/core/src/evaluation/providers/claude-cli.ts @@ -32,8 +32,6 @@ export class ClaudeCliProvider implements Provider { readonly id: string; readonly kind = 'claude-cli' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: ClaudeResolvedConfig; constructor(targetName: string, config: ClaudeResolvedConfig) { diff --git a/packages/core/src/evaluation/providers/claude-sdk.ts b/packages/core/src/evaluation/providers/claude-sdk.ts index a2ee7df26..f98f40162 100644 --- a/packages/core/src/evaluation/providers/claude-sdk.ts +++ b/packages/core/src/evaluation/providers/claude-sdk.ts @@ -51,8 +51,6 @@ export class ClaudeSdkProvider implements Provider { readonly id: string; readonly kind = 'claude-sdk' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: ClaudeResolvedConfig; constructor(targetName: string, config: ClaudeResolvedConfig) { diff --git a/packages/core/src/evaluation/providers/claude.ts b/packages/core/src/evaluation/providers/claude.ts index bc904593d..647fb218e 100644 --- a/packages/core/src/evaluation/providers/claude.ts +++ b/packages/core/src/evaluation/providers/claude.ts @@ -50,8 +50,6 @@ export class ClaudeProvider implements Provider { readonly id: string; readonly kind = 'claude-sdk' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: ClaudeResolvedConfig; constructor(targetName: string, config: ClaudeResolvedConfig) { diff --git a/packages/core/src/evaluation/providers/cli.ts b/packages/core/src/evaluation/providers/cli.ts index d807dfc57..3249b226c 100644 --- a/packages/core/src/evaluation/providers/cli.ts +++ b/packages/core/src/evaluation/providers/cli.ts @@ -385,8 +385,6 @@ export class CliProvider implements Provider { readonly id: string; readonly kind = 'cli'; readonly targetName: string; - readonly supportsBatch = true; - private readonly config: CliResolvedConfig; private readonly runCommand: CommandRunner; private readonly runSandboxCommand: SandboxCommandRunner; diff --git a/packages/core/src/evaluation/providers/codex-cli.ts b/packages/core/src/evaluation/providers/codex-cli.ts index 0682c55e3..99162b51d 100644 --- a/packages/core/src/evaluation/providers/codex-cli.ts +++ b/packages/core/src/evaluation/providers/codex-cli.ts @@ -68,8 +68,6 @@ export class CodexCliProvider implements Provider { readonly id: string; readonly kind: 'codex-cli' | 'codex-app-server'; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: CodexResolvedConfig; private readonly runCodex: CodexRunner; private environmentCheck?: Promise; diff --git a/packages/core/src/evaluation/providers/codex.ts b/packages/core/src/evaluation/providers/codex.ts index 68dc677da..9ac4167fa 100644 --- a/packages/core/src/evaluation/providers/codex.ts +++ b/packages/core/src/evaluation/providers/codex.ts @@ -50,8 +50,6 @@ export class CodexProvider implements Provider { readonly id: string; readonly kind = 'codex-sdk' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: CodexResolvedConfig; constructor(targetName: string, config: CodexResolvedConfig) { diff --git a/packages/core/src/evaluation/providers/copilot-cli.ts b/packages/core/src/evaluation/providers/copilot-cli.ts index 495527bbf..2cc8494fe 100644 --- a/packages/core/src/evaluation/providers/copilot-cli.ts +++ b/packages/core/src/evaluation/providers/copilot-cli.ts @@ -83,8 +83,6 @@ export class CopilotCliProvider implements Provider { readonly id: string; readonly kind = 'copilot-cli' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: CopilotCliResolvedConfig; private readonly runPromptMode: CopilotCliPromptRunner; private readonly spawnAcpProcess: CopilotCliSpawn; diff --git a/packages/core/src/evaluation/providers/copilot-sdk.ts b/packages/core/src/evaluation/providers/copilot-sdk.ts index 74e5ffa84..1f61061cc 100644 --- a/packages/core/src/evaluation/providers/copilot-sdk.ts +++ b/packages/core/src/evaluation/providers/copilot-sdk.ts @@ -108,8 +108,6 @@ export class CopilotSdkProvider implements Provider { readonly id: string; readonly kind = 'copilot-sdk' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: CopilotSdkResolvedConfig; // biome-ignore lint/suspicious/noExplicitAny: SDK client type is dynamically loaded private client: any = null; diff --git a/packages/core/src/evaluation/providers/pi-cli.ts b/packages/core/src/evaluation/providers/pi-cli.ts index 0a7af6d72..9fed355d7 100644 --- a/packages/core/src/evaluation/providers/pi-cli.ts +++ b/packages/core/src/evaluation/providers/pi-cli.ts @@ -53,8 +53,6 @@ export class PiCliProvider implements Provider { readonly id: string; readonly kind = 'pi-cli' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: PiCliResolvedConfig; private readonly runPi: PiProcessRunner; diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts index 2de4ae3dc..6cbaf73dc 100644 --- a/packages/core/src/evaluation/providers/pi-coding-agent.ts +++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts @@ -267,8 +267,6 @@ export class PiCodingAgentProvider implements Provider { readonly id: string; readonly kind = 'pi-coding-agent' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: PiCodingAgentResolvedConfig; constructor(targetName: string, config: PiCodingAgentResolvedConfig) { diff --git a/packages/core/src/evaluation/providers/pi-rpc.ts b/packages/core/src/evaluation/providers/pi-rpc.ts index 3e60bb3e6..145cf813e 100644 --- a/packages/core/src/evaluation/providers/pi-rpc.ts +++ b/packages/core/src/evaluation/providers/pi-rpc.ts @@ -33,8 +33,6 @@ export class PiRpcProvider implements Provider { readonly id: string; readonly kind = 'pi-rpc' as const; readonly targetName: string; - readonly supportsBatch = false; - private readonly config: PiRpcResolvedConfig; private readonly runPi: PiProcessRunner; diff --git a/packages/core/src/evaluation/providers/replay.ts b/packages/core/src/evaluation/providers/replay.ts index 37c3312a2..d2057e01b 100644 --- a/packages/core/src/evaluation/providers/replay.ts +++ b/packages/core/src/evaluation/providers/replay.ts @@ -30,8 +30,6 @@ export class ReplayProvider implements Provider { readonly id: string; readonly kind = 'replay' as const; readonly targetName: string; - readonly supportsBatch = true; - private readonly config: ReplayResolvedConfig; constructor(targetName: string, config: ReplayResolvedConfig) { diff --git a/packages/core/src/evaluation/providers/sdk-child-provider.ts b/packages/core/src/evaluation/providers/sdk-child-provider.ts index 3a2537998..f115ae6f7 100644 --- a/packages/core/src/evaluation/providers/sdk-child-provider.ts +++ b/packages/core/src/evaluation/providers/sdk-child-provider.ts @@ -47,8 +47,6 @@ export interface SdkChildProviderOptions { export class SdkChildProvider implements Provider { readonly id: string; readonly targetName: string; - readonly supportsBatch = false; - readonly kind: SdkChildProviderKind; private readonly config: unknown; diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 94ccd178f..6c54d5dc3 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -118,7 +118,6 @@ export const CliTargetInputSchema = z // Common target fields grader_target: z.string().optional(), workers: z.number().int().min(1).optional(), - batch_requests: z.boolean().optional(), }) .passthrough(); @@ -615,7 +614,6 @@ export interface TargetDeprecationWarning { } const DEPRECATED_TARGET_CAMEL_CASE_FIELDS = new Map([ - ['providerBatching', 'batch_requests'], ['subagentModeAllowed', 'subagent_mode_allowed'], ['fallbackTargets', 'fallback_targets'], ['resourceName', 'endpoint'], @@ -1095,7 +1093,26 @@ function assertNoRemovedTargetFields(definition: ProviderDefinition): void { } if (Object.prototype.hasOwnProperty.call(rawDefinition, 'provider_batching')) { throw new Error( - `target "${definition.name}".provider_batching: field 'provider_batching' has been removed. Use 'batch_requests' instead.`, + buildRemovedRunnerBatchingError( + `target "${definition.name}".provider_batching`, + 'provider_batching', + ), + ); + } + if (Object.prototype.hasOwnProperty.call(rawDefinition, 'batch_requests')) { + throw new Error( + buildRemovedRunnerBatchingError( + `target "${definition.name}".batch_requests`, + 'batch_requests', + ), + ); + } + if (Object.prototype.hasOwnProperty.call(rawDefinition, 'providerBatching')) { + throw new Error( + buildRemovedRunnerBatchingError( + `target "${definition.name}".providerBatching`, + 'providerBatching', + ), ); } if (Object.prototype.hasOwnProperty.call(rawDefinition, 'log_format')) { @@ -1110,6 +1127,10 @@ function assertNoRemovedTargetFields(definition: ProviderDefinition): void { } } +function buildRemovedRunnerBatchingError(location: string, field: string): string { + return `${location}: field '${field}' has been removed. Runner-level batching was removed; providers should implement internal queueing/batching behind per-request invocation.`; +} + export function findDeprecatedCamelCaseTargetWarnings( target: unknown, location: string, @@ -1153,7 +1174,6 @@ interface ResolvedProviderBackendBase { readonly environment?: EnvironmentRecipe; readonly graderTarget?: string; readonly workers?: number; - readonly providerBatching?: boolean; /** * Whether this target can be executed via executor subagents in subagent mode. * Defaults to `true` for all non-CLI providers. Set `false` in targets.yaml @@ -1243,7 +1263,6 @@ export type ResolvedProviderBackend = export const COMMON_PROVIDER_SETTINGS = [ 'runtime', 'environment', - 'batch_requests', 'subagent_mode_allowed', 'fallback_targets', ] as const; @@ -1457,7 +1476,6 @@ export function resolveProviderDefinition( `Target "${parsed.name}" uses removed provider 'copilot-log'. Import Copilot events with 'agentv import copilot' and replay the normalized transcript with provider: replay and transcripts: .`, ); } - const providerBatching = resolveOptionalBoolean(parsed.batch_requests); const subagentModeAllowed = resolveOptionalBoolean(parsed.subagent_mode_allowed); // Shared base fields for all resolved targets @@ -1469,7 +1487,6 @@ export function resolveProviderDefinition( environment: isResolvedEnvironmentRecipe(parsed.environment) ? parsed.environment : undefined, graderTarget: parsed.grader_target, workers: parsed.workers, - providerBatching, subagentModeAllowed, ...(fallbackTargets ? { fallbackTargets } : {}), } as const; diff --git a/packages/core/src/evaluation/providers/types.ts b/packages/core/src/evaluation/providers/types.ts index 4ce170068..5613aab9a 100644 --- a/packages/core/src/evaluation/providers/types.ts +++ b/packages/core/src/evaluation/providers/types.ts @@ -438,15 +438,6 @@ export interface Provider { readonly targetName: string; readonly environment?: EnvironmentRecipe; invoke(request: ProviderRequest): Promise; - /** - * Optional capability marker for provider-managed batching (single session handling multiple requests). - */ - readonly supportsBatch?: boolean; - /** - * Optional batch invocation hook. When defined alongside supportsBatch=true, - * the orchestrator may send multiple requests in a single provider session. - */ - invokeBatch?(requests: readonly ProviderRequest[]): Promise; } export type EnvLookup = Readonly>; @@ -475,8 +466,6 @@ export interface ProviderDefinition { readonly use_target?: string | unknown | undefined; readonly grader_target?: string | undefined; readonly workers?: number | undefined; - // Request batching - readonly batch_requests?: boolean | undefined; readonly subagent_mode_allowed?: boolean | undefined; // Azure fields readonly endpoint?: string | unknown | undefined; diff --git a/packages/core/src/evaluation/providers/vscode-provider.ts b/packages/core/src/evaluation/providers/vscode-provider.ts index a7cd85ec2..71b8577aa 100644 --- a/packages/core/src/evaluation/providers/vscode-provider.ts +++ b/packages/core/src/evaluation/providers/vscode-provider.ts @@ -20,8 +20,6 @@ export class VSCodeProvider implements Provider { readonly id: string; readonly kind: 'vscode' | 'vscode-insiders'; readonly targetName: string; - readonly supportsBatch = true; - private readonly config: VSCodeResolvedConfig; private environmentCheck?: Promise; diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index f28f60a94..1f134ee32 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -15,6 +15,11 @@ type JsonValue = string | number | boolean | null | JsonObject | JsonArray; type JsonObject = { readonly [key: string]: JsonValue }; type JsonArray = readonly JsonValue[]; const LEGACY_ENV_PATTERN = /\$\{\{\s*([A-Z_][A-Z0-9_]*)\s*\}\}/g; +const REMOVED_RUNNER_BATCHING_FIELDS = new Set([ + 'batch_requests', + 'provider_batching', + 'providerBatching', +]); function isObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); @@ -370,6 +375,9 @@ function validateUnknownSettings( const removedForProvider = removedPerProvider[provider]; for (const key of Object.keys(target)) { + if (REMOVED_RUNNER_BATCHING_FIELDS.has(key)) { + continue; + } if (removedFields.has(key)) { errors.push({ severity: 'warning', @@ -389,15 +397,6 @@ function validateUnknownSettings( }); continue; } - if (key === 'provider_batching') { - errors.push({ - severity: 'error', - filePath: absolutePath, - location: `${location}.${key}`, - message: "The 'provider_batching' field has been removed. Use 'batch_requests' instead.", - }); - continue; - } if (removedForProvider?.has(key)) { errors.push({ severity: 'error', @@ -418,6 +417,29 @@ function validateUnknownSettings( } } +function buildRemovedRunnerBatchingMessage(field: string): string { + return `The '${field}' field has been removed. Runner-level batching was removed; providers should implement internal queueing/batching behind per-request invocation.`; +} + +function validateRemovedRunnerBatchingFields( + target: JsonObject, + absolutePath: string, + location: string, + errors: ValidationError[], +): void { + for (const key of Object.keys(target)) { + if (!REMOVED_RUNNER_BATCHING_FIELDS.has(key)) { + continue; + } + errors.push({ + severity: 'error', + filePath: absolutePath, + location: `${location}.${key}`, + message: buildRemovedRunnerBatchingMessage(key), + }); + } +} + /** * Validate a targets file (agentv-targets-v2.1 schema). */ @@ -730,6 +752,7 @@ export async function validateTargetsFile(filePath: string): Promise { - class BatchProvider implements Provider { - readonly id = 'batch:mock'; + it('reports failed progress status for per-case provider errors', async () => { + class ErroringProvider implements Provider { + readonly id = 'mock:progress'; readonly kind = 'mock' as const; readonly targetName = 'mock'; - readonly supportsBatch = true; + private calls = 0; async invoke(): Promise { - throw new Error('invoke not used'); - } - - async invokeBatch( - requests: readonly ProviderRequest[], - ): Promise { - return requests.map((request) => { - if (request.evalCaseId === 'case-2') { - return { - output: [{ role: 'assistant', content: "Error: Batch output missing id 'case-2'" }], - raw: { error: "Batch output missing id 'case-2'" }, - }; - } - + this.calls += 1; + if (this.calls === 2) { return { - output: [{ role: 'assistant', content: 'OK' }], + output: [{ role: 'assistant', content: 'Error: Invalid request' }], + raw: { error: 'Invalid request' }, }; - }); + } + + return { + output: [{ role: 'assistant', content: 'OK' }], + }; } } @@ -891,10 +884,9 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, repoRoot: 'in-memory', target: { ...baseTarget, - providerBatching: true, workers: 1, }, - providerFactory: () => new BatchProvider(), + providerFactory: () => new ErroringProvider(), evaluators: evaluatorRegistry, evalCases, onProgress: async (event) => { @@ -908,15 +900,14 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, expect(events.find((e) => e.testId === 'case-1')?.status).toBe('completed'); const case2 = events.find((e) => e.testId === 'case-2'); expect(case2?.status).toBe('failed'); - expect(case2?.error).toBe("Batch output missing id 'case-2'"); + expect(case2?.error).toBe('Invalid request'); }); - it('disables request batching when cases require workspace setup', async () => { - class BatchCapableProvider implements Provider { - readonly id = 'batch:workspace'; + it('runs workspace setup cases through per-case provider invocation without batch warnings', async () => { + class InternalBatchHelperProvider implements Provider { + readonly id = 'mock:workspace'; readonly kind = 'mock' as const; readonly targetName = 'workspace'; - readonly supportsBatch = true; batchCalls = 0; invokeRequests: ProviderRequest[] = []; @@ -935,7 +926,8 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, const templateDir = mkdtempSync(path.join(tmpdir(), 'agentv-batch-workspace-')); writeFileSync(path.join(templateDir, 'README.md'), 'workspace\n', 'utf8'); - const provider = new BatchCapableProvider(); + const provider = new InternalBatchHelperProvider(); + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); try { const results = await runEvaluation({ @@ -943,7 +935,6 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, repoRoot: 'in-memory', target: { ...baseTarget, - providerBatching: true, workers: 1, }, providerFactory: () => provider, @@ -962,18 +953,19 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, expect(provider.batchCalls).toBe(0); expect(provider.invokeRequests).toHaveLength(1); expect(provider.invokeRequests[0]?.cwd).toBeDefined(); + expect(warnSpy.mock.calls.flat().join('\n')).not.toContain('Batch mode'); } finally { + warnSpy.mockRestore(); const { rm } = await import('node:fs/promises'); await rm(templateDir, { recursive: true, force: true }); } }); - it('disables request batching when cases require environment setup', async () => { - class BatchCapableProvider implements Provider { - readonly id = 'batch:environment'; + it('runs environment setup cases through per-case provider invocation without batch warnings', async () => { + class InternalBatchHelperProvider implements Provider { + readonly id = 'mock:environment'; readonly kind = 'mock' as const; readonly targetName = 'environment'; - readonly supportsBatch = true; batchCalls = 0; invokeRequests: ProviderRequest[] = []; @@ -997,7 +989,8 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, "import { writeFileSync } from 'node:fs'; writeFileSync(process.env.AGENTV_ENVIRONMENT_WORKDIR + '/ready.txt', 'ok');\n", 'utf8', ); - const provider = new BatchCapableProvider(); + const provider = new InternalBatchHelperProvider(); + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); try { const results = await runEvaluation({ @@ -1005,7 +998,6 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, repoRoot: 'in-memory', target: { ...baseTarget, - providerBatching: true, workers: 1, }, providerFactory: () => provider, @@ -1028,7 +1020,9 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, expect(provider.invokeRequests).toHaveLength(1); expect(provider.invokeRequests[0]?.cwd).toBe(workdir); expect(existsSync(path.join(workdir, 'ready.txt'))).toBe(true); + expect(warnSpy.mock.calls.flat().join('\n')).not.toContain('Batch mode'); } finally { + warnSpy.mockRestore(); const { rm } = await import('node:fs/promises'); await rm(sourceDir, { recursive: true, force: true }); } @@ -1198,12 +1192,11 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, } }); - it('disables request batching when repeat attempts are configured', async () => { - class BatchCapableProvider implements Provider { - readonly id = 'batch:repeat'; + it('runs repeat attempts through per-case provider invocation without batch warnings', async () => { + class InternalBatchHelperProvider implements Provider { + readonly id = 'mock:repeat'; readonly kind = 'mock' as const; readonly targetName = 'repeat'; - readonly supportsBatch = true; invokeCalls = 0; batchCalls = 0; @@ -1220,7 +1213,7 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, } } - const provider = new BatchCapableProvider(); + const provider = new InternalBatchHelperProvider(); const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); try { const results = await runEvaluation({ @@ -1228,7 +1221,6 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, repoRoot: 'in-memory', target: { ...baseTarget, - providerBatching: true, workers: 1, }, providerFactory: () => provider, @@ -1240,9 +1232,7 @@ console.log('spreadsheet: revenue,total\\nQ1,42');`, expect(results).toHaveLength(1); expect(provider.batchCalls).toBe(0); expect(provider.invokeCalls).toBe(2); - expect(warnSpy).toHaveBeenCalledWith( - 'Warning: Batch mode is disabled when evaluate_options.repeat.count > 1. Using per-attempt dispatch.', - ); + expect(warnSpy.mock.calls.flat().join('\n')).not.toContain('Batch mode'); } finally { warnSpy.mockRestore(); } diff --git a/packages/core/test/evaluation/providers/pi-coding-agent.test.ts b/packages/core/test/evaluation/providers/pi-coding-agent.test.ts index 3329eab94..df96a0a57 100644 --- a/packages/core/test/evaluation/providers/pi-coding-agent.test.ts +++ b/packages/core/test/evaluation/providers/pi-coding-agent.test.ts @@ -11,7 +11,6 @@ describe('PiCodingAgentProvider', () => { expect(provider.kind).toBe('pi-coding-agent'); expect(provider.id).toBe('pi-coding-agent:test-target'); expect(provider.targetName).toBe('test-target'); - expect(provider.supportsBatch).toBe(false); }); it('rejects when signal is already aborted', async () => { diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 6a3c4c047..89aea62a5 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -847,31 +847,19 @@ describe('resolveProviderDefinition', () => { ).toThrow(/GOOGLE_API_KEY/i); }); - it('honors batch_requests flag in settings', () => { - const target = resolveProviderDefinition( - { - name: 'batched', - provider: 'mock', - batch_requests: true, - }, - {}, - ); - - expect(target.kind).toBe('mock'); - expect(target.providerBatching).toBe(true); - }); - - it('rejects removed provider_batching flag in settings', () => { - expect(() => - resolveProviderDefinition( - { - name: 'batched', - provider: 'mock', - provider_batching: true, - }, - {}, - ), - ).toThrow(/provider_batching.*batch_requests/); + it('rejects removed runner-level batching flags in settings', () => { + for (const field of ['batch_requests', 'provider_batching', 'providerBatching'] as const) { + expect(() => + resolveProviderDefinition( + { + name: 'batched', + provider: 'mock', + [field]: true, + }, + {}, + ), + ).toThrow(new RegExp(`${field}.*Runner-level batching was removed.*per-request invocation`)); + } }); it('resolves cli settings including cwd and timeout', () => { diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts index a7575674b..243ab0500 100644 --- a/packages/core/test/evaluation/validation/targets-validator.test.ts +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -94,7 +94,6 @@ describe('validateTargetsFile', () => { api_format: responses grader_target: grader fallback_targets: [backup-agent] - batch_requests: true - id: openai label: grader config: @@ -113,28 +112,41 @@ describe('validateTargetsFile', () => { expect(result.errors.filter((error) => error.severity === 'warning')).toEqual([]); }); - it('rejects removed provider_batching in favor of batch_requests', async () => { - const filePath = path.join(tempDir, 'removed-provider-batching.yaml'); + it('rejects removed runner-level batching fields', async () => { + const filePath = path.join(tempDir, 'removed-runner-batching.yaml'); await writeFile( filePath, `providers: - id: mock - label: batch-cli + label: batch-requests + batch_requests: true + - id: mock + label: provider-batching provider_batching: true + - id: mock + label: camel-batching + providerBatching: true `, ); const result = await validateTargetsFile(filePath); expect(result.valid).toBe(false); - expect( - result.errors.some( - (error) => - error.severity === 'error' && - error.location === 'providers[0].provider_batching' && - error.message.includes("Use 'batch_requests' instead"), - ), - ).toBe(true); + for (const [index, field] of [ + 'batch_requests', + 'provider_batching', + 'providerBatching', + ].entries()) { + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === `providers[${index}].${field}` && + error.message.includes('Runner-level batching was removed') && + error.message.includes('internal queueing/batching behind per-request invocation'), + ), + ).toBe(true); + } }); it('rejects authored provider name in favor of id', async () => {