diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index c52e39876..c32459c03 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -129,29 +129,6 @@ export const evalRunCommand = command({ description: 'Preserve per-test workspaces after eval (default: keep on failure, cleanup on success)', }), - otelFile: option({ - type: optional(string), - long: 'otel-file', - description: 'Write OTLP JSON trace to file (importable by OTel backends)', - }), - exportOtel: flag({ - long: 'export-otel', - description: 'Export evaluation traces via OTLP/HTTP to configured endpoint', - }), - otelBackend: option({ - type: optional(string), - long: 'otel-backend', - description: 'Use an OTel backend resolver (langfuse, braintrust, confident, or local)', - }), - otelCaptureContent: flag({ - long: 'otel-capture-content', - description: 'Include message content in exported OTel spans (privacy: disabled by default)', - }), - otelGroupTurns: flag({ - long: 'otel-group-turns', - description: - 'Group messages into turn spans for multi-turn evaluations (requires --export-otel)', - }), retryErrors: option({ type: optional(string), long: 'retry-errors', @@ -269,11 +246,6 @@ export const evalRunCommand = command({ workspacePath: args.workspacePath, keepWorkspaces: args.keepWorkspaces, trace: false, - otelFile: args.otelFile, - exportOtel: args.exportOtel, - otelBackend: args.otelBackend, - otelCaptureContent: args.otelCaptureContent, - otelGroupTurns: args.otelGroupTurns, retryErrors: args.retryErrors, resume: args.resume, rerunFailed: args.rerunFailed, diff --git a/apps/cli/src/commands/eval/otel-backends.ts b/apps/cli/src/commands/eval/otel-backends.ts deleted file mode 100644 index 58c9df718..000000000 --- a/apps/cli/src/commands/eval/otel-backends.ts +++ /dev/null @@ -1,167 +0,0 @@ -/** - * OTel backend resolver loading for the eval CLI. - * - * Core owns generic OTLP export only. This module keeps CLI ergonomics for - * `--otel-backend ` by checking project-local resolver files first, then - * falling back to the small set of resolver names already exposed by the CLI. - * - * To add a local resolver, create `.agentv/otel-backends/.mjs` - * (or a Node-loadable `.js`) and export a resolver object as `default`, - * `otelBackend`, or `resolver`. - */ - -import { access } from 'node:fs/promises'; -import path from 'node:path'; -import { pathToFileURL } from 'node:url'; - -import type { - OtelBackendResolution, - OtelBackendResolver, - OtelBackendResolverContext, -} from '@agentv/core'; - -const RESOLVER_EXTENSIONS = ['.mjs', '.js'] as const; - -const builtinOtelBackendResolvers: readonly OtelBackendResolver[] = [ - { - name: 'langfuse', - resolve: ({ env }) => { - const baseUrl = trimTrailingSlash(env.LANGFUSE_HOST ?? 'https://cloud.langfuse.com'); - const publicKey = env.LANGFUSE_PUBLIC_KEY ?? ''; - const secretKey = env.LANGFUSE_SECRET_KEY ?? ''; - - return { - endpoint: `${baseUrl}/api/public/otel/v1/traces`, - headers: { - Authorization: `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString('base64')}`, - }, - }; - }, - }, - { - name: 'braintrust', - resolve: ({ env }) => { - const headers: Record = { - Authorization: `Bearer ${env.BRAINTRUST_API_KEY ?? ''}`, - }; - const parent = - env.BRAINTRUST_PARENT ?? - (env.BRAINTRUST_PROJECT_ID ? `project_id:${env.BRAINTRUST_PROJECT_ID}` : undefined) ?? - (env.BRAINTRUST_PROJECT ? `project_name:${env.BRAINTRUST_PROJECT}` : undefined); - - if (parent) { - headers['x-bt-parent'] = parent; - } - - return { - endpoint: 'https://api.braintrust.dev/otel/v1/traces', - headers, - }; - }, - }, - { - name: 'confident', - resolve: ({ env }) => ({ - endpoint: 'https://otel.confident-ai.com/v1/traces', - headers: { - 'x-confident-api-key': env.CONFIDENT_API_KEY ?? '', - }, - }), - }, -]; - -const builtinOtelBackendResolversByName = new Map( - builtinOtelBackendResolvers.map((resolver) => [resolver.name, resolver]), -); - -export async function resolveOtelBackend( - name: string, - context: OtelBackendResolverContext, -): Promise { - const resolver = await loadOtelBackendResolver(name, context.cwd); - return resolver?.resolve(context); -} - -export async function loadOtelBackendResolver( - name: string, - cwd: string, -): Promise { - const localResolverPath = await findLocalOtelBackendResolver(name, cwd); - if (localResolverPath) { - return importOtelBackendResolver(localResolverPath, name); - } - - return builtinOtelBackendResolversByName.get(name); -} - -export function getBuiltinOtelBackendResolverNames(): readonly string[] { - return builtinOtelBackendResolvers.map((resolver) => resolver.name); -} - -async function findLocalOtelBackendResolver( - name: string, - cwd: string, -): Promise { - if (!isSafeResolverName(name)) { - return undefined; - } - - for (const dir of getResolverSearchDirs(cwd)) { - for (const ext of RESOLVER_EXTENSIONS) { - const candidate = path.join(dir, `${name}${ext}`); - try { - await access(candidate); - return candidate; - } catch { - // Candidate does not exist in this directory. - } - } - } - - return undefined; -} - -function getResolverSearchDirs(cwd: string): readonly string[] { - const dirs: string[] = []; - let current = path.resolve(cwd); - const root = path.parse(current).root; - - while (current !== root) { - dirs.push(path.join(current, '.agentv', 'otel-backends')); - current = path.dirname(current); - } - - return dirs; -} - -function isSafeResolverName(name: string): boolean { - return name.length > 0 && !name.includes('/') && !name.includes('\\') && !name.startsWith('.'); -} - -async function importOtelBackendResolver( - filePath: string, - fallbackName: string, -): Promise { - const mod = await import(pathToFileURL(filePath).href); - const candidate = [mod.default, mod.otelBackend, mod.resolver].find( - (value) => value && typeof value.resolve === 'function', - ); - - if (!candidate) { - throw new Error( - `OTel backend resolver '${fallbackName}' from ${filePath} must export a resolver object`, - ); - } - - return { - ...candidate, - name: - typeof candidate.name === 'string' && candidate.name.length > 0 - ? candidate.name - : fallbackName, - } as OtelBackendResolver; -} - -function trimTrailingSlash(value: string): string { - return value.replace(/\/+$/, ''); -} diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index c8e5b193f..f7b22916a 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -16,7 +16,6 @@ import { type ExperimentArtifactMetadata, type ExperimentConfig, type FailOnError, - type OtelTraceExporter as OtelTraceExporterType, type ResolvedTarget, ResponseCache, RunBudgetTracker, @@ -67,7 +66,6 @@ import { writeInitialRunSummaryArtifact, } from './artifact-writer.js'; import { loadEnvFromHierarchy } from './env.js'; -import { resolveOtelBackend } from './otel-backends.js'; import { type OutputWriter, createOutputWriter } from './output-writer.js'; import { ProgressDisplay, type Verdict, type WorkerProgress } from './progress-display.js'; import { @@ -285,11 +283,6 @@ interface NormalizedOptions { readonly tsConfigCache?: boolean; readonly tsConfigCachePath?: string; readonly verbose: boolean; - readonly otelFile?: string; - readonly exportOtel: boolean; - readonly otelBackend?: string; - readonly otelCaptureContent: boolean; - readonly otelGroupTurns: boolean; readonly retryErrors?: string; readonly resume: boolean; readonly rerunFailed: boolean; @@ -358,14 +351,6 @@ function resultsRepoOverride( return { repo: value }; } -export function resolveTimestampPlaceholder(value: string): string { - if (!value.includes('{timestamp}')) { - return value; - } - const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); - return value.replaceAll('{timestamp}', timestamp); -} - function normalizeOptionalNumber(value: unknown): number | undefined { if (typeof value === 'number' && Number.isFinite(value)) { return value; @@ -732,22 +717,6 @@ function normalizeOptions( normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true || config?.execution?.verbose === true, - // Precedence: CLI > YAML config > TS config - otelFile: - normalizeString(rawOptions.otelFile) ?? - (yamlExecution?.otel_file - ? resolveTimestampPlaceholder(yamlExecution.otel_file) - : undefined) ?? - (config?.execution?.otelFile - ? resolveTimestampPlaceholder(config.execution.otelFile) - : undefined), - exportOtel: normalizeBoolean(rawOptions.exportOtel) || yamlExecution?.export_otel === true, - otelBackend: normalizeString(rawOptions.otelBackend) ?? yamlExecution?.otel_backend, - otelCaptureContent: - normalizeBoolean(rawOptions.otelCaptureContent) || - yamlExecution?.otel_capture_content === true, - otelGroupTurns: - normalizeBoolean(rawOptions.otelGroupTurns) || yamlExecution?.otel_group_turns === true, retryErrors: normalizeString(rawOptions.retryErrors), resume: normalizeBoolean(rawOptions.resume) || normalizeString(rawOptions.rerunFailed) !== undefined, @@ -1594,7 +1563,6 @@ async function runSingleEvalFile(params: { readonly repoRoot: string; readonly options: NormalizedOptions; readonly outputWriter: OutputWriter; - readonly otelExporter?: OtelTraceExporterType | null; readonly cache?: EvaluationCache; readonly evaluationRunner: typeof defaultRunEvaluation; readonly workersOverride?: number; @@ -1621,7 +1589,6 @@ async function runSingleEvalFile(params: { repoRoot, options, outputWriter, - otelExporter, cache, evaluationRunner, workersOverride, @@ -1705,13 +1672,6 @@ async function runSingleEvalFile(params: { }); } - // Use streaming spans only for live remote export. File exports should use - // post-hoc exportResult(result), which has the complete EvaluationResult and - // avoids cross-test interleaving issues under parallel execution. - const useStreamingObserver = !!(otelExporter && options.exportOtel); - const streamingObserver = useStreamingObserver - ? (otelExporter?.createStreamingObserver() ?? null) - : null; const results = await evaluationRunner({ testFilePath, repoRoot, @@ -1750,36 +1710,14 @@ async function runSingleEvalFile(params: { targetHooks: resolvedTargetSelection.targetHooks, replayRecording, providerFactory, - streamCallbacks: streamingObserver?.getStreamCallbacks(), onResult: async (result: EvaluationResult) => { - ( - streamingObserver as { completeFromResult?: (result: EvaluationResult) => void } | null - )?.completeFromResult?.(result); - // Finalize the streaming observer span with score. - streamingObserver?.finalizeEvalCase(result.score, result.error); - // Trim output messages for results JSONL based on --output-messages. // Each message is trimmed to { role, content } only (no toolCalls, startTime, etc.). - // Full output with tool calls goes to OTel. const resultWithVariant = explicitVariant && !result.variant ? { ...result, variant: explicitVariant } : result; const resultWithMetadata = withSourceMetadata(resultWithVariant, testFilePath, options); const trimmedResult = prepareResultForJsonl(resultWithMetadata, options); await outputWriter.append(trimmedResult); - - // Export to OTel if exporter is configured (skip batch export when streaming is active) - if (otelExporter && !streamingObserver) { - try { - await otelExporter.exportResult(result); - } catch (err) { - // Export failures don't fail the evaluation - if (options.verbose) { - console.warn( - `OTel export warning: ${err instanceof Error ? err.message : String(err)}`, - ); - } - } - } }, onProgress: async (event) => { const testCaseKeyId = matrixMode ? `${event.testId}@${targetName}` : event.testId; @@ -1790,11 +1728,6 @@ async function runSingleEvalFile(params: { } const displayId = displayIdTracker.getOrAssign(testCaseKey); - // Start streaming observer when eval case begins execution - if (event.status === 'running' && streamingObserver) { - streamingObserver.startEvalCase(event.testId, targetName, testFilePath); - } - // Map executionStatus to verdict for display let verdict: Verdict | undefined; if (event.executionStatus === 'ok') verdict = 'PASS'; @@ -2107,79 +2040,8 @@ export async function runEvalCommand( } process.env.AGENTV_RUN_DIR = runDir; - // Initialize OTel exporter if --export-otel or --otel-file is set - let otelExporter: OtelTraceExporterType | null = null; - const useFileExport = !!options.otelFile; - - if (options.exportOtel || useFileExport) { - try { - const { OtelTraceExporter } = await import('@agentv/core'); - - // Resolve endpoint and headers - let endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; - let headers: Record = {}; - let resourceAttributes: Record = {}; - - if (options.otelBackend) { - const resolvedBackend = await resolveOtelBackend(options.otelBackend, { - env: process.env, - cwd, - }); - - if (resolvedBackend) { - endpoint = resolvedBackend.endpoint; - headers = { ...headers, ...resolvedBackend.headers }; - resourceAttributes = { ...resourceAttributes, ...resolvedBackend.resourceAttributes }; - for (const warning of resolvedBackend.warnings ?? []) { - console.warn(warning); - } - } else { - console.warn(`Unknown OTel backend resolver: ${options.otelBackend}`); - } - } - - // Parse OTEL_EXPORTER_OTLP_HEADERS env var - if (process.env.OTEL_EXPORTER_OTLP_HEADERS) { - for (const pair of process.env.OTEL_EXPORTER_OTLP_HEADERS.split(',')) { - const [key, ...rest] = pair.split('='); - if (key) headers[key.trim()] = rest.join('=').trim(); - } - } - - const captureContent = - options.otelCaptureContent || process.env.AGENTV_OTEL_CAPTURE_CONTENT === 'true'; - - otelExporter = new OtelTraceExporter({ - endpoint, - headers, - resourceAttributes, - captureContent, - groupTurns: options.otelGroupTurns, - otlpFilePath: options.otelFile ? path.resolve(options.otelFile) : undefined, - }); - - const initialized = await otelExporter.init(); - if (!initialized) { - console.warn( - 'OTel export requested but @opentelemetry packages not available. Install them to enable export.', - ); - otelExporter = null; - } - } catch (err) { - console.warn( - `OTel export initialization failed: ${err instanceof Error ? err.message : String(err)}`, - ); - otelExporter = null; - } - } - console.log(`Artifact directory: ${runDir}`); - // Log file export paths - if (options.otelFile) { - console.log(`OTLP JSON file: ${path.resolve(options.otelFile)}`); - } - // Determine cache state after loading file metadata (need YAML config) // We defer cache creation until after file metadata is loaded const evaluationRunner = await resolveEvaluationRunner(); @@ -2594,7 +2456,6 @@ export async function runEvalCommand( repoRoot, options: fileOptions, outputWriter, - otelExporter, cache, evaluationRunner, workersOverride: perTargetWorkers, @@ -2904,13 +2765,6 @@ export async function runEvalCommand( unsubscribeCopilotSdkLogs(); unsubscribeCopilotCliLogs(); await outputWriter.close().catch(() => undefined); - if (otelExporter) { - try { - await otelExporter.shutdown(); - } catch { - // Silently ignore shutdown errors - } - } } } diff --git a/apps/cli/src/commands/inspect/score.ts b/apps/cli/src/commands/inspect/score.ts index 4141ed850..3560c545e 100644 --- a/apps/cli/src/commands/inspect/score.ts +++ b/apps/cli/src/commands/inspect/score.ts @@ -378,7 +378,7 @@ export const traceScoreCommand = command({ ); if (!hasTrace) { console.error( - `${c.red}Error:${c.reset} Source lacks trace metrics. Use an OTLP trace export via ${c.bold}--otel-file${c.reset} or a run manifest with summary metrics in ${c.bold}index.jsonl${c.reset}.`, + `${c.red}Error:${c.reset} Source lacks trace metrics. Use a run manifest with summary metrics in ${c.bold}index.jsonl${c.reset} or import an external OTLP trace that already contains metrics.`, ); process.exit(1); } diff --git a/apps/cli/src/commands/inspect/utils.ts b/apps/cli/src/commands/inspect/utils.ts index 14583d15b..0d6a89388 100644 --- a/apps/cli/src/commands/inspect/utils.ts +++ b/apps/cli/src/commands/inspect/utils.ts @@ -107,7 +107,7 @@ export interface RawTraceSpan { * Supported sources: * - Run workspace directories / run manifests * - Standalone trace JSONL files for trace-only workflows - * - OTLP JSON trace files written via --otel-file + * - External OTLP JSON trace files */ export function loadResultFile(filePath: string): RawResult[] { const resolvedFilePath = resolveTraceResultPath(filePath); diff --git a/apps/cli/test/commands/eval/otel-backends.test.ts b/apps/cli/test/commands/eval/otel-backends.test.ts deleted file mode 100644 index 728812709..000000000 --- a/apps/cli/test/commands/eval/otel-backends.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; - -import { - getBuiltinOtelBackendResolverNames, - resolveOtelBackend, -} from '../../../src/commands/eval/otel-backends.js'; - -describe('OTel backend resolvers', () => { - let tempDir: string; - - beforeEach(async () => { - tempDir = await mkdtemp(path.join(tmpdir(), 'agentv-otel-backends-')); - }); - - afterEach(async () => { - await rm(tempDir, { recursive: true, force: true }); - }); - - it('keeps the existing CLI backend names available outside core', () => { - expect(getBuiltinOtelBackendResolverNames()).toEqual(['langfuse', 'braintrust', 'confident']); - }); - - it('resolves Langfuse endpoint and Basic auth headers', async () => { - const resolved = await resolveOtelBackend('langfuse', { - cwd: tempDir, - env: { - LANGFUSE_HOST: 'https://langfuse.example.com/', - LANGFUSE_PUBLIC_KEY: 'pk-test', - LANGFUSE_SECRET_KEY: 'sk-test', - }, - }); - - expect(resolved).toEqual({ - endpoint: 'https://langfuse.example.com/api/public/otel/v1/traces', - headers: { - Authorization: `Basic ${Buffer.from('pk-test:sk-test').toString('base64')}`, - }, - }); - }); - - it('resolves Braintrust auth and project routing headers', async () => { - const resolved = await resolveOtelBackend('braintrust', { - cwd: tempDir, - env: { - BRAINTRUST_API_KEY: 'bt-key', - BRAINTRUST_PROJECT: 'agentv-evals', - }, - }); - - expect(resolved).toEqual({ - endpoint: 'https://api.braintrust.dev/otel/v1/traces', - headers: { - Authorization: 'Bearer bt-key', - 'x-bt-parent': 'project_name:agentv-evals', - }, - }); - }); - - it('resolves Confident auth headers', async () => { - const resolved = await resolveOtelBackend('confident', { - cwd: tempDir, - env: { CONFIDENT_API_KEY: 'conf-key' }, - }); - - expect(resolved).toEqual({ - endpoint: 'https://otel.confident-ai.com/v1/traces', - headers: { - 'x-confident-api-key': 'conf-key', - }, - }); - }); - - it('discovers a project-local resolver by backend name', async () => { - const nestedDir = path.join(tempDir, 'evals', 'suite'); - const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); - await mkdir(nestedDir, { recursive: true }); - await mkdir(resolverDir, { recursive: true }); - await writeFile( - path.join(resolverDir, 'local.mjs'), - ` - export default { - resolve: ({ env, cwd }) => ({ - endpoint: env.LOCAL_OTEL_ENDPOINT ?? cwd, - headers: { "x-local": "true" }, - resourceAttributes: { "agentv.test": "local" }, - }), - }; - `, - 'utf8', - ); - - const resolved = await resolveOtelBackend('local', { - cwd: nestedDir, - env: { LOCAL_OTEL_ENDPOINT: 'https://otel.example.com/v1/traces' }, - }); - - expect(resolved).toEqual({ - endpoint: 'https://otel.example.com/v1/traces', - headers: { 'x-local': 'true' }, - resourceAttributes: { 'agentv.test': 'local' }, - }); - }); - - it('uses a local ESM resolver before a built-in resolver with the same name', async () => { - const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); - await mkdir(resolverDir, { recursive: true }); - await writeFile( - path.join(resolverDir, 'langfuse.mjs'), - ` - export const resolver = { - name: "langfuse", - resolve: () => ({ endpoint: "https://local.example.com/v1/traces" }), - }; - `, - 'utf8', - ); - - const resolved = await resolveOtelBackend('langfuse', { - cwd: tempDir, - env: {}, - }); - - expect(resolved).toEqual({ endpoint: 'https://local.example.com/v1/traces' }); - }); - - it('loads CommonJS .js resolver files', async () => { - const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); - await mkdir(resolverDir, { recursive: true }); - await writeFile( - path.join(resolverDir, 'commonjs.js'), - ` - module.exports = { - name: "commonjs", - resolve: () => ({ endpoint: "https://commonjs.example.com/v1/traces" }), - }; - `, - 'utf8', - ); - - const resolved = await resolveOtelBackend('commonjs', { - cwd: tempDir, - env: {}, - }); - - expect(resolved).toEqual({ endpoint: 'https://commonjs.example.com/v1/traces' }); - }); - - it('returns undefined for unknown backend names', async () => { - const resolved = await resolveOtelBackend('unknown', { cwd: tempDir, env: {} }); - - expect(resolved).toBeUndefined(); - }); - - it('ignores TypeScript resolver files because packaged Node cannot import them', async () => { - const resolverDir = path.join(tempDir, '.agentv', 'otel-backends'); - await mkdir(resolverDir, { recursive: true }); - await writeFile( - path.join(resolverDir, 'typescript-only.ts'), - ` - export default { - resolve: () => ({ endpoint: "https://typescript.example.com/v1/traces" }), - }; - `, - 'utf8', - ); - - const resolved = await resolveOtelBackend('typescript-only', { - cwd: tempDir, - env: {}, - }); - - expect(resolved).toBeUndefined(); - }); -}); diff --git a/apps/cli/test/commands/eval/timestamp-placeholder.test.ts b/apps/cli/test/commands/eval/timestamp-placeholder.test.ts deleted file mode 100644 index 139a40654..000000000 --- a/apps/cli/test/commands/eval/timestamp-placeholder.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { describe, expect, it } from 'bun:test'; - -import { resolveTimestampPlaceholder } from '../../../src/commands/eval/run-eval.js'; - -describe('resolveTimestampPlaceholder', () => { - it('replaces {timestamp} with formatted date', () => { - const result = resolveTimestampPlaceholder('trace-{timestamp}.jsonl'); - expect(result).toMatch(/^trace-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z\.jsonl$/); - }); - - it('returns string unchanged when no placeholder', () => { - const result = resolveTimestampPlaceholder('trace.jsonl'); - expect(result).toBe('trace.jsonl'); - }); - - it('replaces multiple {timestamp} occurrences', () => { - const result = resolveTimestampPlaceholder('{timestamp}-{timestamp}.jsonl'); - expect(result).toMatch(/^\d{4}.*-\d{4}.*\.jsonl$/); - }); -}); diff --git a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx index 04a103bbc..896ca40e4 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/running-evals.mdx @@ -177,122 +177,29 @@ then grade the final state with `agentv grade --prepared` without rerunning the target provider. See [Prepare](/docs/tools/prepare/) for the full workflow, manifest shape, and optional trace/session input with `--trace`. -### Trace Persistence +### Trace Correlation -Export execution traces (tool calls, timing, spans) to files for debugging and analysis: - -By default, AgentV writes a per-run workspace with `index.jsonl` as the canonical manifest for -result-oriented workflows. For full-fidelity span inspection, export OTLP JSON explicitly. +AgentV writes a per-run workspace with `index.jsonl`, `summary.json`, and +per-case sidecars as the canonical record for result-oriented workflows. For +full-fidelity span inspection in systems such as Phoenix, Langfuse, Braintrust, +Jaeger, or Grafana, instrument the system under test or provider wrapper to emit +OpenTelemetry/OpenInference spans directly to that backend during execution. ```bash # Summary-level inspection from the run manifest agentv inspect stats .agentv/results//index.jsonl -# Full-fidelity OTLP JSON trace (importable by OTel backends like Jaeger, Grafana) -agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json - -# Inspect the OTLP export -agentv inspect show traces/eval.otlp.json --tree -``` - -`index.jsonl` contains aggregate metrics such as score, latency, cost, token usage, and summary -trace counters. `--otel-file` writes standard OTLP JSON that can be imported into any -OpenTelemetry-compatible backend. - -For Opik specifically, use `--otel-file` for post-run import or provide your own local backend resolver. AgentV does not currently ship a built-in `opik` backend name. - -### Live OTel Export - -Stream traces directly to an observability backend during evaluation using `--export-otel`: - -```bash -# Use a built-in CLI backend resolver (braintrust, langfuse, confident) -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust - -# Include message content and tool I/O in spans (disabled by default for privacy) -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust --otel-capture-content - -# Group messages into turn spans for multi-turn evaluations -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust --otel-group-turns -``` - -#### Braintrust - -Set up your environment: - -```bash -export BRAINTRUST_API_KEY=sk-... -export BRAINTRUST_PROJECT=my-project # associates traces with a Braintrust project -``` - -Run an eval with traces sent to Braintrust: - -```bash -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust --otel-capture-content -``` - -The following environment variables control project association (at least one is required): - -| Variable | Format | Example | -|----------|--------|---------| -| `BRAINTRUST_PROJECT` | Project name | `my-evals` | -| `BRAINTRUST_PROJECT_ID` | Project UUID | `proj_abc123` | -| `BRAINTRUST_PARENT` | Raw `x-bt-parent` header | `project_name:my-evals` | - -Each eval test case produces a trace with: -- **Root span** (`agentv.eval`) — test ID, target, score, duration -- **LLM call spans** (`chat `) — model name, token usage (input/output/cached) -- **Tool call spans** (`execute_tool `) — tool name, arguments, results (with `--otel-capture-content`) -- **Turn spans** (`agentv.turn.N`) — groups messages by conversation turn (with `--otel-group-turns`) -- **Grader events** — per-grader scores attached to the root span - -:::tip[Claude provider + trace-claude-code plugin] -When using the Claude provider, AgentV injects `CC_PARENT_SPAN_ID` and `CC_ROOT_SPAN_ID` into the Claude subprocess. If the [trace-claude-code](https://github.com/braintrustdata/braintrust-claude-plugin) plugin is installed, it attaches Claude Code CLI-level tool spans (Read, Write, Bash, etc.) as children of the AgentV eval trace, giving you full visibility into both the eval framework and the agent's internal actions. -::: - -#### Langfuse - -```bash -export LANGFUSE_PUBLIC_KEY=pk-... -export LANGFUSE_SECRET_KEY=sk-... -# Optional: export LANGFUSE_HOST=https://cloud.langfuse.com - -agentv eval evals/my-eval.yaml --export-otel --otel-backend langfuse --otel-capture-content -``` - -#### Local Backend Resolvers - -For project-specific backend routing, create `.agentv/otel-backends/.mjs` and select it -with `--otel-backend `: - -```js -export default { - name: 'my-backend', - resolve: ({ env }) => ({ - endpoint: env.MY_OTEL_ENDPOINT ?? 'https://otel.example.com/v1/traces', - headers: { Authorization: `Bearer ${env.MY_OTEL_TOKEN ?? ''}` }, - }), -}; -``` - -```bash -agentv eval evals/my-eval.yaml --export-otel --otel-backend my-backend +# Inspect AgentV-owned per-case artifacts and transcript sidecars +agentv inspect show .agentv/results//index.jsonl --tree ``` -Backend resolvers keep platform-specific endpoint, header, and project-routing logic outside -AgentV core. AgentV also loads Node-compatible `.js` resolver files when you prefer -CommonJS or your project configures `.js` as ESM. - -#### Custom OTLP Endpoint - -For generic OTLP export without a backend resolver, configure via environment variables: - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT=https://your-backend/v1/traces -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer token" - -agentv eval evals/my-eval.yaml --export-otel -``` +`index.jsonl` contains aggregate metrics such as score, latency, cost, token +usage, summary trace counters, and paths to generated artifacts. When another +observability system emitted spans independently, include safe `external_trace` +metadata in the result or trace artifact so AgentV can correlate a case to the +external trace ID, span ID, or UI URL without making that backend the source of +truth. AgentV does not synthesize OTLP from completed eval transcripts as the +primary observability path. ### Parallelism @@ -556,7 +463,6 @@ Use `config.yaml` for portable defaults that can be committed with the eval proj execution: verbose: true keep_workspaces: false - otel_file: .agentv/results/otel-{timestamp}.json refs: global-default: file://{{ env.AGENTV_REPO_ROOT }}/.agentv/default-test.yaml ``` @@ -577,7 +483,6 @@ eval_patterns: | `verbose` | `--verbose` | boolean | `false` | Enable verbose logging | | `keep_workspaces` | `--keep-workspaces` | boolean | `false` | Always keep temp workspaces after eval | | `workspace_path` | `--workspace-path` | string | none | Machine-local existing workspace directory | -| `otel_file` | `--otel-file` | string | none | Write OTLP JSON trace to file | | `refs` | none | object | none | Project-defined named references for fields that support `ref://name`, such as shared `default_test` files | ### TypeScript config (`agentv.config.ts`) @@ -589,13 +494,10 @@ export default defineConfig({ execution: { verbose: true, keepWorkspaces: false, - otelFile: '.agentv/results/otel-{timestamp}.json', }, }); ``` -The `{timestamp}` placeholder is replaced with an ISO-like timestamp (e.g., `2026-03-05T14-30-00-000Z`) at execution time. - **Precedence:** CLI flags > project-local `.agentv/config.local.yaml` over `.agentv/config.yaml` > home/global `$AGENTV_HOME/config.local.yaml` over `$AGENTV_HOME/config.yaml` (or `~/.agentv/...`) > `agentv.config.ts` > built-in defaults. ## Response Cache diff --git a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx index 9b99ba303..9f3411f83 100644 --- a/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/next/evaluation/sdk.mdx @@ -408,7 +408,6 @@ export default defineConfig({ workers: 5, maxRetries: 2, verbose: true, - otelFile: '.agentv/results/otel-{timestamp}.json', }, output: { dir: './results' }, limits: { maxCostUsd: 10.0 }, @@ -417,14 +416,13 @@ export default defineConfig({ The config file is auto-discovered by the CLI from your project root and validated with Zod at startup. -## Observability Export +## Trace Correlation -AgentV's observability surface is OpenTelemetry. For post-run workflows: - -- Use `agentv eval ... --otel-file traces/eval.otlp.json` to write OTLP JSON you can import into systems such as Opik. -- Use `agentv eval ... --export-otel --otel-backend ` for live export when a built-in or local resolver exists. - -AgentV does not currently ship a dedicated Opik authoring facade or built-in `opik` backend resolver. Keep the eval definition in YAML and route observability through OTLP export. +AgentV stores eval-native run bundles, transcript sidecars, metrics, grading +artifacts, and summaries. For OpenTelemetry/OpenInference traces, instrument the +system under test or provider wrapper to emit spans directly to the external +backend, then correlate AgentV cases to those traces with safe `external_trace` +IDs or UI URLs when available. ## Scaffold Commands diff --git a/apps/web/src/content/docs/docs/next/integrations/langfuse.mdx b/apps/web/src/content/docs/docs/next/integrations/langfuse.mdx index a7906e774..d7de5c4b1 100644 --- a/apps/web/src/content/docs/docs/next/integrations/langfuse.mdx +++ b/apps/web/src/content/docs/docs/next/integrations/langfuse.mdx @@ -1,146 +1,31 @@ --- title: Langfuse -description: Export AgentV evaluation traces to Langfuse via OpenTelemetry +description: Correlate AgentV runs with traces emitted directly to Langfuse. sidebar: order: 1 --- -AgentV streams evaluation traces to [Langfuse](https://langfuse.com) using standard OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend resolver handles endpoint construction and authentication automatically. - -## Quick Start - -Set your Langfuse credentials as environment variables: - -```bash -export LANGFUSE_PUBLIC_KEY=pk-lf-... -export LANGFUSE_SECRET_KEY=sk-lf-... -``` - -Run an eval with Langfuse export enabled: - -```bash -agentv eval evals/my-eval.yaml --export-otel --otel-backend langfuse -``` - -Traces appear in your Langfuse dashboard within seconds. - -:::tip -You can also set these in a `.env` file in your project root. See the [working example](https://github.com/EntityProcess/agentv/tree/main/examples/features/langfuse-export) for a complete setup. -::: - -## How It Works - -AgentV uses the vendor-neutral OpenTelemetry protocol (OTLP/HTTP) to send traces. When you select the `langfuse` backend: - -1. **Endpoint** is constructed as `{LANGFUSE_HOST}/api/public/otel/v1/traces` (defaults to `https://cloud.langfuse.com`) -2. **Authentication** uses HTTP Basic Auth built from `LANGFUSE_PUBLIC_KEY:LANGFUSE_SECRET_KEY` -3. **No SDK dependency** — AgentV sends standard OTLP payloads that Langfuse's OTel-compatible ingestion endpoint accepts directly - -## Span Semantics — What Shows Up in Langfuse - -Each eval test case produces a trace with the following span hierarchy: - -| Span | Name pattern | Key attributes | -|------|-------------|----------------| -| Root | `agentv.eval` | test ID, target, score, duration | -| LLM call | `chat ` | model name, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | -| Tool call | `execute_tool ` | tool name, arguments, results (with `--otel-capture-content`) | -| Turn | `agentv.turn.N` | groups messages by conversation turn (with `--otel-group-turns`) | - -Langfuse dashboards recognize the `gen_ai.*` semantic conventions and display token usage, model names, and cost breakdowns automatically. - -## CLI Flags Reference - -| Flag | Description | -|------|-------------| -| `--export-otel` | Enable live OTel export | -| `--otel-backend langfuse` | Use the Langfuse endpoint and auth resolver | -| `--otel-capture-content` | Include message and tool content in spans (disabled by default for privacy) | -| `--otel-group-turns` | Add `agentv.turn.N` parent spans that group messages by conversation turn | - -:::caution[Privacy] -`--otel-capture-content` sends full message and tool I/O to Langfuse. Only enable this when your Langfuse instance has appropriate access controls for the data being evaluated. -::: - -## Config.yaml Alternative - -Instead of passing CLI flags every time, declare OTel settings in `.agentv/config.yaml`: - -```yaml -export_otel: true -otel_backend: langfuse +AgentV does not export completed eval transcripts or run bundles to Langfuse. +Keep AgentV run artifacts as the source of truth for scoring, transcripts, +metrics, and summaries. If you need Langfuse trace inspection, instrument the +system under test or provider wrapper to emit OpenTelemetry/OpenInference spans +directly to Langfuse during execution. + +## Correlation + +When the external run produces a Langfuse trace ID or UI URL, attach it to the +AgentV result metadata as safe `external_trace` metadata: + +```json +{ + "external_trace": { + "provider": "langfuse", + "trace_id": "trace-123", + "ui_url": "https://cloud.langfuse.com/project/example/traces/trace-123" + } +} ``` -This is equivalent to running with `--export-otel --otel-backend langfuse` on every eval. CLI flags override config.yaml values when both are present. - -You can combine this with other config options: - -```yaml -export_otel: true -otel_backend: langfuse -verbose: true -``` - -## Self-Hosted Langfuse - -For self-hosted Langfuse instances, set the `LANGFUSE_HOST` environment variable: - -```bash -export LANGFUSE_HOST=https://your-langfuse-instance.com -``` - -AgentV constructs the OTel endpoint as `{LANGFUSE_HOST}/api/public/otel/v1/traces`. The authentication mechanism is the same — Basic Auth from your public and secret keys. - -## CI/CD (GitHub Actions) - -Export eval traces to Langfuse on every push: - -```yaml -name: Eval with Langfuse -on: [push] -jobs: - eval: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - - run: npm install -g agentv - - run: agentv eval evals/*.yaml --export-otel --otel-backend langfuse - env: - LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} - LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} -``` - -:::note -Store `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` as GitHub Actions secrets. Never commit credentials to your repository. -::: - -## Troubleshooting - -### Authentication failures - -If you see 401 or 403 errors, verify your keys are set correctly: - -```bash -# Check that both variables are present -echo "Public: ${LANGFUSE_PUBLIC_KEY:0:10}..." -echo "Secret: ${LANGFUSE_SECRET_KEY:0:10}..." -``` - -Ensure you are using the correct key pair for the Langfuse project you expect traces to appear in. - -### Traces not appearing - -- **Propagation delay** — traces may take a few seconds to appear in the Langfuse dashboard after an eval completes. -- **Wrong project** — each key pair is scoped to a specific Langfuse project. Confirm you are viewing the correct project in the dashboard. -- **Self-hosted endpoint** — if using `LANGFUSE_HOST`, verify the URL is reachable and includes the protocol (`https://`). - -### Rate limiting (429 responses) - -AgentV includes built-in exponential backoff for transient errors. If you are running many concurrent evals, you may still hit rate limits. Reduce concurrency or contact Langfuse support for higher limits. - -## Working Example - -The [`examples/features/langfuse-export/`](https://github.com/EntityProcess/agentv/tree/main/examples/features/langfuse-export) directory contains a complete working setup with config.yaml, .env.example, and sample eval file. Clone the repo and follow the README to get traces flowing in minutes. +Do not store API keys, authorization headers, cookies, or raw private payloads +in `external_trace`. AgentV may use the safe URL as a link-out; Langfuse remains +the external trace viewer and AgentV remains the eval artifact owner. diff --git a/apps/web/src/content/docs/docs/next/tools/inspect.mdx b/apps/web/src/content/docs/docs/next/tools/inspect.mdx index e7c7913bb..6b85c0278 100644 --- a/apps/web/src/content/docs/docs/next/tools/inspect.mdx +++ b/apps/web/src/content/docs/docs/next/tools/inspect.mdx @@ -11,9 +11,11 @@ Supported sources: - Run workspaces or `index.jsonl` manifests for summary-level fallback - Legacy simple trace JSONL files for read-only migration scenarios -- OTLP JSON files written via `agentv eval --otel-file ...` +- External OTLP JSON files -For full tool-call inspection, prefer OTLP JSON exports over eval manifests. +For full external span inspection, use an OTLP JSON trace emitted by the system +under test or provider. For AgentV-owned evidence, prefer run manifests and +their transcript sidecars. ## Subcommands diff --git a/apps/web/src/content/docs/docs/next/tools/prepare.mdx b/apps/web/src/content/docs/docs/next/tools/prepare.mdx index b9f2a8dbb..2469e59ba 100644 --- a/apps/web/src/content/docs/docs/next/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/next/tools/prepare.mdx @@ -73,7 +73,7 @@ Use `--response` when the final answer text should be graded independently of th `prepare` is not a replacement for live observability. Configure live tracing in the harness or target itself: - Use provider-native settings, target hooks, or environment variables to enable session logs. -- Use AgentV's OTLP options during normal eval runs, such as `--otel-file` or `--export-otel`, when AgentV is the runner. +- Emit OpenTelemetry/OpenInference spans from the system under test or provider when you need external trace inspection. - For Opik, Langfuse, or another export-capable backend, treat their traces as external artifacts that can be imported or projected back into AgentV later. - For Phoenix, use only optional link-out correlation when safe `external_trace` metadata points to spans already emitted independently by Codex, Arize, or another hook. diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx index 4cf0df159..d3ecc6828 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/running-evals.mdx @@ -144,122 +144,29 @@ then grade the final state with `agentv grade --prepared` without rerunning the target provider. See [Prepare](/docs/v4.42.4/tools/prepare/) for the full workflow, manifest shape, and optional trace/session input with `--trace`. -### Trace Persistence +### Trace Correlation -Export execution traces (tool calls, timing, spans) to files for debugging and analysis: - -By default, AgentV writes a per-run workspace with `index.jsonl` as the canonical manifest for -result-oriented workflows. For full-fidelity span inspection, export OTLP JSON explicitly. +AgentV writes a per-run workspace with `index.jsonl`, `summary.json`, and +per-case sidecars as the canonical record for result-oriented workflows. For +full-fidelity span inspection in systems such as Phoenix, Langfuse, Braintrust, +Jaeger, or Grafana, instrument the system under test or provider wrapper to emit +OpenTelemetry/OpenInference spans directly to that backend during execution. ```bash # Summary-level inspection from the run manifest agentv inspect stats .agentv/results/runs//index.jsonl -# Full-fidelity OTLP JSON trace (importable by OTel backends like Jaeger, Grafana) -agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json - -# Inspect the OTLP export -agentv inspect show traces/eval.otlp.json --tree -``` - -`index.jsonl` contains aggregate metrics such as score, latency, cost, token usage, and summary -trace counters. `--otel-file` writes standard OTLP JSON that can be imported into any -OpenTelemetry-compatible backend. - -For Opik specifically, use `--otel-file` for post-run import or provide your own local backend resolver. AgentV does not currently ship a built-in `opik` backend name. - -### Live OTel Export - -Stream traces directly to an observability backend during evaluation using `--export-otel`: - -```bash -# Use a built-in CLI backend resolver (braintrust, langfuse, confident) -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust - -# Include message content and tool I/O in spans (disabled by default for privacy) -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust --otel-capture-content - -# Group messages into turn spans for multi-turn evaluations -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust --otel-group-turns -``` - -#### Braintrust - -Set up your environment: - -```bash -export BRAINTRUST_API_KEY=sk-... -export BRAINTRUST_PROJECT=my-project # associates traces with a Braintrust project -``` - -Run an eval with traces sent to Braintrust: - -```bash -agentv eval evals/my-eval.yaml --export-otel --otel-backend braintrust --otel-capture-content -``` - -The following environment variables control project association (at least one is required): - -| Variable | Format | Example | -|----------|--------|---------| -| `BRAINTRUST_PROJECT` | Project name | `my-evals` | -| `BRAINTRUST_PROJECT_ID` | Project UUID | `proj_abc123` | -| `BRAINTRUST_PARENT` | Raw `x-bt-parent` header | `project_name:my-evals` | - -Each eval test case produces a trace with: -- **Root span** (`agentv.eval`) — test ID, target, score, duration -- **LLM call spans** (`chat `) — model name, token usage (input/output/cached) -- **Tool call spans** (`execute_tool `) — tool name, arguments, results (with `--otel-capture-content`) -- **Turn spans** (`agentv.turn.N`) — groups messages by conversation turn (with `--otel-group-turns`) -- **Grader events** — per-grader scores attached to the root span - -:::tip[Claude provider + trace-claude-code plugin] -When using the Claude provider, AgentV injects `CC_PARENT_SPAN_ID` and `CC_ROOT_SPAN_ID` into the Claude subprocess. If the [trace-claude-code](https://github.com/braintrustdata/braintrust-claude-plugin) plugin is installed, it attaches Claude Code CLI-level tool spans (Read, Write, Bash, etc.) as children of the AgentV eval trace, giving you full visibility into both the eval framework and the agent's internal actions. -::: - -#### Langfuse - -```bash -export LANGFUSE_PUBLIC_KEY=pk-... -export LANGFUSE_SECRET_KEY=sk-... -# Optional: export LANGFUSE_HOST=https://cloud.langfuse.com - -agentv eval evals/my-eval.yaml --export-otel --otel-backend langfuse --otel-capture-content -``` - -#### Local Backend Resolvers - -For project-specific backend routing, create `.agentv/otel-backends/.mjs` and select it -with `--otel-backend `: - -```js -export default { - name: 'my-backend', - resolve: ({ env }) => ({ - endpoint: env.MY_OTEL_ENDPOINT ?? 'https://otel.example.com/v1/traces', - headers: { Authorization: `Bearer ${env.MY_OTEL_TOKEN ?? ''}` }, - }), -}; -``` - -```bash -agentv eval evals/my-eval.yaml --export-otel --otel-backend my-backend +# Inspect AgentV-owned per-case artifacts and transcript sidecars +agentv inspect show .agentv/results/runs//index.jsonl --tree ``` -Backend resolvers keep platform-specific endpoint, header, and project-routing logic outside -AgentV core. AgentV also loads Node-compatible `.js` resolver files when you prefer -CommonJS or your project configures `.js` as ESM. - -#### Custom OTLP Endpoint - -For generic OTLP export without a backend resolver, configure via environment variables: - -```bash -export OTEL_EXPORTER_OTLP_ENDPOINT=https://your-backend/v1/traces -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer token" - -agentv eval evals/my-eval.yaml --export-otel -``` +`index.jsonl` contains aggregate metrics such as score, latency, cost, token +usage, summary trace counters, and paths to generated artifacts. When another +observability system emitted spans independently, include safe `external_trace` +metadata in the result or trace artifact so AgentV can correlate a case to the +external trace ID, span ID, or UI URL without making that backend the source of +truth. AgentV does not synthesize OTLP from completed eval transcripts as the +primary observability path. ### Parallelism @@ -502,14 +409,12 @@ Project-local YAML config takes precedence over home/global YAML config. AgentV execution: verbose: true keep_workspaces: false - otel_file: .agentv/results/otel-{timestamp}.json ``` | Field | CLI equivalent | Type | Default | Description | |-------|---------------|------|---------|-------------| | `verbose` | `--verbose` | boolean | `false` | Enable verbose logging | | `keep_workspaces` | `--keep-workspaces` | boolean | `false` | Always keep temp workspaces after eval | -| `otel_file` | `--otel-file` | string | none | Write OTLP JSON trace to file | ### TypeScript config (`agentv.config.ts`) @@ -520,13 +425,10 @@ export default defineConfig({ execution: { verbose: true, keepWorkspaces: false, - otelFile: '.agentv/results/otel-{timestamp}.json', }, }); ``` -The `{timestamp}` placeholder is replaced with an ISO-like timestamp (e.g., `2026-03-05T14-30-00-000Z`) at execution time. - **Precedence:** CLI flags > project-local `.agentv/config.yaml` > home/global `$AGENTV_HOME/config.yaml` (or `~/.agentv/config.yaml`) > `agentv.config.ts` > built-in defaults. ## Response Cache diff --git a/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx b/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx index 423c9e30b..83f81f091 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/evaluation/sdk.mdx @@ -353,7 +353,6 @@ export default defineConfig({ workers: 5, maxRetries: 2, verbose: true, - otelFile: '.agentv/results/otel-{timestamp}.json', }, output: { dir: './results' }, limits: { maxCostUsd: 10.0 }, @@ -362,14 +361,13 @@ export default defineConfig({ The config file is auto-discovered by the CLI from your project root and validated with Zod at startup. -## Observability Export +## Trace Correlation -AgentV's observability surface is OpenTelemetry. For post-run workflows: - -- Use `agentv eval ... --otel-file traces/eval.otlp.json` to write OTLP JSON you can import into systems such as Opik. -- Use `agentv eval ... --export-otel --otel-backend ` for live export when a built-in or local resolver exists. - -AgentV does not currently ship a dedicated Opik authoring facade or built-in `opik` backend resolver. Keep the eval definition in YAML and route observability through OTLP export. +AgentV stores eval-native run bundles, transcript sidecars, metrics, grading +artifacts, and summaries. For OpenTelemetry/OpenInference traces, instrument the +system under test or provider wrapper to emit spans directly to the external +backend, then correlate AgentV cases to those traces with safe `external_trace` +IDs or UI URLs when available. ## Scaffold Commands diff --git a/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx b/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx index 9cb7b3f32..7ee5b25a1 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/guides/human-review.mdx @@ -70,7 +70,7 @@ Create a `feedback.json` file in the run workspace, alongside `index.jsonl`: results/ 2026-03-14T10-32-00_claude/ index.jsonl # run manifest - trace.otlp.json # optional OTLP trace export + transcripts/ # normalized transcript artifacts feedback.json # ← your review annotations ``` diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/langfuse.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/langfuse.mdx index 7275bb014..0b94a27e8 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/langfuse.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/langfuse.mdx @@ -1,6 +1,6 @@ --- title: Langfuse -description: Export AgentV evaluation traces to Langfuse via OpenTelemetry +description: Correlate AgentV runs with traces emitted directly to Langfuse. sidebar: order: 1 slug: docs/v4.42.4/integrations/langfuse @@ -8,142 +8,27 @@ editUrl: false pagefind: false --- -AgentV streams evaluation traces to [Langfuse](https://langfuse.com) using standard OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend resolver handles endpoint construction and authentication automatically. - -## Quick Start - -Set your Langfuse credentials as environment variables: - -```bash -export LANGFUSE_PUBLIC_KEY=pk-lf-... -export LANGFUSE_SECRET_KEY=sk-lf-... -``` - -Run an eval with Langfuse export enabled: - -```bash -agentv eval evals/my-eval.yaml --export-otel --otel-backend langfuse -``` - -Traces appear in your Langfuse dashboard within seconds. - -:::tip -You can also set these in a `.env` file in your project root. See the [working example](https://github.com/EntityProcess/agentv/tree/main/examples/features/langfuse-export) for a complete setup. -::: - -## How It Works - -AgentV uses the vendor-neutral OpenTelemetry protocol (OTLP/HTTP) to send traces. When you select the `langfuse` backend: - -1. **Endpoint** is constructed as `{LANGFUSE_HOST}/api/public/otel/v1/traces` (defaults to `https://cloud.langfuse.com`) -2. **Authentication** uses HTTP Basic Auth built from `LANGFUSE_PUBLIC_KEY:LANGFUSE_SECRET_KEY` -3. **No SDK dependency** — AgentV sends standard OTLP payloads that Langfuse's OTel-compatible ingestion endpoint accepts directly - -## Span Semantics — What Shows Up in Langfuse - -Each eval test case produces a trace with the following span hierarchy: - -| Span | Name pattern | Key attributes | -|------|-------------|----------------| -| Root | `agentv.eval` | test ID, target, score, duration | -| LLM call | `chat ` | model name, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` | -| Tool call | `execute_tool ` | tool name, arguments, results (with `--otel-capture-content`) | -| Turn | `agentv.turn.N` | groups messages by conversation turn (with `--otel-group-turns`) | - -Langfuse dashboards recognize the `gen_ai.*` semantic conventions and display token usage, model names, and cost breakdowns automatically. - -## CLI Flags Reference - -| Flag | Description | -|------|-------------| -| `--export-otel` | Enable live OTel export | -| `--otel-backend langfuse` | Use the Langfuse endpoint and auth resolver | -| `--otel-capture-content` | Include message and tool content in spans (disabled by default for privacy) | -| `--otel-group-turns` | Add `agentv.turn.N` parent spans that group messages by conversation turn | - -:::caution[Privacy] -`--otel-capture-content` sends full message and tool I/O to Langfuse. Only enable this when your Langfuse instance has appropriate access controls for the data being evaluated. -::: - -## Config.yaml Alternative - -Instead of passing CLI flags every time, declare OTel settings in `.agentv/config.yaml`: - -```yaml -export_otel: true -otel_backend: langfuse +AgentV does not export completed eval transcripts or run bundles to Langfuse. +Keep AgentV run artifacts as the source of truth for scoring, transcripts, +metrics, and summaries. If you need Langfuse trace inspection, instrument the +system under test or provider wrapper to emit OpenTelemetry/OpenInference spans +directly to Langfuse during execution. + +## Correlation + +When the external run produces a Langfuse trace ID or UI URL, attach it to the +AgentV result metadata as safe `external_trace` metadata: + +```json +{ + "external_trace": { + "provider": "langfuse", + "trace_id": "trace-123", + "ui_url": "https://cloud.langfuse.com/project/example/traces/trace-123" + } +} ``` -This is equivalent to running with `--export-otel --otel-backend langfuse` on every eval. CLI flags override config.yaml values when both are present. - -You can combine this with other config options: - -```yaml -export_otel: true -otel_backend: langfuse -verbose: true -``` - -## Self-Hosted Langfuse - -For self-hosted Langfuse instances, set the `LANGFUSE_HOST` environment variable: - -```bash -export LANGFUSE_HOST=https://your-langfuse-instance.com -``` - -AgentV constructs the OTel endpoint as `{LANGFUSE_HOST}/api/public/otel/v1/traces`. The authentication mechanism is the same — Basic Auth from your public and secret keys. - -## CI/CD (GitHub Actions) - -Export eval traces to Langfuse on every push: - -```yaml -name: Eval with Langfuse -on: [push] -jobs: - eval: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 22 - - run: npm install -g agentv - - run: agentv eval evals/*.yaml --export-otel --otel-backend langfuse - env: - LANGFUSE_PUBLIC_KEY: ${{ secrets.LANGFUSE_PUBLIC_KEY }} - LANGFUSE_SECRET_KEY: ${{ secrets.LANGFUSE_SECRET_KEY }} -``` - -:::note -Store `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` as GitHub Actions secrets. Never commit credentials to your repository. -::: - -## Troubleshooting - -### Authentication failures - -If you see 401 or 403 errors, verify your keys are set correctly: - -```bash -# Check that both variables are present -echo "Public: ${LANGFUSE_PUBLIC_KEY:0:10}..." -echo "Secret: ${LANGFUSE_SECRET_KEY:0:10}..." -``` - -Ensure you are using the correct key pair for the Langfuse project you expect traces to appear in. - -### Traces not appearing - -- **Propagation delay** — traces may take a few seconds to appear in the Langfuse dashboard after an eval completes. -- **Wrong project** — each key pair is scoped to a specific Langfuse project. Confirm you are viewing the correct project in the dashboard. -- **Self-hosted endpoint** — if using `LANGFUSE_HOST`, verify the URL is reachable and includes the protocol (`https://`). - -### Rate limiting (429 responses) - -AgentV includes built-in exponential backoff for transient errors. If you are running many concurrent evals, you may still hit rate limits. Reduce concurrency or contact Langfuse support for higher limits. - -## Working Example - -The [`examples/features/langfuse-export/`](https://github.com/EntityProcess/agentv/tree/main/examples/features/langfuse-export) directory contains a complete working setup with config.yaml, .env.example, and sample eval file. Clone the repo and follow the README to get traces flowing in minutes. +Do not store API keys, authorization headers, cookies, or raw private payloads +in `external_trace`. AgentV may use the safe URL as a link-out; Langfuse remains +the external trace viewer and AgentV remains the eval artifact owner. diff --git a/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx index b029b4dcc..a7cee0c2a 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/integrations/phoenix.mdx @@ -88,18 +88,10 @@ evaluator code, but the adapter does not attempt a lossy automatic conversion. ## Traces vs Datasets The Phoenix adapter creates dataset and experiment payloads. It is separate from -AgentV's OpenTelemetry trace export. - -For trace export, use AgentV's standard OTel options: - -```bash -agentv eval evals/my-eval.yaml --otel-file traces/eval.otlp.json -``` - -For live OTel export to Phoenix, use a local `.agentv/otel-backends/phoenix.mjs` -resolver that imports `phoenixOtelBackend` from `@agentv/phoenix-adapter`, then select it -with `--otel-backend phoenix`. General live export options are documented in -[Running Evaluations](/docs/v4.42.4/evaluation/running-evals/#live-otel-export/). +external trace correlation. If you need Phoenix span inspection, instrument the +system under test or provider wrapper to emit OpenTelemetry/OpenInference spans +directly to Phoenix, then attach safe `external_trace` metadata to the AgentV +run artifact for link-out correlation. ## Package Docs diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/inspect.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/inspect.mdx index 59e3a5cfd..54cb2d977 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/inspect.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/inspect.mdx @@ -14,9 +14,11 @@ Supported sources: - Run workspaces or `index.jsonl` manifests for summary-level fallback - Legacy simple trace JSONL files for read-only migration scenarios -- OTLP JSON files written via `agentv eval --otel-file ...` +- External OTLP JSON files -For full tool-call inspection, prefer OTLP JSON exports over eval manifests. +For full external span inspection, use an OTLP JSON trace emitted by the system +under test or provider. For AgentV-owned evidence, prefer run manifests and +their transcript sidecars. ## Subcommands diff --git a/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx b/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx index 6856b521b..94a83b7e4 100644 --- a/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx +++ b/apps/web/src/content/docs/docs/v4.42.4/tools/prepare.mdx @@ -76,7 +76,7 @@ Use `--response` when the final answer text should be graded independently of th `prepare` is not a replacement for live observability. Configure live tracing in the harness or target itself: - Use provider-native settings, target hooks, or environment variables to enable session logs. -- Use AgentV's OTLP options during normal eval runs, such as `--otel-file` or `--export-otel`, when AgentV is the runner. +- Emit OpenTelemetry/OpenInference spans from the system under test or provider when you need external trace inspection. - For Opik, Phoenix, Langfuse, or another backend, treat their traces as external artifacts that can be imported or projected back into AgentV later. AgentV remains responsible for eval definitions, workspace setup, grading, result bundles, and CI gates. Live trace storage, dashboards, and provider-specific run monitoring belong in the observability backend or the external harness. diff --git a/bun.lock b/bun.lock index d6fbdd77e..1e891deb9 100644 --- a/bun.lock +++ b/bun.lock @@ -103,14 +103,6 @@ "@types/nunjucks": "^3.2.6", "zod-to-json-schema": "^3.25.1", }, - "optionalDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/core": "^2.5.1", - "@opentelemetry/exporter-trace-otlp-http": "^0.212.0", - "@opentelemetry/resources": "^2.5.1", - "@opentelemetry/sdk-trace-node": "^2.5.1", - "@opentelemetry/semantic-conventions": "^1.39.0", - }, "peerDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.88", "@earendil-works/pi-coding-agent": "^0.74.0", @@ -511,32 +503,6 @@ "@openai/codex-win32-x64": ["@openai/codex@0.136.0-win32-x64", "", { "os": "win32", "cpu": "x64" }, "sha512-zS6DAmvjdWeAB1CL9KTUMkwzTwfXtxHy8GAtePw2a93jIqawoG07fBxAXuyoHZ3QXQkwEgqBx1zEEh33gdIKAw=="], - "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], - - "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.212.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-TEEVrLbNROUkYY51sBJGk7lO/OLjuepch8+hmpM6ffMJQ2z/KVCjdHuCFX6fJj8OkJP2zckPjrJzQtXU3IAsFg=="], - - "@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.5.1", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-MHbu8XxCHcBn6RwvCt2Vpn1WnLMNECfNKYB14LI5XypcgH4IE0/DiVifVR9tAkwPMyLXN8dOoPJfya3IryLQVw=="], - - "@opentelemetry/core": ["@opentelemetry/core@2.5.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA=="], - - "@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.212.0", "", { "dependencies": { "@opentelemetry/core": "2.5.1", "@opentelemetry/otlp-exporter-base": "0.212.0", "@opentelemetry/otlp-transformer": "0.212.0", "@opentelemetry/resources": "2.5.1", "@opentelemetry/sdk-trace-base": "2.5.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-v/0wMozNoiEPRolzC4YoPo4rAT0q8r7aqdnRw3Nu7IDN0CGFzNQazkfAlBJ6N5y0FYJkban7Aw5WnN73//6YlA=="], - - "@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.212.0", "", { "dependencies": { "@opentelemetry/core": "2.5.1", "@opentelemetry/otlp-transformer": "0.212.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-HoMv5pQlzbuxiMS0hN7oiUtg8RsJR5T7EhZccumIWxYfNo/f4wFc7LPDfFK6oHdG2JF/+qTocfqIHoom+7kLpw=="], - - "@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.212.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.212.0", "@opentelemetry/core": "2.5.1", "@opentelemetry/resources": "2.5.1", "@opentelemetry/sdk-logs": "0.212.0", "@opentelemetry/sdk-metrics": "2.5.1", "@opentelemetry/sdk-trace-base": "2.5.1", "protobufjs": "8.0.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-bj7zYFOg6Db7NUwsRZQ/WoVXpAf41WY2gsd3kShSfdpZQDRKHWJiRZIg7A8HvWsf97wb05rMFzPbmSHyjEl9tw=="], - - "@opentelemetry/resources": ["@opentelemetry/resources@2.5.1", "", { "dependencies": { "@opentelemetry/core": "2.5.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-BViBCdE/GuXRlp9k7nS1w6wJvY5fnFX5XvuEtWsTAOQFIO89Eru7lGW3WbfbxtCuZ/GbrJfAziXG0w0dpxL7eQ=="], - - "@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.212.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.212.0", "@opentelemetry/core": "2.5.1", "@opentelemetry/resources": "2.5.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-qglb5cqTf0mOC1sDdZ7nfrPjgmAqs2OxkzOPIf2+Rqx8yKBK0pS7wRtB1xH30rqahBIut9QJDbDePyvtyqvH/Q=="], - - "@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@2.5.1", "", { "dependencies": { "@opentelemetry/core": "2.5.1", "@opentelemetry/resources": "2.5.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.9.0 <1.10.0" } }, "sha512-RKMn3QKi8nE71ULUo0g/MBvq1N4icEBo7cQSKnL3URZT16/YH3nSVgWegOjwx7FRBTrjOIkMJkCUn/ZFIEfn4A=="], - - "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.5.1", "", { "dependencies": { "@opentelemetry/core": "2.5.1", "@opentelemetry/resources": "2.5.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw=="], - - "@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.5.1", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.5.1", "@opentelemetry/core": "2.5.1", "@opentelemetry/sdk-trace-base": "2.5.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-9lopQ6ZoElETOEN0csgmtEV5/9C7BMfA7VtF4Jape3i954b6sTY2k3Xw3CxUTKreDck/vpAuJM+EDo4zheUw+A=="], - - "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.39.0", "", {}, "sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg=="], - "@oslojs/encoding": ["@oslojs/encoding@1.1.0", "", {}, "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ=="], "@pagefind/darwin-arm64": ["@pagefind/darwin-arm64@1.4.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-2vMqkbv3lbx1Awea90gTaBsvpzgRs7MuSgKDxW0m9oV1GPZCZbZBJg/qL83GIUEN2BFlY46dtUZi54pwH+/pTQ=="], @@ -557,7 +523,7 @@ "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], - "@protobufjs/codegen": ["@protobufjs/codegen@2.0.4", "", {}, "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="], + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], @@ -565,13 +531,13 @@ "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], - "@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.1", "", {}, "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew=="], "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], - "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="], "@reduxjs/toolkit": ["@reduxjs/toolkit@2.11.2", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "@standard-schema/utils": "^0.3.0", "immer": "^11.0.0", "redux": "^5.0.1", "redux-thunk": "^3.1.0", "reselect": "^5.1.0" }, "peerDependencies": { "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" }, "optionalPeers": ["react", "react-redux"] }, "sha512-Kd6kAHTA6/nUpp8mySPqj3en3dm0tdMIgbttnQ1xFMVpufoj+ADi8pXLBsd4xzTRHQa7t/Jv8W5UnCuW4kuWMQ=="], @@ -1585,7 +1551,7 @@ "property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], - "protobufjs": ["protobufjs@8.0.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw=="], + "protobufjs": ["protobufjs@7.5.6", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.1", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg=="], "proxy-agent": ["proxy-agent@6.5.0", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "^4.3.4", "http-proxy-agent": "^7.0.1", "https-proxy-agent": "^7.0.6", "lru-cache": "^7.14.1", "pac-proxy-agent": "^7.1.0", "proxy-from-env": "^1.1.0", "socks-proxy-agent": "^8.0.5" } }, "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A=="], @@ -1921,12 +1887,12 @@ "@github/copilot-sdk/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], - "@google/genai/protobufjs": ["protobufjs@7.5.6", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.1", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg=="], - "@mdx-js/mdx/source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "@mistralai/mistralai/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@protobufjs/fetch/@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], + "@reduxjs/toolkit/immer": ["immer@11.1.4", "", {}, "sha512-XREFCPo6ksxVzP4E0ekD5aMdf8WMwmdNaz6vuvxgI40UaEiu6q3p8X52aU6GdyvLY3XXX/8R7JOTXStz/nBbRw=="], "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], @@ -2019,12 +1985,6 @@ "@aws-crypto/util/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@2.2.0", "", { "dependencies": { "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA=="], - "@google/genai/protobufjs/@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], - - "@google/genai/protobufjs/@protobufjs/inquire": ["@protobufjs/inquire@1.1.1", "", {}, "sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew=="], - - "@google/genai/protobufjs/@protobufjs/utf8": ["@protobufjs/utf8@1.1.1", "", {}, "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg=="], - "@tanstack/router-plugin/chokidar/readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], "ansi-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], diff --git a/docs/adr/0001-keep-phoenix-observability-integration-out-of-core.md b/docs/adr/0001-keep-phoenix-observability-integration-out-of-core.md index 0558ac705..d049acceb 100644 --- a/docs/adr/0001-keep-phoenix-observability-integration-out-of-core.md +++ b/docs/adr/0001-keep-phoenix-observability-integration-out-of-core.md @@ -16,7 +16,15 @@ target or Dashboard Phoenix-runtime dependency. ## Context -AgentV exports evaluation traces through generic OpenTelemetry/OTLP plumbing and is adding a trace artifact contract for post-hoc trace evaluation. A focused follow-up proposed adding a Phoenix OTel backend preset for `--otel-backend phoenix`, but that raised a scope concern: Phoenix project routing, collector endpoint conventions, API keys, dataset concepts, and experiment behavior are backend-specific. +AgentV previously experimented with exporting evaluation traces through generic +OpenTelemetry/OTLP plumbing. A focused follow-up proposed adding a Phoenix OTel +backend preset, but that raised a scope concern: Phoenix project routing, +collector endpoint conventions, API keys, dataset concepts, and experiment +behavior are backend-specific. The later product boundary tightened this +further: AgentV should not synthesize OTLP from completed eval transcripts as +the primary observability path. The system under test, provider wrapper, or +runtime hook should emit OpenTelemetry/OpenInference spans directly when an +external trace backend is needed. AgentV's architecture principles prefer a lightweight core with extension points and adapters. Built-ins should be universal primitives that most users compose. Backend-specific observability integrations should not make AgentV core behave like a hosted trace or experiment platform. @@ -25,74 +33,45 @@ Relevant existing seams already point in this direction: - Provider and grader registries support narrow registration points. - `.agentv/providers/`, `.agentv/assertions/`, and `.agentv/graders/` use convention-based local discovery instead of a broad plugin host. - Earlier Phoenix adapter experiments kept Phoenix-specific behavior outside core and reported unsupported mappings explicitly. Those experiments are not the supported product path for AgentV completed runs or transcripts. -- The trace evaluation plan requires generic OTLP/OpenInference mapping without Phoenix-specific assumptions in core. +- Trace evaluation can still import or receive external OTLP/OpenInference-style + traces through a separate capability, but AgentV run bundles remain the + canonical eval artifact. ## Decision -Do not add direct Phoenix export or Phoenix-specific OTel backend preset logic to `packages/core`. +Do not add direct Phoenix export, Phoenix-specific OTel backend preset logic, or +AgentV transcript-to-OTLP export as a core eval-run path. AgentV core should own: -- generic OTLP/HTTP export configuration; -- OTLP JSON file export; - trace artifact types and boundary conversion; -- generic OTLP/OpenInference import/export mapping where it is backend-neutral; +- generic OTLP/OpenInference import or receive mapping where it is backend-neutral + and separate from eval-run export; - small registry/discovery primitives for extension points. -Phoenix integration should live outside core behind a narrow local adapter or -resolver boundary when needed. No maintained workspace package currently owns -that boundary. The first implementation does not need package loading or package -naming; a local resolver module is enough. Such a custom boundary may expose: +Phoenix integration should live outside core behind narrow correlation metadata +when needed. Such a boundary may expose: -- a Phoenix OTel backend resolver; -- Phoenix/OpenInference span-kind mapping; - link-out helpers for externally emitted trace/session correlation; -- explicit unsupported/lossy mapping reports. +- explicit unsupported/lossy mapping reports for imported or received traces. ## Minimal extension seam -Historical note: this ADR originally considered a first-class `--otel-backend phoenix` -ergonomics path. That must not be used to make Phoenix a Dashboard dependency or -an AgentV-owned artifact destination. Any future Phoenix work should be framed as -link-out correlation for externally emitted spans. - -A resolver should be approximately: - -```ts -export interface OtelBackendResolver { - readonly name: string; - resolve(context: { - env: Record; - cwd: string; - }): { - endpoint: string; - headers?: Record; - warnings?: string[]; - }; -} -``` - -Registration/discovery should remain boring and local-first. In this ADR, "plugin" should not imply a coding-agent plugin or package marketplace; this is only a backend resolver module seam: - -- support explicit TypeScript registration for programmatic callers; -- optionally discover Node-loadable `.agentv/otel-backends/*.mjs` or `*.js`, where the filename is the backend name; -- keep `execution.otel_backend: ` and `--otel-backend ` as the user-facing selectors; -- do not add package names, package auto-installation, a remote marketplace, trust prompts, or a general-purpose plugin host for this need. - -The earlier prototype exposed a resolver so users could opt in from project config -or a local `.agentv/otel-backends/phoenix.mjs` file. Treat that as a -custom/legacy path, not as the supported AgentV-to-Phoenix product boundary. +Historical note: this ADR originally considered first-class backend resolver +ergonomics for Phoenix. That path has been removed from AgentV's eval-run CLI and +project config surface. Any future Phoenix work should be framed as link-out +correlation for externally emitted spans or as separate trace import/evaluation, +not as AgentV-owned completed-run export. ## Migration path for Phoenix -1. Keep current generic OTLP configuration working: - - `OTEL_EXPORTER_OTLP_ENDPOINT` - - `OTEL_EXPORTER_OTLP_HEADERS` - - `--otel-file` for offline OTLP JSON export -2. Add a tiny backend resolver seam only if ergonomic backend names are needed. -3. Keep any custom Phoenix endpoint/header/project routing outside core and outside the supported AgentV artifact path. -4. Keep Phoenix out of Dashboard runtime fetch paths; use safe external links instead. -5. Consider moving existing vendor-specific core presets to the same resolver model later, but do not couple that cleanup to the Phoenix decision unless the implementation already touches the preset registry. +1. Remove AgentV eval-run OTLP export flags and project config fields. +2. Keep custom Phoenix endpoint/header/project routing in the system under test, + provider wrapper, or external instrumentation layer. +3. Keep Phoenix out of Dashboard runtime fetch paths; use safe external links + through `external_trace` metadata instead. +4. Treat import or evaluation of externally emitted OTLP/OpenInference traces as + separate from completed AgentV run export. ## Consequences @@ -100,14 +79,15 @@ Positive: - Keeps core aligned with AgentV's lightweight-core and composition principles. - Prevents Phoenix concepts from leaking into the generic trace model. -- Gives Phoenix users a link-out correlation path without blocking generic OTLP users. +- Gives Phoenix users a link-out correlation path without making AgentV an OTLP exporter. - Reuses AgentV's existing pattern of narrow registries and convention-based local discovery. Negative: -- Any maintained Phoenix OTel resolver must stay outside the zero-infra Dashboard path. -- Existing vendor presets in core remain an architectural inconsistency until migrated. -- Package-level resolver sharing may need a future decision if many backend adapters emerge. +- Users who want external trace inspection must instrument their system under + test, provider wrapper, or runtime hook directly. +- Trace import/evaluation remains a separate capability rather than an eval-run + export flag. ## Tracker impact @@ -116,6 +96,7 @@ Negative: ## Open questions -- Should the existing `langfuse`, `braintrust`, and `confident` core presets migrate to resolver modules in a follow-up cleanup? -- Should resolver loading stay limited to local Node-loadable `.agentv/otel-backends/*.mjs`/`*.js`, or should `agentv.config.ts` support direct resolver imports first? -- What exact Phoenix project-routing headers should the adapter emit across local Phoenix and hosted Phoenix variants? +- What exact metadata should each provider wrapper expose so AgentV can attach + safe `external_trace` correlation without leaking credentials? +- Which external OTLP/OpenInference trace import shapes should AgentV evaluate + first? diff --git a/docs/adr/0003-keep-opik-export-as-post-run-adapter-over-agentv-result-bundles.md b/docs/adr/0003-keep-opik-export-as-post-run-adapter-over-agentv-result-bundles.md index c2370b02b..dbfdac141 100644 --- a/docs/adr/0003-keep-opik-export-as-post-run-adapter-over-agentv-result-bundles.md +++ b/docs/adr/0003-keep-opik-export-as-post-run-adapter-over-agentv-result-bundles.md @@ -58,9 +58,9 @@ contract avoids that persisted trace sidecar surface. This is the right source for Opik trace/span projection. -### Current OTel export surface +### Removed OTel export surface -`packages/core/src/observability/otel-exporter.ts` is a runtime OTel emitter. It is useful for live export, but it is the wrong boundary for the AgentV-to-Opik completed-run exporter because: +`packages/core/src/observability/otel-exporter.ts` was a runtime OTel emitter. It has been removed from the eval-run path, and it was the wrong boundary for the AgentV-to-Opik completed-run exporter because: - it runs during execution rather than after a completed run; - it does not preserve AgentV grading/assertion artifacts as first-class export inputs; diff --git a/docs/adr/0008-normalized-transcript-artifact-contract.md b/docs/adr/0008-normalized-transcript-artifact-contract.md index b8711cc43..ae6bb69e5 100644 --- a/docs/adr/0008-normalized-transcript-artifact-contract.md +++ b/docs/adr/0008-normalized-transcript-artifact-contract.md @@ -108,9 +108,11 @@ advertise a public `trace.json`/`trace_path` sidecar; external observability systems can be correlated through `external_trace` link metadata when available. AgentV may derive additional event-oriented projections from `transcript.jsonl` -for Dashboard queries, tool-trajectory scoring, OpenTelemetry/OpenInference -mapping, or export adapters. Those projections are secondary indexes. They do -not replace `transcript.jsonl` as the portable transcript contract. +for Dashboard queries, tool-trajectory scoring, or trace import/evaluation +adapters. Those projections are secondary indexes. They do not replace +`transcript.jsonl` as the portable transcript contract, and AgentV does not use +completed transcript-to-OTLP conversion as the primary path for external +observability backends. ## Consequences @@ -145,9 +147,10 @@ not replace `transcript.jsonl` as the portable transcript contract. lists, files read/modified, errors, timing, and token rollups are derived metrics and belong in `metrics.json` or result metadata. - **Make OpenTelemetry/Phoenix spans the transcript contract.** Rejected. - OpenTelemetry/OpenInference mapping is valuable as an adapter/export - projection, but AgentV-owned run bundles remain the source of truth and do not - project completed transcripts into Phoenix. + OpenTelemetry/OpenInference mapping is valuable for imported or received + external traces, but AgentV-owned run bundles remain the source of truth and do + not project completed transcripts into Phoenix or synthesize OTLP as the + primary observability path. ## Non-Goals diff --git a/docs/plans/trace-envelope-implementation-spec.md b/docs/plans/trace-envelope-implementation-spec.md index 69b16260a..30a8a47ff 100644 --- a/docs/plans/trace-envelope-implementation-spec.md +++ b/docs/plans/trace-envelope-implementation-spec.md @@ -163,7 +163,6 @@ conversion_warnings: artifacts: trace_path: outputs/trace.json - otlp_path: outputs/trace.otlp.json answer_path: outputs/answer.md transcript_path: outputs/transcript.jsonl raw_evidence_dir: raw/ @@ -244,8 +243,8 @@ explicit known-field conversion plus Zod validation. It should not look like | Final answer | The final assistant answer is the last relevant LLM output plus an envelope artifact pointer. | `gen_ai.output.messages` opt-in on the final LLM/root span; OpenInference `output.value`/`output.mime_type`. | `artifacts.answer_path`; optional root event `agentv.final_answer` with `agentv.artifact_path`. | `outputs/answer.md` is the generated human-readable projection; spans remain the trace source of truth. | | Provider error | Set `status.code=ERROR` and message on the failed span and root span. Add exception event when details are available. | OTel exception event attributes `exception.type`, `exception.message`, `exception.stacktrace`; OpenInference reserves `exception.message`, `exception.stacktrace`, `exception.escaped`; span status `ERROR`. | `agentv.failure_stage`, `agentv.failure_reason_code`, envelope `conversion_warnings` when import is lossy rather than execution-failed. | A grader failure is score provenance, not provider execution error. Keep these separate. | | Subagent/nested tool evidence | Nested root-like span under the calling tool span, e.g. `execute_tool runSubagent` -> `invoke_agent `. Preserve imported parentage. | `gen_ai.operation.name=invoke_agent`; `gen_ai.agent.name`; `gen_ai.provider.name`; `openinference.span.kind=AGENT`; `session.id`. | `agentv.parent_tool_call_id`, `agentv.subagent=true` if needed for derived grader views. | VS Code/Copilot documents subagent invocations as nested `invoke_agent` spans under the parent `execute_tool runSubagent`; use that pattern when AgentV has enough evidence. | -| Score/evaluator provenance | Keep v1 score provenance in envelope `scores[]`. For OTLP export, also emit either root events or evaluator spans. | OTel GenAI evaluation event: `gen_ai.evaluation.name`, `gen_ai.evaluation.score.value`, `gen_ai.evaluation.score.label`, `gen_ai.evaluation.explanation`, `gen_ai.response.id`; OpenInference `openinference.span.kind=EVALUATOR`. | Existing `agentv.score` and `agentv.grader.*` root events remain compatibility output. Envelope `scores[].target_span_id`, `scores[].evidence.span_ids`, and `scores[].evidence.tool_call_ids` are the AgentV provenance contract. | Do not require `agentv.score` to import or score an external trace. Existing `inspect` OTLP import currently does; `.6` should remove that limitation for trace-only scoring. | -| Token usage | Put model token usage on the LLM span; aggregate usage can be repeated on root if useful for dashboards. | `gen_ai.usage.input_tokens`; `gen_ai.usage.output_tokens`; `gen_ai.usage.cache_read.input_tokens`; `gen_ai.usage.cache_creation.input_tokens`; `gen_ai.usage.reasoning.output_tokens`; OpenInference `llm.token_count.prompt`, `llm.token_count.completion`, `llm.token_count.total`, `llm.token_count.prompt_details.cache_read`, `llm.token_count.prompt_details.cache_write`, `llm.token_count.completion_details.reasoning`. | Envelope can carry no separate token summary; derive `TraceSummary`/timing artifacts from spans. | Use OTel GenAI attributes first because AgentV already exports them. Add OpenInference aliases only where an adapter/backend requires them. | +| Score/evaluator provenance | Keep v1 score provenance in envelope `scores[]`. Optional post-run adapters may project scores as root events or evaluator spans. | OTel GenAI evaluation event: `gen_ai.evaluation.name`, `gen_ai.evaluation.score.value`, `gen_ai.evaluation.score.label`, `gen_ai.evaluation.explanation`, `gen_ai.response.id`; OpenInference `openinference.span.kind=EVALUATOR`. | Existing `agentv.score` and `agentv.grader.*` root events are compatibility input when importing historical AgentV-shaped OTLP. Envelope `scores[].target_span_id`, `scores[].evidence.span_ids`, and `scores[].evidence.tool_call_ids` are the AgentV provenance contract. | Do not require `agentv.score` to import or score an external trace. | +| Token usage | Put model token usage on the LLM span; aggregate usage can be repeated on root if useful for dashboards. | `gen_ai.usage.input_tokens`; `gen_ai.usage.output_tokens`; `gen_ai.usage.cache_read.input_tokens`; `gen_ai.usage.cache_creation.input_tokens`; `gen_ai.usage.reasoning.output_tokens`; OpenInference `llm.token_count.prompt`, `llm.token_count.completion`, `llm.token_count.total`, `llm.token_count.prompt_details.cache_read`, `llm.token_count.prompt_details.cache_write`, `llm.token_count.completion_details.reasoning`. | Envelope can carry no separate token summary; derive `TraceSummary`/timing artifacts from spans. | Prefer OTel GenAI attributes for imported OTLP and add OpenInference aliases only where an adapter/backend requires them. | | Cost | Prefer OpenInference cost attributes when writing OpenInference-rich spans. Root aggregate cost may remain `agentv.trace.cost_usd` for compatibility. | OpenInference `llm.cost.prompt`, `llm.cost.completion`, `llm.cost.total`, `llm.cost.prompt_details.*`, `llm.cost.completion_details.*`. | `agentv.trace.cost_usd` compatibility attribute; result JSONL `cost_usd` remains derived/stable. | OTel GenAI cost attributes are not stable in current AgentV usage. Mark any new OTel cost key uncertain unless pinned to the current spec. | | Duration/timing | Span start/end times are canonical. Derived `duration_ms` comes from `end_time_unix_nano - start_time_unix_nano`. | OTLP span `start_time_unix_nano`, `end_time_unix_nano`; optional `gen_ai.response.time_to_first_chunk` for streaming LLM latency. | `agentv.duration_inferred=true` and conversion warning when timing was inferred from source order. | Do not store a separate authored duration as canonical when spans have times. | | Redaction/capture | Redaction is envelope policy plus omitted/filtered content attributes. | OTel GenAI docs mark message content attributes sensitive; VS Code/Copilot defaults content capture off and gates it via explicit settings/env. OpenInference supports privacy/masking concepts, but exact field-level masking keys should be pinned during implementation. | Envelope `capture.*`; span attributes `agentv.redaction.level`, `agentv.redaction.fields`, `agentv.content_ref` where useful. | Default should be metadata-only. Do not persist prompts, tool args/results, screenshots, or thinking blocks by default. | @@ -315,10 +314,9 @@ Minimal code slices: leave index JSONL unchanged unless a later discovery surface needs an additive index pointer. -6. OTLP import/export bridge. - Reuse `packages/core/src/observability/otel-exporter.ts` and - `otlp-json-file-exporter.ts` where possible, but move reusable span assembly - below the exporter so envelope writing and `--otel-file` do not drift. +6. OTLP import bridge. + Keep reusable span mapping in trace-envelope or trace-normalization modules so + envelope writing and external trace mapping do not drift. Sequencing: @@ -362,9 +360,8 @@ bun test packages/core/test/evaluation/trace-trajectory.test.ts \ packages/core/test/evaluation/replay-fixtures.test.ts \ packages/core/test/import/transcript-provider.test.ts -bun test packages/core/test/observability/otel-exporter.test.ts \ - packages/core/test/observability/file-exporters.test.ts \ - packages/core/test/observability/streaming-observer.test.ts +bun test packages/core/test/evaluation/trace-envelope.test.ts \ + packages/core/test/evaluation/trace-normalization.test.ts bun test apps/cli/test/commands/eval/artifact-writer.test.ts \ apps/cli/test/commands/eval/output-messages.test.ts @@ -411,7 +408,7 @@ Artifacts to inspect: - per-test `outputs/trace.json` - per-test `outputs/transcript.jsonl` - per-test `outputs/answer.md` -- generated OTLP JSON, if the implementation writes an OTLP sidecar +- external OTLP JSON import evidence, if the implementation imports an OTLP file - `examples/showcase/trace-evaluation/fixtures/replay-target-output.jsonl` remains unchanged unless a migration is explicitly accepted @@ -440,9 +437,10 @@ Recommended defaults are included so implementation is not blocked. that relationship from `Message.toolCalls`; preserve imported source parentage for external OTLP. -3. Should score provenance be root events or evaluator spans in OTLP export? - Recommended default: keep envelope `scores[]` authoritative; emit current - `agentv.grader.*` root events for compatibility; add +3. Should score provenance be root events or evaluator spans in a post-run + adapter? + Recommended default: keep envelope `scores[]` authoritative; preserve + `agentv.grader.*` root events only as historical import compatibility; add `gen_ai.evaluation.result` events or OpenInference `EVALUATOR` spans only when a consumer needs them. @@ -458,8 +456,6 @@ Local AgentV inputs read for this spec: - `docs/plans/trace-evaluation-architecture.md` - `docs/adr/0001-keep-phoenix-observability-integration-out-of-core.md` - `packages/core/src/evaluation/trace.ts` -- `packages/core/src/observability/otel-exporter.ts` -- `packages/core/src/observability/otlp-json-file-exporter.ts` - `packages/core/src/import/types.ts` - `packages/core/src/import/claude-parser.ts` - `packages/core/src/import/codex-parser.ts` diff --git a/docs/plans/trace-evaluation-architecture.md b/docs/plans/trace-evaluation-architecture.md index 072a1c6d5..f0834c246 100644 --- a/docs/plans/trace-evaluation-architecture.md +++ b/docs/plans/trace-evaluation-architecture.md @@ -26,11 +26,17 @@ or indexes into Phoenix. Phoenix is optional link-out correlation only when safe `external_trace` metadata points to spans already emitted independently by Codex, Arize, or another hook. +Update, 2026-07-03: AgentV's eval-run OTLP exporter surface has been removed. +Any references below to AgentV exporting OTLP are historical plan text. Current +trace work should treat OTLP/OpenInference as external import/receive formats or +post-run adapter formats, while systems under test and provider wrappers emit +their own telemetry directly. + --- ## Problem Frame -AgentV already captures tool calls in provider `Message.toolCalls`, persists compact `TraceSummary` data, exports AgentV OTLP spans, and has early post-hoc trace commands. That is enough for local AgentV result inspection, but it is not enough to evaluate production traces or third-party agent sessions with the same grader contract. +AgentV already captures tool calls in provider `Message.toolCalls`, persists compact `TraceSummary` data, and has early post-hoc trace commands. That is enough for local AgentV result inspection, but it is not enough to evaluate production traces or third-party agent sessions with the same grader contract. The best-practice direction is clear: larger players own trace stores, dashboards, datasets, and experiment UIs. AgentV's niche is the repo-local, declarative evaluation harness for coding agents and tool-using agents. To connect those worlds, AgentV needs a durable trace artifact contract between raw trace sources and graders. @@ -48,7 +54,7 @@ The best-practice direction is clear: larger players own trace stores, dashboard **Standards and Integrations** -- R6. AgentV OTLP export must continue to emit standards-aligned GenAI spans, especially `invoke_agent`, `chat`, and `execute_tool` operations. +- R6. External OTLP/OpenInference imports should preserve standards-aligned GenAI spans, especially `invoke_agent`, `chat`, and `execute_tool` operations. - R7. AgentV must map trace artifacts to and from OTLP/OpenInference-style traces without making Phoenix-specific assumptions in core. - R8. Phoenix integration, if present, must be link-out correlation for externally emitted traces; Phoenix must not become the AgentV trace, dataset, experiment, transcript, or index backend. - R9. Unsupported or lossy mappings must be explicit in conversion reports instead of silently approximated. @@ -56,7 +62,7 @@ The best-practice direction is clear: larger players own trace stores, dashboard **Post-Hoc Trace And Transcript Evaluation** - R10. Existing deterministic trace graders, including `tool-trajectory` and `execution-metrics`, must run against trace artifacts, not only live provider output messages or compact summaries. -- R11. Post-hoc evaluation must accept AgentV run artifacts, AgentV OTLP files, Langfuse/OTLP exports, imported coding-agent transcripts, Pi session JSONL, and compact transcript JSONL through source-specific adapters. +- R11. Post-hoc evaluation must accept AgentV run artifacts, external OTLP/OpenInference exports, imported coding-agent transcripts, Pi session JSONL, and compact transcript JSONL through source-specific adapters. - R12. Grader output must cite trace artifact evidence, such as matched tool call IDs, positions, timing, or source event IDs. - R13. Trace evaluation must preserve current lightweight result output by deriving compact summaries from trace artifacts. @@ -86,7 +92,7 @@ The best-practice direction is clear: larger players own trace stores, dashboard ## Key Technical Decisions - **Start from realistic characterization evals:** The first implementation phase should collect a small set of real trace fixtures and write evals that answer useful agent-quality questions. The trace artifact contract should be pressure-tested by those evals before broad schema or adapter work expands. -- **Normalize first, grade second:** Graders should consume AgentV's trace artifact contract. Importers translate raw sources into the contract; exporters translate the contract into backend-neutral OTLP/OpenInference shapes. This avoids coupling graders to Pi, VS Code, or provider-specific logs. +- **Normalize first, grade second:** Graders should consume AgentV's trace artifact contract. Importers translate raw sources into the contract; optional post-run adapters translate the contract into backend-neutral shapes when needed. This avoids coupling graders to Pi, VS Code, or provider-specific logs. - **OTel is an interchange layer, not the canonical model:** VS Code and industry tooling make OTLP/HTTP and GenAI span semantics important, but entireio-style logs and Pi sessions prove valuable traces are often transcript or lifecycle JSON. AgentV should support OTel strongly without making it mandatory. - **Tool sequence grading is turn-centric, not span-centric:** The trace artifact should model sessions, turns, messages, tool calls, tool results, and selected branches as a projection over the canonical trace artifact. - **Coding-agent transcripts are trace sources:** `agentv import claude`, `agentv import codex`, `agentv import copilot`, and `agentv eval --transcript` already establish transcript import as offline grading infrastructure. The architecture should extend that path into trace artifact normalization instead of creating a separate trace-only mechanism. @@ -108,7 +114,7 @@ AgentV should introduce a trace artifact layer between raw sources and evaluatio ```mermaid flowchart TB A[AgentV run output] --> N[Trace artifact] - B[OTLP / OpenInference export] --> N + B[External OTLP / OpenInference export] --> N D[Pi session JSONL] --> N E[Compact transcript JSONL] --> N F[Imported coding-agent transcript] --> N @@ -116,7 +122,7 @@ flowchart TB N --> S[Trace summary derivation] N --> G[Trace graders] N --> Q[Transcript replay provider] - N --> O[OTLP / OpenInference export] + N --> O[Optional post-run adapter] S --> R[AgentV result artifacts] G --> R @@ -201,12 +207,12 @@ The exact schema belongs in implementation, but these concepts should be stable: - **Test Scenarios:** Cover assistant tool calls with IDs, calls without IDs, tool outputs, token usage, model metadata, errors, missing timing, and no-tool-call runs. - **Verification:** Existing eval result baselines should continue to include compact trace summaries while full trace artifacts are available where configured. -### U3. OTLP and OpenInference Import/Export Mapping +### U3. OTLP and OpenInference Import Mapping -- **Goal:** Map trace artifacts to and from OTLP JSON/HTTP-compatible spans using GenAI and OpenInference-compatible semantics where available. -- **Files:** `packages/core/src/observability/otel-exporter.ts`, `packages/core/src/observability/otlp-json-file-exporter.ts`, `apps/cli/src/commands/inspect/utils.ts`, tests under `packages/core/test/observability/` and `apps/cli/test/`. +- **Goal:** Map external OTLP/OpenInference JSON spans into trace artifacts using GenAI and OpenInference-compatible semantics where available. +- **Files:** `apps/cli/src/commands/inspect/utils.ts`, trace normalization utilities under `packages/core/src/evaluation/`, and focused trace CLI tests. - **Patterns:** Continue human-readable span names (`invoke_agent `, `chat `, `execute_tool `) plus machine-stable attributes. Use `gen_ai.operation.name`, `gen_ai.tool.name`, `gen_ai.tool.call.id`, token usage attributes, and `agentv.*` only where standards do not cover the concept. -- **Test Scenarios:** Import an AgentV OTLP file into a trace artifact, export it back, and verify tool call order, call IDs, token usage, durations, redaction state, and grader score events survive when representable. +- **Test Scenarios:** Import an external OTLP file into a trace artifact and verify tool call order, call IDs, token usage, durations, redaction state, and grader evidence survive when representable. - **Verification:** `agentv trace show` and `agentv trace score` should work from OTLP artifacts without requiring an `agentv.score` attribute for trace-only evaluation. ### U4. Phoenix Read-Only Correlation Path @@ -275,10 +281,10 @@ The exact schema belongs in implementation, but these concepts should be stable: ### U8. CLI and Artifact Workflow -- **Goal:** Make trace import, inspection, scoring, and export usable from the CLI without forcing users into one backend. +- **Goal:** Make trace import, inspection, and scoring usable from the CLI without forcing users into one backend. - **Files:** `apps/cli/src/commands/inspect/`, `apps/cli/src/commands/eval/`, result layout files under `apps/cli/src/commands/eval/`, docs/examples as needed. - **Patterns:** Extend existing `agentv trace show`, `trace stats`, and `trace score` commands rather than creating parallel command families. -- **Test Scenarios:** CLI should accept run workspaces, `index.jsonl`, AgentV OTLP JSON, generic OTLP JSON, imported transcript JSONL, Pi JSONL, and compact transcript JSONL. It should report source kind, conversion warnings, and grader results. +- **Test Scenarios:** CLI should accept run workspaces, `index.jsonl`, external OTLP JSON, imported transcript JSONL, Pi JSONL, and compact transcript JSONL. It should report source kind, conversion warnings, and grader results. - **Verification:** Functional CLI tests should use `bun apps/cli/src/cli.ts ...`; core changes require `bun run build` before CLI testing. ### U9. Documentation and Best-Practice Recipes @@ -287,7 +293,7 @@ The exact schema belongs in implementation, but these concepts should be stable: - **Files:** `apps/web/src/content/docs/` if public docs exist for this area, package READMEs, boundary ADRs, showcase material under `examples/showcase/trace-evaluation/`. - **Patterns:** Document patterns instead of adding core primitives when composition is enough. - **Test Scenarios:** Validate example eval YAML and include at least one trace-import fixture for post-hoc scoring. -- **Verification:** Docs should show recipes for local trace scoring, read-only Phoenix correlation boundaries, Pi session scoring, and OTLP export. +- **Verification:** Docs should show recipes for local trace scoring, read-only Phoenix correlation boundaries, Pi session scoring, and external OTLP import. --- @@ -334,7 +340,7 @@ Outside this product's identity: - **Core data model:** `TraceSummary` becomes a derived compact view of trace artifacts, not the highest-fidelity trace contract. - **Graders:** Trace-aware graders move from output-message parsing toward trace artifact parsing while keeping compatibility with current eval results. - **CLI:** Post-hoc trace commands become a real evaluation path, not just inspection of AgentV-generated artifacts. -- **Observability:** OTel export remains standards-aligned and becomes round-trippable enough for offline scoring. +- **Observability:** External OTel/OpenInference imports remain standards-aligned enough for offline scoring. - **Adapters:** Pi and transcript importers stay outside primitive grader logic, preserving AgentV's lightweight core. Phoenix remains read-only external correlation when safe `external_trace` metadata exists. - **Privacy:** Redaction and content capture become an explicit boundary concern across import, persistence, export, and grading. @@ -355,12 +361,11 @@ Outside this product's identity: - `packages/core/src/evaluation/trace.ts` defines current `TraceSummary`, tool trajectory config, and execution metric helpers. - `packages/core/src/evaluation/providers/types.ts` defines provider `Message` and `ToolCall`, the current richest AgentV trace source. - `packages/core/src/evaluation/graders/tool-trajectory.ts` currently grades from output messages first, then compact trace summaries. -- `apps/cli/src/commands/inspect/utils.ts` currently imports AgentV OTLP JSON into raw results but mostly reconstructs summaries and requires AgentV result metadata for full support. +- `apps/cli/src/commands/inspect/utils.ts` imports external OTLP JSON into trace read models for inspection and scoring. - `apps/cli/src/commands/import/` already exposes `claude`, `codex`, and `copilot` transcript import subcommands for offline grading. - `packages/core/src/import/transcript-provider.ts` and `packages/core/src/import/types.ts` define the current transcript replay path used by `agentv eval --transcript`. - `packages/core/src/evaluation/cache/response-cache.ts` implements the existing opt-in response cache and safety behavior. - `packages/core/src/evaluation/config.ts` exposes TS config `cache.enabled` and `cache.path`, while `apps/cli/src/commands/eval/run-eval.ts` currently uses CLI flags and YAML cache config when constructing `ResponseCache`. -- `packages/core/src/observability/otel-exporter.ts` already emits `agentv.eval`, `chat`, and `execute_tool` spans with GenAI attributes and optional content capture. - Microsoft VS Code at `ebb335fad028ca0f3582e822d14a1bb9109725e7` uses OTLP/HTTP, optional local JSONL/SQLite persistence, `invoke_agent`, `chat`, and `execute_tool` spans, W3C trace context propagation, and privacy-gated content capture. - entireio/cli at `b98014a60b474ddf139a91231cdc4640eac62e5f` does not expose agent traces through OTel; its useful trace sources are compact transcript JSONL and lifecycle events with tool-use/file metadata. - Pi public session format and `badlogicgames/pi-mono` show branchable JSONL sessions with session headers, message entries, embedded assistant tool calls, separate tool results, model metadata, token usage, and optional inline images/thinking blocks. diff --git a/examples/features/README.md b/examples/features/README.md index d01a7e2eb..23276af5a 100644 --- a/examples/features/README.md +++ b/examples/features/README.md @@ -110,10 +110,9 @@ Focused examples for specific AgentV capabilities. Find your use case below, the --- -### Export results to an observability platform +### Extract structured data | Example | Description | |---------|-------------| -| [langfuse-export](langfuse-export/) | Export eval traces to Langfuse via OpenTelemetry OTLP/HTTP | | [document-extraction](document-extraction/) | Evaluate structured data extracted from documents | --- @@ -147,7 +146,7 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [copilot-log-eval](copilot-log-eval/) | Offline evaluation | | [default-graders](default-graders/) | Getting started | | [deterministic-graders](deterministic-graders/) | Deterministic assertions | -| [document-extraction](document-extraction/) | Observability & export | +| [document-extraction](document-extraction/) | Document extraction | | [env-interpolation](env-interpolation/) | Dataset & input | | [eval-assert-demo](eval-assert-demo/) | Custom graders | | [execution-metrics](execution-metrics/) | Cost, latency & tokens | @@ -156,7 +155,6 @@ Focused examples for specific AgentV capabilities. Find your use case below, the | [file-changes-graders](file-changes-graders/) | Workspace & targets | | [functional-grading](functional-grading/) | Custom graders | | [input-files-shorthand](input-files-shorthand/) | Dataset & input | -| [langfuse-export](langfuse-export/) | Observability & export | | [latency-assertions](latency-assertions/) | Tool & agent evaluation | | [local-cli](local-cli/) | Workspace & targets | | [multi-turn-conversation](multi-turn-conversation/) | LLM grading | diff --git a/examples/features/langfuse-export/.agentv/config.yaml b/examples/features/langfuse-export/.agentv/config.yaml deleted file mode 100644 index d8744b19b..000000000 --- a/examples/features/langfuse-export/.agentv/config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -$schema: agentv-config-v2 - -# Langfuse OTel Export Configuration -# -# Credentials are loaded from .env (LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY). -# No CLI flags needed — this config enables export automatically. - -execution: - export_otel: true - otel_backend: langfuse - otel_group_turns: true - otel_file: .agentv/results/otel-{timestamp}.json diff --git a/examples/features/langfuse-export/.env.example b/examples/features/langfuse-export/.env.example deleted file mode 100644 index 7cfd3a4db..000000000 --- a/examples/features/langfuse-export/.env.example +++ /dev/null @@ -1,6 +0,0 @@ -# Langfuse credentials — get these from your Langfuse project settings -LANGFUSE_PUBLIC_KEY=pk-lf-... -LANGFUSE_SECRET_KEY=sk-lf-... - -# Optional: self-hosted Langfuse (defaults to cloud.langfuse.com) -# LANGFUSE_HOST=https://your-langfuse-instance.com diff --git a/examples/features/langfuse-export/README.md b/examples/features/langfuse-export/README.md deleted file mode 100644 index f0b8d8d2e..000000000 --- a/examples/features/langfuse-export/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Langfuse OTel Export - -Demonstrates exporting eval traces to [Langfuse](https://langfuse.com) via OpenTelemetry. - -AgentV uses OTLP/HTTP — no Langfuse SDK required. The `langfuse` backend resolver handles endpoint and auth automatically. - -## Setup - -1. Copy `.env.example` to `.env` and fill in your Langfuse credentials: - -```bash -cp .env.example .env -``` - -2. Run evals — traces appear in your Langfuse dashboard automatically: - -```bash -agentv eval examples/features/langfuse-export/evals/suite.yaml -``` - -No `--export-otel` or `--otel-backend` flags needed — the config.yaml handles it. - -## How It Works - -- `.agentv/config.yaml` declares `export_otel: true` and `otel_backend: langfuse` -- AgentV constructs Basic Auth from `LANGFUSE_PUBLIC_KEY:LANGFUSE_SECRET_KEY` -- Spans are sent to `https://cloud.langfuse.com/api/public/otel/v1/traces` (or your self-hosted instance via `LANGFUSE_HOST`) -- Uses GenAI semantic conventions (`gen_ai.*`) that Langfuse dashboards recognize - -## Self-Hosted Langfuse - -Add `LANGFUSE_HOST` to your `.env`: - -```bash -LANGFUSE_HOST=https://your-langfuse-instance.com -``` - -## CLI Override - -Config.yaml defaults can always be overridden by CLI flags: - -```bash -# Disable OTel export for a single run -agentv eval evals/suite.yaml # export_otel in config still applies - -# Use a different backend -agentv eval evals/suite.yaml --export-otel --otel-backend braintrust -``` diff --git a/examples/features/langfuse-export/evals/suite.yaml b/examples/features/langfuse-export/evals/suite.yaml deleted file mode 100644 index 16bd6e948..000000000 --- a/examples/features/langfuse-export/evals/suite.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Langfuse Export Example Eval -# -# Run: agentv eval examples/features/langfuse-export/evals/suite.yaml -# -# Traces will appear in your Langfuse dashboard with gen_ai.* semantic -# conventions, per-span token usage, and turn-level grouping. - -description: Sample eval with Langfuse trace export - -target: default - -tests: - - id: greeting - input: "Hello, who are you?" - assert: - - "Responds with a self-introduction" - - - id: math - input: "What is 15 * 23?" - assert: - - "Provides the correct answer: 345" - - - id: multi-turn - input: - - role: user - content: "Remember the number 42." - - role: assistant - content: "Got it, I'll remember 42." - - role: user - content: "What number did I ask you to remember?" - assert: - - "Correctly recalls the number 42" diff --git a/examples/features/trace-analysis/README.md b/examples/features/trace-analysis/README.md index ef3b71cc4..928e24c3c 100644 --- a/examples/features/trace-analysis/README.md +++ b/examples/features/trace-analysis/README.md @@ -11,7 +11,7 @@ bun agentv trace list # Show summary trace details from the run manifest bun agentv trace show .agentv/results/default//index.jsonl -# Show hierarchical trace tree from an OTLP export +# Show hierarchical trace tree from an external OTLP export bun agentv trace show traces/eval.otlp.json --tree # Filter to a specific test @@ -30,8 +30,8 @@ bun agentv trace stats .agentv/results/default//index.jsonl --format ## What's in the Example Data The sample run workspace contains 5 test results from a multi-agent evaluation. `trace` accepts the -workspace directory directly for summary fallback. For full tool-call inspection, export OTLP JSON -and inspect that file instead. +workspace directory directly for summary fallback. For full external span inspection, instrument the +system under test or provider to emit OTLP JSON and inspect that file instead. | Test ID | Score | Target | Trace | |---|---|---|---| diff --git a/packages/core/package.json b/packages/core/package.json index 6e733ff2b..9b577cf59 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -51,14 +51,6 @@ "yaml": "^2.8.3", "zod": "^3.23.8" }, - "optionalDependencies": { - "@opentelemetry/api": "^1.9.0", - "@opentelemetry/core": "^2.5.1", - "@opentelemetry/exporter-trace-otlp-http": "^0.212.0", - "@opentelemetry/resources": "^2.5.1", - "@opentelemetry/sdk-trace-node": "^2.5.1", - "@opentelemetry/semantic-conventions": "^1.39.0" - }, "peerDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.88", "@earendil-works/pi-coding-agent": "^0.74.0" diff --git a/packages/core/src/evaluation/config.ts b/packages/core/src/evaluation/config.ts index 3e7ceef79..34ebbe1cb 100644 --- a/packages/core/src/evaluation/config.ts +++ b/packages/core/src/evaluation/config.ts @@ -26,27 +26,44 @@ import { z } from 'zod'; +const ExecutionConfigSchema = z + .object({ + /** Number of parallel workers (default: 3) */ + workers: z.number().int().min(1).max(50).optional(), + /** Maximum retries on failure (default: 2) */ + maxRetries: z.number().int().min(0).optional(), + /** Agent timeout in milliseconds. No timeout if not set. */ + agentTimeoutMs: z.number().int().min(0).optional(), + /** Enable verbose logging */ + verbose: z.boolean().optional(), + /** Always keep temp workspaces after eval */ + keepWorkspaces: z.boolean().optional(), + }) + .passthrough() + .superRefine((value, ctx) => { + if ('otelFile' in value) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['otelFile'], + message: + 'execution.otelFile has been removed. Emit OpenTelemetry/OpenInference traces from the system under test or provider and correlate AgentV run artifacts with external_trace metadata.', + }); + } + }) + .transform(({ workers, maxRetries, agentTimeoutMs, verbose, keepWorkspaces }) => ({ + ...(workers !== undefined && { workers }), + ...(maxRetries !== undefined && { maxRetries }), + ...(agentTimeoutMs !== undefined && { agentTimeoutMs }), + ...(verbose !== undefined && { verbose }), + ...(keepWorkspaces !== undefined && { keepWorkspaces }), + })); + /** * Schema for AgentV project-level configuration. */ const AgentVConfigSchema = z.object({ /** Default execution settings */ - execution: z - .object({ - /** Number of parallel workers (default: 3) */ - workers: z.number().int().min(1).max(50).optional(), - /** Maximum retries on failure (default: 2) */ - maxRetries: z.number().int().min(0).optional(), - /** Agent timeout in milliseconds. No timeout if not set. */ - agentTimeoutMs: z.number().int().min(0).optional(), - /** Enable verbose logging */ - verbose: z.boolean().optional(), - /** Always keep temp workspaces after eval */ - keepWorkspaces: z.boolean().optional(), - /** Write OTLP JSON trace to this path (supports {timestamp} placeholder) */ - otelFile: z.string().optional(), - }) - .optional(), + execution: ExecutionConfigSchema.optional(), /** Output settings */ output: z diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 6e4afdc83..c7d9f2538 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -41,11 +41,6 @@ export type ExecutionDefaults = { readonly verbose?: boolean; readonly keep_workspaces?: boolean; readonly workspace_path?: string; - readonly otel_file?: string; - readonly export_otel?: boolean; - readonly otel_backend?: string; - readonly otel_capture_content?: boolean; - readonly otel_group_turns?: boolean; }; export type ResultPushConflictPolicy = 'block'; @@ -213,7 +208,11 @@ function parseConfigObject( ...(tags && { tags }), }; } catch (error) { - logWarning(`Could not parse AgentV config at ${configPath}: ${(error as Error).message}`); + const message = (error as Error).message; + if (message.includes('execution.otel_') || message.includes('execution.export_otel')) { + throw new Error(`Invalid AgentV config at ${configPath}: ${message}`); + } + logWarning(`Could not parse AgentV config at ${configPath}: ${message}`); return null; } } @@ -720,37 +719,7 @@ export function parseExecutionDefaults( logWarning(`Invalid execution.workspace_path in ${configPath}, expected non-empty string`); } - const otelFile = obj.otel_file; - if (typeof otelFile === 'string' && otelFile.trim().length > 0) { - result.otel_file = otelFile.trim(); - } else if (otelFile !== undefined) { - logWarning(`Invalid execution.otel_file in ${configPath}, expected non-empty string`); - } - - if (typeof obj.export_otel === 'boolean') { - result.export_otel = obj.export_otel; - } else if (obj.export_otel !== undefined) { - logWarning(`Invalid execution.export_otel in ${configPath}, expected boolean`); - } - - const otelBackend = obj.otel_backend; - if (typeof otelBackend === 'string' && otelBackend.trim().length > 0) { - result.otel_backend = otelBackend.trim(); - } else if (otelBackend !== undefined) { - logWarning(`Invalid execution.otel_backend in ${configPath}, expected non-empty string`); - } - - if (typeof obj.otel_capture_content === 'boolean') { - result.otel_capture_content = obj.otel_capture_content; - } else if (obj.otel_capture_content !== undefined) { - logWarning(`Invalid execution.otel_capture_content in ${configPath}, expected boolean`); - } - - if (typeof obj.otel_group_turns === 'boolean') { - result.otel_group_turns = obj.otel_group_turns; - } else if (obj.otel_group_turns !== undefined) { - logWarning(`Invalid execution.otel_group_turns in ${configPath}, expected boolean`); - } + rejectRemovedOtelExecutionDefaults(obj, configPath); if (obj.pool_workspaces !== undefined) { logWarning( @@ -767,6 +736,29 @@ export function parseExecutionDefaults( return Object.keys(result).length > 0 ? (result as ExecutionDefaults) : undefined; } +function rejectRemovedOtelExecutionDefaults( + obj: Record, + configPath: string, +): void { + const removedFields = [ + 'otel_file', + 'export_otel', + 'otel_backend', + 'otel_capture_content', + 'otel_group_turns', + ].filter((field) => obj[field] !== undefined); + if (removedFields.length === 0) { + return; + } + throw new Error( + `${removedFields + .map((field) => `execution.${field}`) + .join( + ', ', + )} in ${configPath} ${removedFields.length === 1 ? 'has' : 'have'} been removed. The system under test or provider should emit OpenTelemetry/OpenInference traces directly; use AgentV run artifacts and external_trace metadata for correlation.`, + ); +} + export function parseResultsConfig(raw: unknown, configPath: string): ResultsConfig | undefined { if (raw === undefined || raw === null) { return undefined; diff --git a/packages/core/src/evaluation/trace-envelope.ts b/packages/core/src/evaluation/trace-envelope.ts index f5e0508b5..00568d608 100644 --- a/packages/core/src/evaluation/trace-envelope.ts +++ b/packages/core/src/evaluation/trace-envelope.ts @@ -10,11 +10,11 @@ * * Derived views such as Provider `Message[]`, `transcript.json`, * `TraceSummary`, compact tool trajectories, replay provider responses, and - * OTLP JSON export bodies must project from this artifact. Transcript - * projections use AgentV transcript events on the root span so compatibility - * rows can include input/system turns without changing replay's assistant-only view. - * Do not introduce a second canonical graph for those compatibility/read - * models. + * external OTLP/OpenInference mappings must project from this artifact. + * Transcript projections use AgentV transcript events on the root span so + * compatibility rows can include input/system turns without changing replay's + * assistant-only view. Do not introduce a second canonical graph for those + * compatibility/read models. * * To extend the wire shape, add snake_case fields to the focused Zod schema, * convert them explicitly in the matching to/from helper, and keep opaque maps diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d594036f0..80b7ab2a9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -201,7 +201,6 @@ export { export { syncProject, syncProjects } from './project-sync.js'; export { trimBaselineResult } from './evaluation/baseline.js'; export { DEFAULT_CATEGORY, deriveCategory, normalizeCategoryPath } from './evaluation/category.js'; -export * from './observability/index.js'; // Registry exports export { diff --git a/packages/core/src/observability/index.ts b/packages/core/src/observability/index.ts deleted file mode 100644 index dbeaf6db7..000000000 --- a/packages/core/src/observability/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type { - OtelBackendResolution, - OtelBackendResolver, - OtelBackendResolverContext, - OtelExportOptions, -} from './types.js'; -export { OtelTraceExporter, OtelStreamingObserver } from './otel-exporter.js'; -export { OtlpJsonFileExporter } from './otlp-json-file-exporter.js'; diff --git a/packages/core/src/observability/otel-exporter.ts b/packages/core/src/observability/otel-exporter.ts deleted file mode 100644 index 5a7fa1cad..000000000 --- a/packages/core/src/observability/otel-exporter.ts +++ /dev/null @@ -1,630 +0,0 @@ -import type { - Message, - ProviderStreamCallbacks, - ProviderTokenUsage, -} from '../evaluation/providers/types.js'; -import type { EvaluationResult } from '../evaluation/types.js'; -import type { OtelExportOptions } from './types.js'; - -export type { OtelExportOptions }; - -// --------------------------------------------------------------------------- -// OTel type aliases (resolved dynamically at init) -// --------------------------------------------------------------------------- - -// biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically -type OtelApi = any; -// biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically -type NodeTracerProvider = any; -// biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically -type Tracer = any; - -// --------------------------------------------------------------------------- -// Exporter -// --------------------------------------------------------------------------- - -export class OtelTraceExporter { - private provider: NodeTracerProvider | null = null; - private tracer: Tracer | null = null; - private api: OtelApi | null = null; - // biome-ignore lint/suspicious/noExplicitAny: OTel types loaded dynamically - private W3CPropagator: any = null; - - constructor(private readonly options: OtelExportOptions) {} - - /** Initialize the OTel SDK. Returns false if OTel packages are not available. */ - async init(): Promise { - try { - const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([ - import('@opentelemetry/sdk-trace-node'), - import('@opentelemetry/resources'), - import('@opentelemetry/semantic-conventions'), - import('@opentelemetry/api'), - import('@opentelemetry/core').catch(() => null), - ]); - - const { NodeTracerProvider: Provider, SimpleSpanProcessor } = sdkTraceNode; - const { resourceFromAttributes } = resourcesMod; - const { ATTR_SERVICE_NAME } = semconvMod; - - const resourceAttributes = { - [ATTR_SERVICE_NAME]: this.options.serviceName ?? 'agentv', - ...this.options.resourceAttributes, - }; - const resource = resourceFromAttributes(resourceAttributes); - - // biome-ignore lint/suspicious/noExplicitAny: OTel processor types loaded dynamically - const processors: any[] = []; - - // Remote OTLP exporter (only when endpoint is configured) - if (this.options.endpoint) { - const otlpHttp = await import('@opentelemetry/exporter-trace-otlp-http'); - const { OTLPTraceExporter } = otlpHttp; - const exporter = new OTLPTraceExporter({ - url: this.options.endpoint, - headers: this.options.headers, - }); - processors.push(new SimpleSpanProcessor(exporter)); - } - - // OTLP JSON file exporter - if (this.options.otlpFilePath) { - const { OtlpJsonFileExporter } = await import('./otlp-json-file-exporter.js'); - processors.push( - new SimpleSpanProcessor( - new OtlpJsonFileExporter(this.options.otlpFilePath, resourceAttributes), - ), - ); - } - - if (processors.length === 0) { - return false; - } - - this.provider = new Provider({ - resource, - spanProcessors: processors, - }); - this.provider.register(); - this.api = api; - this.tracer = api.trace.getTracer('agentv', '1.0.0'); - this.W3CPropagator = coreMod?.W3CTraceContextPropagator ?? null; - return true; - } catch { - return false; - } - } - - /** Export a single evaluation result as an OTel trace. */ - async exportResult(result: EvaluationResult): Promise { - if (!this.tracer || !this.api) return; - - const api = this.api; - const tracer = this.tracer; - const captureContent = this.options.captureContent ?? false; - - // Determine timing - const startHr = toHrTime(result.startTime ?? result.timestamp); - const endHr = toHrTime(result.endTime ?? result.timestamp); - - // Support trace composition via W3C traceparent propagation - let parentCtx = api.ROOT_CONTEXT; - const traceparent = process.env.TRACEPARENT; - if (traceparent && this.W3CPropagator) { - try { - const propagator = new this.W3CPropagator(); - parentCtx = propagator.extract( - api.ROOT_CONTEXT, - { traceparent, tracestate: process.env.TRACESTATE ?? '' }, - { - get: (carrier: Record, key: string) => carrier[key], - keys: (carrier: Record) => Object.keys(carrier), - }, - ); - } catch { - // Malformed TRACEPARENT — fall back to standalone trace - } - } - - tracer.startActiveSpan( - 'agentv.eval', - { startTime: startHr }, - parentCtx, - (rootSpan: { - setAttribute: (...args: unknown[]) => void; - addEvent: (...args: unknown[]) => void; - setStatus: (...args: unknown[]) => void; - end: (...args: unknown[]) => void; - }) => { - // GenAI semantic convention attributes - rootSpan.setAttribute('gen_ai.operation.name', 'evaluate'); - rootSpan.setAttribute('gen_ai.system', 'agentv'); - - // Core attributes - rootSpan.setAttribute('agentv.test_id', result.testId); - rootSpan.setAttribute('agentv.target', result.target); - if (result.suite) rootSpan.setAttribute('agentv.suite', result.suite); - rootSpan.setAttribute('agentv.score', result.score); - if (captureContent && result.output.length > 0) { - rootSpan.setAttribute('agentv.output_text', result.output); - } - - // Flat execution metrics - if (result.durationMs != null) - rootSpan.setAttribute('agentv.trace.duration_ms', result.durationMs); - if (result.costUsd != null) rootSpan.setAttribute('agentv.trace.cost_usd', result.costUsd); - if (result.tokenUsage) { - if (result.tokenUsage.input != null) { - rootSpan.setAttribute('agentv.trace.token_input', result.tokenUsage.input); - } - if (result.tokenUsage.output != null) { - rootSpan.setAttribute('agentv.trace.token_output', result.tokenUsage.output); - } - if (result.tokenUsage.cached != null) { - rootSpan.setAttribute('agentv.trace.token_cached', result.tokenUsage.cached); - } - } - - // Trace summary attributes (tool-specific) - if (result.trace) { - const t = result.trace; - rootSpan.setAttribute('agentv.trace.event_count', t.eventCount); - rootSpan.setAttribute( - 'agentv.trace.tool_names', - Object.keys(t.toolCalls).sort().join(','), - ); - if (t.llmCallCount != null) - rootSpan.setAttribute('agentv.trace.llm_call_count', t.llmCallCount); - } - - // Child spans from result-local trace messages. The execution trace - // sidecar owns the canonical span graph for export/import work. - // Some callers may still export older result artifacts while migrating, - // so tolerate a missing trace instead of crashing the exporter. - const traceMessages = result.trace?.messages ?? []; - if (traceMessages.length > 0) { - const parentCtx = api.trace.setSpan(api.context.active(), rootSpan); - - if (this.options.groupTurns) { - const turns = groupMessagesIntoTurns(traceMessages); - if (turns.length > 1) { - for (const [i, turn] of turns.entries()) { - api.context.with(parentCtx, () => { - tracer.startActiveSpan( - `agentv.turn.${i + 1}`, - {}, - (turnSpan: { - end: (...args: unknown[]) => void; - }) => { - const turnCtx = api.trace.setSpan(api.context.active(), turnSpan); - for (const msg of turn.messages) { - this.exportMessage(tracer, api, turnCtx, msg, captureContent); - } - turnSpan.end(); - }, - ); - }); - } - } else { - for (const msg of traceMessages) { - this.exportMessage(tracer, api, parentCtx, msg, captureContent); - } - } - } else { - for (const msg of traceMessages) { - this.exportMessage(tracer, api, parentCtx, msg, captureContent); - } - } - } - - // Grader scores as span events - if (result.scores) { - for (const score of result.scores) { - rootSpan.addEvent(`agentv.grader.${score.name}`, { - 'agentv.grader.score': score.score, - 'agentv.grader.type': score.type, - ...(score.verdict ? { 'agentv.grader.verdict': score.verdict } : {}), - }); - } - } - - // Status - if (result.error) { - rootSpan.setStatus({ code: api.SpanStatusCode.ERROR, message: result.error }); - } else { - rootSpan.setStatus({ code: api.SpanStatusCode.OK }); - } - - rootSpan.end(endHr); - }, - ); - } - - /** Flush pending spans and shut down. */ - async shutdown(): Promise { - await this.provider?.shutdown(); - } - - /** Create a streaming observer for real-time span export */ - createStreamingObserver(): OtelStreamingObserver | null { - if (!this.tracer || !this.api) return null; - // Extract TRACEPARENT for trace composition - let parentCtx: unknown; - const traceparent = process.env.TRACEPARENT; - if (traceparent && this.W3CPropagator) { - try { - const propagator = new this.W3CPropagator(); - parentCtx = propagator.extract( - this.api.ROOT_CONTEXT, - { traceparent, tracestate: process.env.TRACESTATE ?? '' }, - { - get: (carrier: Record, key: string) => carrier[key], - keys: (carrier: Record) => Object.keys(carrier), - }, - ); - } catch { - // Malformed TRACEPARENT — ignore - } - } - return new OtelStreamingObserver( - this.tracer, - this.api, - this.options.captureContent ?? false, - parentCtx, - ); - } - - // ----------------------------------------------------------------------- - // Private helpers - // ----------------------------------------------------------------------- - - private exportMessage( - tracer: Tracer, - api: OtelApi, - parentCtx: unknown, - msg: Message, - captureContent: boolean, - ): void { - const isAssistant = msg.role === 'assistant'; - const model = msg.metadata?.model ? String(msg.metadata.model) : undefined; - const spanName = isAssistant ? `chat ${model ?? 'unknown'}` : `gen_ai.message.${msg.role}`; - - const startHr = toHrTime(msg.startTime); - const endHr = toHrTime(msg.endTime); - - api.context.with(parentCtx, () => { - tracer.startActiveSpan( - spanName, - { startTime: startHr }, - parentCtx, - (span: { - setAttribute: (...args: unknown[]) => void; - end: (...args: unknown[]) => void; - }) => { - if (isAssistant) { - span.setAttribute('gen_ai.operation.name', 'chat'); - } - if (model) { - span.setAttribute('gen_ai.request.model', model); - span.setAttribute('gen_ai.response.model', model); - } - - // Per-span token usage (GenAI conventions) - if (msg.tokenUsage) { - if (msg.tokenUsage.input != null) { - span.setAttribute('gen_ai.usage.input_tokens', msg.tokenUsage.input); - } - if (msg.tokenUsage.output != null) { - span.setAttribute('gen_ai.usage.output_tokens', msg.tokenUsage.output); - } - if (msg.tokenUsage.cached != null) { - span.setAttribute('gen_ai.usage.cache_read.input_tokens', msg.tokenUsage.cached); - } - } - - if (captureContent && msg.content != null) { - span.setAttribute( - 'gen_ai.output.messages', - typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content), - ); - } - - // Tool call child spans - if (msg.toolCalls) { - const msgCtx = api.trace.setSpan(api.context.active(), span); - for (const tc of msg.toolCalls) { - api.context.with(msgCtx, () => { - tracer.startActiveSpan( - `execute_tool ${tc.tool}`, - {}, - msgCtx, - (toolSpan: { - setAttribute: (...args: unknown[]) => void; - end: (...args: unknown[]) => void; - }) => { - toolSpan.setAttribute('gen_ai.tool.name', tc.tool); - if (tc.id) toolSpan.setAttribute('gen_ai.tool.call.id', tc.id); - - if (captureContent) { - if (tc.input != null) { - toolSpan.setAttribute( - 'gen_ai.tool.call.arguments', - typeof tc.input === 'string' ? tc.input : JSON.stringify(tc.input), - ); - } - if (tc.output != null) { - toolSpan.setAttribute( - 'gen_ai.tool.call.result', - typeof tc.output === 'string' ? tc.output : JSON.stringify(tc.output), - ); - } - } - - toolSpan.end(); - }, - ); - }); - } - } - - span.end(endHr); - }, - ); - }); - } -} - -// --------------------------------------------------------------------------- -// Streaming observer -// --------------------------------------------------------------------------- - -/** - * Streaming observer that creates OTel spans in real-time during eval execution. - * Spans are exported immediately via SimpleSpanProcessor as each tool call / LLM response completes. - */ -export class OtelStreamingObserver { - // biome-ignore lint/suspicious/noExplicitAny: OTel span type loaded dynamically - private rootSpan: any = null; - // biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically - private rootCtx: any = null; - private observedChildSpans = false; - private pendingMetrics: { - durationMs?: number; - costUsd?: number; - tokenUsage?: ProviderTokenUsage; - trace?: { - eventCount: number; - toolCalls: Record; - llmCallCount?: number; - }; - } | null = null; - - constructor( - private readonly tracer: Tracer, - private readonly api: OtelApi, - private readonly captureContent: boolean, - // biome-ignore lint/suspicious/noExplicitAny: OTel context loaded dynamically - private readonly parentCtx?: any, - ) {} - - /** Create root eval span immediately (visible in backend right away) */ - startEvalCase(testId: string, target: string, evalSet?: string): void { - this.pendingMetrics = null; - this.observedChildSpans = false; - const ctx = this.parentCtx ?? this.api.context.active(); - this.rootSpan = this.tracer.startSpan('agentv.eval', undefined, ctx); - this.rootSpan.setAttribute('gen_ai.operation.name', 'evaluate'); - this.rootSpan.setAttribute('gen_ai.system', 'agentv'); - this.rootSpan.setAttribute('agentv.test_id', testId); - this.rootSpan.setAttribute('agentv.target', target); - if (evalSet) this.rootSpan.setAttribute('agentv.suite', evalSet); - this.rootCtx = this.api.trace.setSpan(this.api.context.active(), this.rootSpan); - } - - /** Create and immediately export a tool span */ - onToolCall( - name: string, - input: unknown, - output: unknown, - _durationMs: number, - toolCallId?: string, - ): void { - if (!this.rootCtx) return; - this.observedChildSpans = true; - this.api.context.with(this.rootCtx, () => { - const span = this.tracer.startSpan(`execute_tool ${name}`, undefined, this.rootCtx); - span.setAttribute('gen_ai.tool.name', name); - if (toolCallId) span.setAttribute('gen_ai.tool.call.id', toolCallId); - if (this.captureContent) { - if (input != null) - span.setAttribute( - 'gen_ai.tool.call.arguments', - typeof input === 'string' ? input : JSON.stringify(input), - ); - if (output != null) - span.setAttribute( - 'gen_ai.tool.call.result', - typeof output === 'string' ? output : JSON.stringify(output), - ); - } - span.end(); - }); - } - - /** Create and immediately export an LLM span */ - onLlmCall(model: string, tokenUsage?: ProviderTokenUsage): void { - if (!this.rootCtx) return; - this.observedChildSpans = true; - this.api.context.with(this.rootCtx, () => { - const span = this.tracer.startSpan(`chat ${model}`, undefined, this.rootCtx); - span.setAttribute('gen_ai.operation.name', 'chat'); - span.setAttribute('gen_ai.request.model', model); - span.setAttribute('gen_ai.response.model', model); - if (tokenUsage) { - if (tokenUsage.input != null) - span.setAttribute('gen_ai.usage.input_tokens', tokenUsage.input); - if (tokenUsage.output != null) - span.setAttribute('gen_ai.usage.output_tokens', tokenUsage.output); - if (tokenUsage.cached != null) - span.setAttribute('gen_ai.usage.cache_read.input_tokens', tokenUsage.cached); - } - span.end(); - }); - } - - /** Record final execution metrics before the root span is finalized. */ - recordEvalMetrics(result: { - durationMs?: number; - costUsd?: number; - tokenUsage?: ProviderTokenUsage; - trace?: { - eventCount: number; - toolCalls: Record; - llmCallCount?: number; - }; - }): void { - this.pendingMetrics = result; - } - - /** Finalize root span with score/verdict after evaluation completes */ - finalizeEvalCase(score: number, error?: string): void { - if (!this.rootSpan) return; - this.rootSpan.setAttribute('agentv.score', score); - if (this.pendingMetrics?.durationMs != null) { - this.rootSpan.setAttribute('agentv.trace.duration_ms', this.pendingMetrics.durationMs); - } - if (this.pendingMetrics?.costUsd != null) { - this.rootSpan.setAttribute('agentv.trace.cost_usd', this.pendingMetrics.costUsd); - } - if (this.pendingMetrics?.tokenUsage) { - if (this.pendingMetrics.tokenUsage.input != null) { - this.rootSpan.setAttribute( - 'agentv.trace.token_input', - this.pendingMetrics.tokenUsage.input, - ); - } - if (this.pendingMetrics.tokenUsage.output != null) { - this.rootSpan.setAttribute( - 'agentv.trace.token_output', - this.pendingMetrics.tokenUsage.output, - ); - } - if (this.pendingMetrics.tokenUsage.cached != null) { - this.rootSpan.setAttribute( - 'agentv.trace.token_cached', - this.pendingMetrics.tokenUsage.cached, - ); - } - } - if (this.pendingMetrics?.trace) { - this.rootSpan.setAttribute('agentv.trace.event_count', this.pendingMetrics.trace.eventCount); - this.rootSpan.setAttribute( - 'agentv.trace.tool_names', - Object.keys(this.pendingMetrics.trace.toolCalls).sort().join(','), - ); - if (this.pendingMetrics.trace.llmCallCount != null) { - this.rootSpan.setAttribute( - 'agentv.trace.llm_call_count', - this.pendingMetrics.trace.llmCallCount, - ); - } - } - if (error) { - this.rootSpan.setStatus({ code: this.api.SpanStatusCode.ERROR, message: error }); - } else { - this.rootSpan.setStatus({ code: this.api.SpanStatusCode.OK }); - } - this.rootSpan.end(); - this.rootSpan = null; - this.rootCtx = null; - this.observedChildSpans = false; - this.pendingMetrics = null; - } - - /** Backfill child spans from the completed result when the provider emitted no live callbacks. */ - completeFromResult(result: EvaluationResult): void { - this.recordEvalMetrics({ - durationMs: result.durationMs, - costUsd: result.costUsd, - tokenUsage: result.tokenUsage, - trace: result.trace, - }); - - if (this.observedChildSpans || !this.rootCtx) { - return; - } - - const model = - result.trace.messages.find((msg) => msg.role === 'assistant')?.metadata?.model ?? - result.target ?? - 'unknown'; - - this.onLlmCall(String(model), result.tokenUsage); - - for (const message of result.trace.messages) { - for (const toolCall of message.toolCalls ?? []) { - this.onToolCall( - toolCall.tool, - toolCall.input, - toolCall.output, - toolCall.durationMs ?? 0, - toolCall.id, - ); - } - } - } - - /** Return the active eval span's trace ID and span ID for Braintrust trace bridging */ - getActiveSpanIds(): { parentSpanId: string; rootSpanId: string } | null { - if (!this.rootSpan) return null; - try { - const spanCtx = this.rootSpan.spanContext?.() ?? this.rootSpan._spanContext; - if (!spanCtx?.traceId || !spanCtx?.spanId) return null; - return { parentSpanId: spanCtx.spanId, rootSpanId: spanCtx.traceId }; - } catch { - return null; - } - } - - /** Get ProviderStreamCallbacks for passing to providers */ - getStreamCallbacks(): ProviderStreamCallbacks { - return { - onToolCallEnd: (name, input, output, durationMs, toolCallId) => - this.onToolCall(name, input, output, durationMs, toolCallId), - onLlmCallEnd: (model, tokenUsage) => this.onLlmCall(model, tokenUsage), - getActiveSpanIds: () => this.getActiveSpanIds(), - }; - } -} - -// --------------------------------------------------------------------------- -// Turn grouping -// --------------------------------------------------------------------------- - -interface Turn { - messages: Message[]; -} - -function groupMessagesIntoTurns(messages: readonly Message[]): Turn[] { - const turns: Turn[] = []; - let current: Message[] = []; - for (const msg of messages) { - if (msg.role === 'user' && current.length > 0) { - turns.push({ messages: current }); - current = []; - } - current.push(msg); - } - if (current.length > 0) turns.push({ messages: current }); - return turns; -} - -// --------------------------------------------------------------------------- -// Utilities -// --------------------------------------------------------------------------- - -/** Convert an optional ISO timestamp to an HrTime-compatible value (milliseconds). */ -function toHrTime(iso?: string): number | undefined { - if (!iso) return undefined; - return new Date(iso).getTime(); -} diff --git a/packages/core/src/observability/otlp-json-file-exporter.ts b/packages/core/src/observability/otlp-json-file-exporter.ts deleted file mode 100644 index 67dfdeda7..000000000 --- a/packages/core/src/observability/otlp-json-file-exporter.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { mkdir } from 'node:fs/promises'; -import { dirname } from 'node:path'; - -// biome-ignore lint/suspicious/noExplicitAny: OTel ReadableSpan loaded dynamically -type ReadableSpan = any; - -/** - * SpanExporter that writes OTLP JSON (the standard OTel wire format) to a file. - * The file can be imported by any OTel-compatible backend. - */ -export class OtlpJsonFileExporter { - // biome-ignore lint/suspicious/noExplicitAny: serialized span data - private spans: any[] = []; - private filePath: string; - private resourceAttributes: Record; - - constructor(filePath: string, resourceAttributes: Record = {}) { - this.filePath = filePath; - this.resourceAttributes = { ...resourceAttributes }; - } - - export(spans: ReadableSpan[], resultCallback: (result: { code: number }) => void): void { - for (const span of spans) { - this.resourceAttributes = { - ...this.resourceAttributes, - ...getSpanResourceAttributes(span), - }; - this.spans.push({ - traceId: span.spanContext().traceId, - spanId: span.spanContext().spanId, - parentSpanId: span.parentSpanId || undefined, - name: span.name, - kind: span.kind, - startTimeUnixNano: hrTimeToNanos(span.startTime), - endTimeUnixNano: hrTimeToNanos(span.endTime), - attributes: convertAttributes(span.attributes), - status: span.status, - events: span.events?.map( - (e: { name: string; time: [number, number]; attributes?: Record }) => ({ - name: e.name, - timeUnixNano: hrTimeToNanos(e.time), - attributes: convertAttributes(e.attributes), - }), - ), - }); - } - resultCallback({ code: 0 }); // SUCCESS - } - - async shutdown(): Promise { - await this.flush(); - } - - async forceFlush(): Promise { - await this.flush(); - } - - private async flush(): Promise { - if (this.spans.length === 0) return; - - await mkdir(dirname(this.filePath), { recursive: true }); - - const otlpJson = { - resourceSpans: [ - { - resource: { attributes: convertAttributes(this.resourceAttributes) }, - scopeSpans: [ - { - scope: { name: 'agentv', version: '1.0.0' }, - spans: this.spans, - }, - ], - }, - ], - }; - - const { writeFile } = await import('node:fs/promises'); - await writeFile(this.filePath, JSON.stringify(otlpJson, null, 2)); - } -} - -function getSpanResourceAttributes(span: ReadableSpan): Record { - if (span.resource?.attributes && typeof span.resource.attributes === 'object') { - return span.resource.attributes; - } - return {}; -} - -function hrTimeToNanos(hrTime: [number, number]): string { - return String(hrTime[0] * 1_000_000_000 + hrTime[1]); -} - -function convertAttributes( - attrs: Record | undefined, -): Array<{ key: string; value: unknown }> { - return Object.entries(attrs || {}).map(([key, value]) => ({ - key, - value: serializeAttributeValue(value), - })); -} - -function serializeAttributeValue(value: unknown): unknown { - if (typeof value === 'string') return { stringValue: value }; - if (typeof value === 'number') { - return Number.isInteger(value) ? { intValue: value } : { doubleValue: value }; - } - if (typeof value === 'boolean') return { boolValue: value }; - if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } }; - return { stringValue: String(value) }; -} diff --git a/packages/core/src/observability/types.ts b/packages/core/src/observability/types.ts deleted file mode 100644 index 0526527d0..000000000 --- a/packages/core/src/observability/types.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** Options for configuring the OTel trace exporter. */ -export interface OtelExportOptions { - /** OTLP endpoint URL */ - readonly endpoint?: string; - /** Custom headers (e.g., auth) */ - readonly headers?: Record; - /** Resource attributes to attach to the trace provider */ - readonly resourceAttributes?: Record; - /** Whether to include message content in spans */ - readonly captureContent?: boolean; - /** Service name for OTel resource */ - readonly serviceName?: string; - /** When true, group messages into turn spans for multi-turn evals */ - readonly groupTurns?: boolean; - /** Path to write OTLP JSON file (importable by OTel backends) */ - readonly otlpFilePath?: string; -} - -export interface OtelBackendResolverContext { - readonly env: Record; - readonly cwd: string; -} - -export interface OtelBackendResolution { - readonly endpoint: string; - readonly headers?: Record; - readonly resourceAttributes?: Record; - readonly warnings?: readonly string[]; -} - -/** Generic resolver contract for OTel backend endpoint/header/resource routing. */ -export interface OtelBackendResolver { - readonly name: string; - resolve(context: OtelBackendResolverContext): OtelBackendResolution; -} diff --git a/packages/core/test/evaluation/config.test.ts b/packages/core/test/evaluation/config.test.ts index 818debae1..636051361 100644 --- a/packages/core/test/evaluation/config.test.ts +++ b/packages/core/test/evaluation/config.test.ts @@ -13,11 +13,12 @@ describe('defineConfig execution defaults', () => { expect(config.execution?.keepWorkspaces).toBe(true); }); - it('accepts otelFile string', () => { - const config = defineConfig({ - execution: { otelFile: '.agentv/results/otel-{timestamp}.json' }, - }); - expect(config.execution?.otelFile).toBe('.agentv/results/otel-{timestamp}.json'); + it('rejects removed otelFile export config', () => { + expect(() => + defineConfig({ + execution: { otelFile: '.agentv/results/otel-{timestamp}.json' }, + } as never), + ).toThrow(/execution\.otelFile has been removed/); }); it('accepts all execution fields together', () => { @@ -28,7 +29,6 @@ describe('defineConfig execution defaults', () => { agentTimeoutMs: 120_000, verbose: true, keepWorkspaces: false, - otelFile: 'otel.json', }, }); expect(config.execution).toEqual({ @@ -37,7 +37,6 @@ describe('defineConfig execution defaults', () => { agentTimeoutMs: 120_000, verbose: true, keepWorkspaces: false, - otelFile: 'otel.json', }); }); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 722d4192a..e3ee1aeb4 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -823,12 +823,10 @@ describe('parseExecutionDefaults', () => { } }); - it('parses otel_file string', () => { - const result = parseExecutionDefaults( - { otel_file: '.agentv/results/otel.json' }, - '/test/config.yaml', - ); - expect(result?.otel_file).toBe('.agentv/results/otel.json'); + it('rejects removed OTLP export fields', () => { + expect(() => + parseExecutionDefaults({ otel_file: '.agentv/results/otel.json' }, '/test/config.yaml'), + ).toThrow(/execution\.otel_file.*has been removed/); }); it('parses all supported fields together', () => { @@ -836,14 +834,12 @@ describe('parseExecutionDefaults', () => { { verbose: true, keep_workspaces: false, - otel_file: 'otel.json', }, '/test/config.local.yaml', ); expect(result).toEqual({ verbose: true, keep_workspaces: false, - otel_file: 'otel.json', }); }); @@ -873,53 +869,18 @@ describe('parseExecutionDefaults', () => { expect(result).toEqual({ verbose: true }); }); - it('parses export_otel boolean', () => { - const result = parseExecutionDefaults({ export_otel: true }, '/test/config.yaml'); - expect(result?.export_otel).toBe(true); - }); - - it('ignores non-boolean export_otel', () => { - const result = parseExecutionDefaults({ export_otel: 'yes' }, '/test/config.yaml'); - expect(result?.export_otel).toBeUndefined(); - }); - - it('parses otel_backend string', () => { - const result = parseExecutionDefaults({ otel_backend: 'langfuse' }, '/test/config.yaml'); - expect(result?.otel_backend).toBe('langfuse'); - }); - - it('ignores empty otel_backend', () => { - const result = parseExecutionDefaults({ otel_backend: ' ' }, '/test/config.yaml'); - expect(result?.otel_backend).toBeUndefined(); - }); - - it('parses otel_capture_content boolean', () => { - const result = parseExecutionDefaults({ otel_capture_content: true }, '/test/config.yaml'); - expect(result?.otel_capture_content).toBe(true); - }); - - it('parses otel_group_turns boolean', () => { - const result = parseExecutionDefaults({ otel_group_turns: true }, '/test/config.yaml'); - expect(result?.otel_group_turns).toBe(true); - }); - - it('parses all OTel fields together', () => { - const result = parseExecutionDefaults( - { - export_otel: true, - otel_backend: 'langfuse', - otel_file: 'otel.json', - otel_capture_content: false, - otel_group_turns: true, - }, - '/test/config.yaml', - ); - expect(result).toEqual({ - export_otel: true, - otel_backend: 'langfuse', - otel_file: 'otel.json', - otel_capture_content: false, - otel_group_turns: true, - }); + it('rejects all removed OTel fields together', () => { + expect(() => + parseExecutionDefaults( + { + export_otel: true, + otel_backend: 'langfuse', + otel_file: 'otel.json', + otel_capture_content: false, + otel_group_turns: true, + }, + '/test/config.yaml', + ), + ).toThrow(/execution\.otel_backend/); }); }); diff --git a/packages/core/test/observability/file-exporters.test.ts b/packages/core/test/observability/file-exporters.test.ts deleted file mode 100644 index 7b341ea32..000000000 --- a/packages/core/test/observability/file-exporters.test.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { afterEach, describe, expect, it } from 'bun:test'; -import { readFile, rm } from 'node:fs/promises'; -import path from 'node:path'; -import { OtlpJsonFileExporter } from '../../src/observability/otlp-json-file-exporter.js'; - -const testDir = path.join(import.meta.dir, '.test-file-exporters'); - -afterEach(async () => { - await rm(testDir, { recursive: true, force: true }).catch(() => {}); -}); - -// --------------------------------------------------------------------------- -// Mock span helpers -// --------------------------------------------------------------------------- - -function makeSpan(overrides: { - traceId?: string; - spanId?: string; - parentSpanId?: string; - name?: string; - kind?: number; - startTime?: [number, number]; - endTime?: [number, number]; - attributes?: Record; - resourceAttributes?: Record; - status?: { code: number }; - events?: Array<{ name: string; time: [number, number]; attributes?: Record }>; -}) { - const traceId = overrides.traceId ?? 'abc123'; - const spanId = overrides.spanId ?? 'span1'; - return { - spanContext: () => ({ traceId, spanId }), - parentSpanId: overrides.parentSpanId, - name: overrides.name ?? 'test-span', - kind: overrides.kind ?? 0, - startTime: overrides.startTime ?? [1000, 0], - endTime: overrides.endTime ?? [1001, 0], - attributes: overrides.attributes ?? {}, - resource: { attributes: overrides.resourceAttributes ?? {} }, - status: overrides.status ?? { code: 0 }, - events: overrides.events ?? [], - }; -} - -// --------------------------------------------------------------------------- -// OtlpJsonFileExporter -// --------------------------------------------------------------------------- - -describe('OtlpJsonFileExporter', () => { - it('writes OTLP JSON with resourceSpans structure', async () => { - const filePath = path.join(testDir, 'otlp', 'trace.json'); - const exporter = new OtlpJsonFileExporter(filePath); - - const span = makeSpan({ - name: 'agentv.eval', - attributes: { 'agentv.test_id': 'test-1', 'agentv.score': 0.9 }, - events: [ - { - name: 'agentv.grader.match', - time: [1000, 500_000_000], - attributes: { 'agentv.grader.score': 1 }, - }, - ], - }); - - exporter.export([span], (result) => { - expect(result.code).toBe(0); - }); - - await exporter.shutdown(); - - const content = await readFile(filePath, 'utf8'); - const parsed = JSON.parse(content); - - expect(parsed.resourceSpans).toHaveLength(1); - expect(parsed.resourceSpans[0].scopeSpans).toHaveLength(1); - expect(parsed.resourceSpans[0].scopeSpans[0].scope.name).toBe('agentv'); - - const spans = parsed.resourceSpans[0].scopeSpans[0].spans; - expect(spans).toHaveLength(1); - expect(spans[0].name).toBe('agentv.eval'); - expect(spans[0].traceId).toBe('abc123'); - expect(spans[0].spanId).toBe('span1'); - expect(spans[0].startTimeUnixNano).toBe('1000000000000'); - expect(spans[0].endTimeUnixNano).toBe('1001000000000'); - - // Check attributes serialization - const testIdAttr = spans[0].attributes.find((a: { key: string }) => a.key === 'agentv.test_id'); - expect(testIdAttr.value).toEqual({ stringValue: 'test-1' }); - - const scoreAttr = spans[0].attributes.find((a: { key: string }) => a.key === 'agentv.score'); - expect(scoreAttr.value).toEqual({ doubleValue: 0.9 }); - - // Check events - expect(spans[0].events).toHaveLength(1); - expect(spans[0].events[0].name).toBe('agentv.grader.match'); - }); - - it('collects spans across multiple export calls', async () => { - const filePath = path.join(testDir, 'otlp', 'multi.json'); - const exporter = new OtlpJsonFileExporter(filePath); - - exporter.export([makeSpan({ spanId: 's1', name: 'span-1' })], (r) => expect(r.code).toBe(0)); - exporter.export([makeSpan({ spanId: 's2', name: 'span-2' })], (r) => expect(r.code).toBe(0)); - - await exporter.shutdown(); - - const parsed = JSON.parse(await readFile(filePath, 'utf8')); - const spans = parsed.resourceSpans[0].scopeSpans[0].spans; - expect(spans).toHaveLength(2); - expect(spans[0].name).toBe('span-1'); - expect(spans[1].name).toBe('span-2'); - }); - - it('no-ops when no spans were exported', async () => { - const filePath = path.join(testDir, 'otlp', 'empty.json'); - const exporter = new OtlpJsonFileExporter(filePath); - await exporter.shutdown(); - - // File should not be created - await expect(readFile(filePath, 'utf8')).rejects.toThrow(); - }); - - it('serializes attribute types correctly', async () => { - const filePath = path.join(testDir, 'otlp', 'attrs.json'); - const exporter = new OtlpJsonFileExporter(filePath); - - exporter.export( - [ - makeSpan({ - attributes: { - str: 'hello', - int: 42, - float: 3.14, - bool: true, - arr: ['a', 'b'], - }, - }), - ], - () => {}, - ); - - await exporter.shutdown(); - - const parsed = JSON.parse(await readFile(filePath, 'utf8')); - const attrs = parsed.resourceSpans[0].scopeSpans[0].spans[0].attributes; - const byKey = Object.fromEntries( - attrs.map((a: { key: string; value: unknown }) => [a.key, a.value]), - ); - - expect(byKey.str).toEqual({ stringValue: 'hello' }); - expect(byKey.int).toEqual({ intValue: 42 }); - expect(byKey.float).toEqual({ doubleValue: 3.14 }); - expect(byKey.bool).toEqual({ boolValue: true }); - expect(byKey.arr).toEqual({ - arrayValue: { - values: [{ stringValue: 'a' }, { stringValue: 'b' }], - }, - }); - }); - - it('serializes resource attributes into the OTLP resource span', async () => { - const filePath = path.join(testDir, 'otlp', 'resource-attrs.json'); - const exporter = new OtlpJsonFileExporter(filePath, { - 'service.name': 'agentv', - 'openinference.project.name': 'phoenix-project', - }); - - exporter.export( - [ - makeSpan({ - resourceAttributes: { - 'agentv.resource.flag': true, - }, - }), - ], - () => {}, - ); - - await exporter.shutdown(); - - const parsed = JSON.parse(await readFile(filePath, 'utf8')); - const attrs = parsed.resourceSpans[0].resource.attributes; - const byKey = Object.fromEntries( - attrs.map((a: { key: string; value: unknown }) => [a.key, a.value]), - ); - - expect(byKey['service.name']).toEqual({ stringValue: 'agentv' }); - expect(byKey['openinference.project.name']).toEqual({ stringValue: 'phoenix-project' }); - expect(byKey['agentv.resource.flag']).toEqual({ boolValue: true }); - }); -}); diff --git a/packages/core/test/observability/otel-exporter.test.ts b/packages/core/test/observability/otel-exporter.test.ts deleted file mode 100644 index 85741113f..000000000 --- a/packages/core/test/observability/otel-exporter.test.ts +++ /dev/null @@ -1,379 +0,0 @@ -/** - * Tests for OTel trace exporter. - * These tests exercise logic that does NOT require actual OTel SDK packages. - */ - -import { afterEach, describe, expect, it } from 'bun:test'; -import { buildTraceFromMessages } from '../../src/evaluation/trace.js'; -import { OtelTraceExporter } from '../../src/observability/otel-exporter.js'; - -// --------------------------------------------------------------------------- -// OtelTraceExporter class -// --------------------------------------------------------------------------- - -describe('OTel OtelTraceExporter', () => { - describe('constructor', () => { - it('does not throw when constructed with minimal options', () => { - expect(() => new OtelTraceExporter({})).not.toThrow(); - }); - - it('does not throw when constructed with full options', () => { - expect( - () => - new OtelTraceExporter({ - endpoint: 'https://example.com/v1/traces', - headers: { Authorization: 'Bearer test' }, - captureContent: true, - serviceName: 'my-service', - }), - ).not.toThrow(); - }); - }); - - describe('init()', () => { - it('returns false when OTel packages are not importable', async () => { - // In a test environment without OTel packages installed as real deps, - // the dynamic import will fail and init() should return false. - const exporter = new OtelTraceExporter({ endpoint: 'https://example.com/v1/traces' }); - const result = await exporter.init(); - // If OTel packages happen to be installed, this will be true—either outcome is valid - expect(typeof result).toBe('boolean'); - }); - }); - - describe('exportResult() without init', () => { - it('silently no-ops when called before init()', async () => { - const exporter = new OtelTraceExporter({}); - // Should not throw even though tracer/api are null - await expect( - exporter.exportResult({ - testId: 'test-1', - target: 'my-agent', - score: 0.95, - output: [{ role: 'assistant' as const, content: 'hello' }], - timestamp: new Date().toISOString(), - } as unknown as Parameters[0]), - ).resolves.toBeUndefined(); - }); - }); - - describe('shutdown() without init', () => { - it('resolves cleanly when called before init()', async () => { - const exporter = new OtelTraceExporter({}); - await expect(exporter.shutdown()).resolves.toBeUndefined(); - }); - }); -}); - -// --------------------------------------------------------------------------- -// W3C traceparent propagation -// --------------------------------------------------------------------------- - -describe('W3C traceparent propagation', () => { - const VALID_TRACEPARENT = '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01'; - const VALID_TRACE_ID = '0af7651916cd43dd8448eb211c80319c'; - const VALID_PARENT_SPAN_ID = 'b7ad6b7169203331'; - - const savedTraceparent = process.env.TRACEPARENT; - const savedTracestate = process.env.TRACESTATE; - - afterEach(() => { - if (savedTraceparent !== undefined) { - process.env.TRACEPARENT = savedTraceparent; - } else { - process.env.TRACEPARENT = undefined; - } - if (savedTracestate !== undefined) { - process.env.TRACESTATE = savedTracestate; - } else { - process.env.TRACESTATE = undefined; - } - }); - - /** - * Helper: create an OtelTraceExporter wired to an InMemorySpanExporter. - * We create our own NodeTracerProvider with an InMemorySpanExporter and - * inject it into the exporter's private fields, bypassing init(). - */ - async function createTestExporter() { - try { - const [sdkTraceNode, api] = await Promise.all([ - import('@opentelemetry/sdk-trace-node'), - import('@opentelemetry/api'), - ]); - - const { NodeTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = sdkTraceNode; - const memExporter = new InMemorySpanExporter(); - - const provider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memExporter)], - }); - provider.register(); - - const exporter = new OtelTraceExporter({ - endpoint: 'http://localhost:4318/v1/traces', - }); - - // Inject private fields so exportResult() works without HTTP exporter - // biome-ignore lint/suspicious/noExplicitAny: test access to private fields - const exp = exporter as any; - exp.provider = provider; - exp.api = api; - exp.tracer = provider.getTracer('agentv-test', '1.0.0'); - - // Inject W3C propagator for traceparent tests - try { - const coreMod = await import('@opentelemetry/core'); - exp.W3CPropagator = coreMod.W3CTraceContextPropagator; - } catch { - // W3C propagation tests will be skipped - } - - return { exporter, memExporter, provider }; - } catch { - return null; - } - } - - const makeResult = () => - ({ - testId: 'test-tp', - target: 'my-agent', - score: 1, - output: 'ok', - trace: buildTraceFromMessages({ - output: [{ role: 'assistant' as const, content: 'ok' }], - finalOutput: 'ok', - target: 'my-agent', - testId: 'test-tp', - }), - timestamp: new Date().toISOString(), - }) as unknown as Parameters[0]; - - it('creates a standalone trace when TRACEPARENT is not set', async () => { - process.env.TRACEPARENT = undefined; - const setup = await createTestExporter(); - if (!setup) return; // OTel not available — skip - - await setup.exporter.exportResult(makeResult()); - - const spans = setup.memExporter.getFinishedSpans(); - expect(spans.length).toBeGreaterThanOrEqual(1); - - const root = spans.find((s) => s.name === 'agentv.eval'); - expect(root).toBeDefined(); - // No parent — parentSpanContext should be undefined - expect(root?.parentSpanContext).toBeUndefined(); - - await setup.exporter.shutdown(); - }); - - it('inherits traceId and parentSpanId from a valid TRACEPARENT', async () => { - process.env.TRACEPARENT = VALID_TRACEPARENT; - const setup = await createTestExporter(); - if (!setup) return; - - await setup.exporter.exportResult(makeResult()); - - const spans = setup.memExporter.getFinishedSpans(); - const root = spans.find((s) => s.name === 'agentv.eval'); - expect(root).toBeDefined(); - expect(root?.spanContext().traceId).toBe(VALID_TRACE_ID); - expect(root?.parentSpanContext?.spanId).toBe(VALID_PARENT_SPAN_ID); - - await setup.exporter.shutdown(); - }); - - it('falls back to standalone trace when TRACEPARENT is malformed', async () => { - process.env.TRACEPARENT = 'not-a-valid-traceparent'; - const setup = await createTestExporter(); - if (!setup) return; - - await setup.exporter.exportResult(makeResult()); - - const spans = setup.memExporter.getFinishedSpans(); - const root = spans.find((s) => s.name === 'agentv.eval'); - expect(root).toBeDefined(); - // Malformed traceparent is ignored by the propagator — new root trace - expect(root?.spanContext().traceId).not.toBe(VALID_TRACE_ID); - - await setup.exporter.shutdown(); - }); - - it('propagates TRACESTATE when present alongside TRACEPARENT', async () => { - process.env.TRACEPARENT = VALID_TRACEPARENT; - process.env.TRACESTATE = 'vendor=opaque'; - const setup = await createTestExporter(); - if (!setup) return; - - await setup.exporter.exportResult(makeResult()); - - const spans = setup.memExporter.getFinishedSpans(); - const root = spans.find((s) => s.name === 'agentv.eval'); - expect(root).toBeDefined(); - // traceId should match the parent - expect(root?.spanContext().traceId).toBe(VALID_TRACE_ID); - expect(root?.spanContext().traceState?.get('vendor')).toBe('opaque'); - - await setup.exporter.shutdown(); - }); -}); - -// --------------------------------------------------------------------------- -// Per-span token usage metrics -// --------------------------------------------------------------------------- - -describe('Per-span token usage metrics', () => { - const savedTraceparent = process.env.TRACEPARENT; - - afterEach(() => { - if (savedTraceparent !== undefined) { - process.env.TRACEPARENT = savedTraceparent; - } else { - process.env.TRACEPARENT = undefined; - } - }); - - async function createTestExporter() { - try { - const [sdkTraceNode, api] = await Promise.all([ - import('@opentelemetry/sdk-trace-node'), - import('@opentelemetry/api'), - ]); - - const { NodeTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = sdkTraceNode; - const memExporter = new InMemorySpanExporter(); - - const provider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memExporter)], - }); - - const exporter = new OtelTraceExporter({ - endpoint: 'http://localhost:4318/v1/traces', - }); - - // biome-ignore lint/suspicious/noExplicitAny: test access to private fields - const exp = exporter as any; - exp.provider = provider; - exp.api = api; - exp.tracer = provider.getTracer('agentv-test', '1.0.0'); - - return { exporter, memExporter }; - } catch { - return null; - } - } - - it('sets token usage attributes on child spans when tokenUsage is present', async () => { - process.env.TRACEPARENT = undefined; - const setup = await createTestExporter(); - if (!setup) return; - - const result = { - testId: 'test-tokens', - target: 'my-agent', - score: 1, - timestamp: new Date().toISOString(), - output: 'hello', - trace: buildTraceFromMessages({ - output: [ - { - role: 'assistant', - content: 'hello', - metadata: { model: 'gpt-4' }, - tokenUsage: { input: 100, output: 50, cached: 25 }, - }, - ], - finalOutput: 'hello', - target: 'my-agent', - testId: 'test-tokens', - }), - } as unknown as Parameters[0]; - - await setup.exporter.exportResult(result); - - const spans = setup.memExporter.getFinishedSpans(); - const msgSpan = spans.find((s) => s.name.startsWith('chat ')); - expect(msgSpan).toBeDefined(); - expect(msgSpan?.attributes['gen_ai.usage.input_tokens']).toBe(100); - expect(msgSpan?.attributes['gen_ai.usage.output_tokens']).toBe(50); - expect(msgSpan?.attributes['gen_ai.usage.cache_read.input_tokens']).toBe(25); - - await setup.exporter.shutdown(); - }); - - it('omits token usage attributes when tokenUsage is not present', async () => { - process.env.TRACEPARENT = undefined; - const setup = await createTestExporter(); - if (!setup) return; - - const result = { - testId: 'test-no-tokens', - target: 'my-agent', - score: 1, - timestamp: new Date().toISOString(), - output: 'hello', - trace: buildTraceFromMessages({ - output: [ - { - role: 'assistant', - content: 'hello', - metadata: { model: 'gpt-4' }, - }, - ], - finalOutput: 'hello', - target: 'my-agent', - testId: 'test-no-tokens', - }), - } as unknown as Parameters[0]; - - await setup.exporter.exportResult(result); - - const spans = setup.memExporter.getFinishedSpans(); - const msgSpan = spans.find((s) => s.name.startsWith('chat ')); - expect(msgSpan).toBeDefined(); - expect(msgSpan?.attributes['gen_ai.usage.input_tokens']).toBeUndefined(); - expect(msgSpan?.attributes['gen_ai.usage.output_tokens']).toBeUndefined(); - expect(msgSpan?.attributes['gen_ai.usage.cache_read.input_tokens']).toBeUndefined(); - - await setup.exporter.shutdown(); - }); - - it('omits cached attribute when only input and output are present', async () => { - process.env.TRACEPARENT = undefined; - const setup = await createTestExporter(); - if (!setup) return; - - const result = { - testId: 'test-partial-tokens', - target: 'my-agent', - score: 1, - timestamp: new Date().toISOString(), - output: 'hello', - trace: buildTraceFromMessages({ - output: [ - { - role: 'assistant', - content: 'hello', - metadata: { model: 'gpt-4' }, - tokenUsage: { input: 200, output: 75 }, - }, - ], - finalOutput: 'hello', - target: 'my-agent', - testId: 'test-partial-tokens', - }), - } as unknown as Parameters[0]; - - await setup.exporter.exportResult(result); - - const spans = setup.memExporter.getFinishedSpans(); - const msgSpan = spans.find((s) => s.name.startsWith('chat ')); - expect(msgSpan).toBeDefined(); - expect(msgSpan?.attributes['gen_ai.usage.input_tokens']).toBe(200); - expect(msgSpan?.attributes['gen_ai.usage.output_tokens']).toBe(75); - expect(msgSpan?.attributes['gen_ai.usage.cache_read.input_tokens']).toBeUndefined(); - - await setup.exporter.shutdown(); - }); -}); diff --git a/packages/core/test/observability/otel-group-turns.test.ts b/packages/core/test/observability/otel-group-turns.test.ts deleted file mode 100644 index 945e1eabf..000000000 --- a/packages/core/test/observability/otel-group-turns.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { Message } from '../../src/evaluation/providers/types.js'; - -// Extract and test the groupMessagesIntoTurns logic directly -interface Turn { - messages: Message[]; -} - -function groupMessagesIntoTurns(messages: readonly Message[]): Turn[] { - const turns: Turn[] = []; - let current: Message[] = []; - for (const msg of messages) { - if (msg.role === 'user' && current.length > 0) { - turns.push({ messages: current }); - current = []; - } - current.push(msg); - } - if (current.length > 0) turns.push({ messages: current }); - return turns; -} - -describe('groupMessagesIntoTurns', () => { - it('returns a single turn for single-turn conversation', () => { - const messages: Message[] = [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi there' }, - ]; - const turns = groupMessagesIntoTurns(messages); - expect(turns).toHaveLength(1); - expect(turns[0].messages).toHaveLength(2); - expect(turns[0].messages[0].role).toBe('user'); - expect(turns[0].messages[1].role).toBe('assistant'); - }); - - it('splits multi-turn conversation at each user message', () => { - const messages: Message[] = [ - { role: 'user', content: 'Turn 1' }, - { role: 'assistant', content: 'Response 1' }, - { role: 'user', content: 'Turn 2' }, - { role: 'assistant', content: 'Response 2' }, - { role: 'user', content: 'Turn 3' }, - { role: 'assistant', content: 'Response 3' }, - ]; - const turns = groupMessagesIntoTurns(messages); - expect(turns).toHaveLength(3); - expect(turns[0].messages).toEqual([ - { role: 'user', content: 'Turn 1' }, - { role: 'assistant', content: 'Response 1' }, - ]); - expect(turns[1].messages).toEqual([ - { role: 'user', content: 'Turn 2' }, - { role: 'assistant', content: 'Response 2' }, - ]); - expect(turns[2].messages).toEqual([ - { role: 'user', content: 'Turn 3' }, - { role: 'assistant', content: 'Response 3' }, - ]); - }); - - it('handles assistant-only messages (no user prefix)', () => { - const messages: Message[] = [ - { role: 'assistant', content: 'System greeting' }, - { role: 'assistant', content: 'Another message' }, - ]; - const turns = groupMessagesIntoTurns(messages); - expect(turns).toHaveLength(1); - expect(turns[0].messages).toHaveLength(2); - }); - - it('handles empty input', () => { - const turns = groupMessagesIntoTurns([]); - expect(turns).toHaveLength(0); - }); - - it('handles user message with tool calls before next user message', () => { - const messages: Message[] = [ - { role: 'user', content: 'Do something' }, - { role: 'assistant', content: 'OK', toolCalls: [{ tool: 'search', input: 'query' }] }, - { role: 'tool', content: 'result' }, - { role: 'assistant', content: 'Here is the answer' }, - { role: 'user', content: 'Follow up' }, - { role: 'assistant', content: 'Sure' }, - ]; - const turns = groupMessagesIntoTurns(messages); - expect(turns).toHaveLength(2); - expect(turns[0].messages).toHaveLength(4); - expect(turns[1].messages).toHaveLength(2); - }); -}); - -describe('OtelTraceExporter groupTurns integration', () => { - it('OtelExportOptions accepts groupTurns field', async () => { - const { OtelTraceExporter } = await import('../../src/observability/otel-exporter.js'); - // Verify the exporter can be constructed with groupTurns option - const exporter = new OtelTraceExporter({ - endpoint: 'http://localhost:4318/v1/traces', - groupTurns: true, - }); - expect(exporter).toBeDefined(); - }); - - it('OtelExportOptions works without groupTurns (default behavior)', async () => { - const { OtelTraceExporter } = await import('../../src/observability/otel-exporter.js'); - const exporter = new OtelTraceExporter({ - endpoint: 'http://localhost:4318/v1/traces', - }); - expect(exporter).toBeDefined(); - }); -}); diff --git a/packages/core/test/observability/streaming-observer.test.ts b/packages/core/test/observability/streaming-observer.test.ts deleted file mode 100644 index ae4102aa4..000000000 --- a/packages/core/test/observability/streaming-observer.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Tests for OtelStreamingObserver. - * Uses mock tracer/api objects to verify span creation and attribute setting. - */ - -import { describe, expect, it } from 'bun:test'; -import { OtelStreamingObserver } from '../../src/observability/otel-exporter.js'; - -// --------------------------------------------------------------------------- -// Mock OTel primitives -// --------------------------------------------------------------------------- - -interface MockSpan { - name: string; - attributes: Record; - status?: { code: number; message?: string }; - ended: boolean; -} - -function createMockSpan(name: string): MockSpan { - return { - name, - attributes: {}, - ended: false, - setAttribute(key: string, value: unknown) { - this.attributes[key] = value; - }, - setStatus(status: { code: number; message?: string }) { - this.status = status; - }, - end() { - this.ended = true; - }, - }; -} - -function createMockTracer(spans: MockSpan[]) { - return { - startSpan(name: string) { - const span = createMockSpan(name); - spans.push(span); - return span; - }, - }; -} - -function createMockApi() { - return { - trace: { - setSpan(_ctx: unknown, _span: unknown) { - return { spanSet: true }; - }, - }, - context: { - active() { - return {}; - }, - with(_ctx: unknown, fn: () => void) { - fn(); - }, - }, - SpanStatusCode: { - OK: 1, - ERROR: 2, - }, - }; -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -describe('OtelStreamingObserver', () => { - it('creates a root eval span on startEvalCase', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - observer.startEvalCase('test-1', 'my-target', 'my-suite'); - - expect(spans).toHaveLength(1); - expect(spans[0].name).toBe('agentv.eval'); - expect(spans[0].attributes['agentv.test_id']).toBe('test-1'); - expect(spans[0].attributes['agentv.target']).toBe('my-target'); - expect(spans[0].attributes['agentv.suite']).toBe('my-suite'); - expect(spans[0].attributes['gen_ai.system']).toBe('agentv'); - expect(spans[0].ended).toBe(false); - }); - - it('creates a tool span on onToolCall', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), true); - - observer.startEvalCase('test-1', 'target'); - observer.onToolCall('read_file', { path: '/a.txt' }, 'contents', 150, 'tc-1'); - - expect(spans).toHaveLength(2); - const toolSpan = spans[1]; - expect(toolSpan.name).toBe('execute_tool read_file'); - expect(toolSpan.attributes['gen_ai.tool.name']).toBe('read_file'); - expect(toolSpan.attributes['gen_ai.tool.call.id']).toBe('tc-1'); - expect(toolSpan.attributes['gen_ai.tool.call.arguments']).toBe('{"path":"/a.txt"}'); - expect(toolSpan.attributes['gen_ai.tool.call.result']).toBe('contents'); - expect(toolSpan.ended).toBe(true); - }); - - it('omits content attributes when captureContent is false', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - observer.startEvalCase('test-1', 'target'); - observer.onToolCall('bash', 'ls', 'output', 50); - - const toolSpan = spans[1]; - expect(toolSpan.attributes['gen_ai.tool.call.arguments']).toBeUndefined(); - expect(toolSpan.attributes['gen_ai.tool.call.result']).toBeUndefined(); - }); - - it('creates an LLM span on onLlmCall', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - observer.startEvalCase('test-1', 'target'); - observer.onLlmCall('claude-sonnet-4-20250514', { input: 100, output: 50, cached: 20 }); - - expect(spans).toHaveLength(2); - const llmSpan = spans[1]; - expect(llmSpan.name).toBe('chat claude-sonnet-4-20250514'); - expect(llmSpan.attributes['gen_ai.operation.name']).toBe('chat'); - expect(llmSpan.attributes['gen_ai.request.model']).toBe('claude-sonnet-4-20250514'); - expect(llmSpan.attributes['gen_ai.usage.input_tokens']).toBe(100); - expect(llmSpan.attributes['gen_ai.usage.output_tokens']).toBe(50); - expect(llmSpan.attributes['gen_ai.usage.cache_read.input_tokens']).toBe(20); - expect(llmSpan.ended).toBe(true); - }); - - it('finalizes root span with OK status and score', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - observer.startEvalCase('test-1', 'target'); - observer.finalizeEvalCase(0.85); - - const rootSpan = spans[0]; - expect(rootSpan.attributes['agentv.score']).toBe(0.85); - expect(rootSpan.status).toEqual({ code: 1 }); - expect(rootSpan.ended).toBe(true); - }); - - it('finalizes root span with ERROR status on error', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - observer.startEvalCase('test-1', 'target'); - observer.finalizeEvalCase(0, 'timeout'); - - const rootSpan = spans[0]; - expect(rootSpan.attributes['agentv.score']).toBe(0); - expect(rootSpan.status).toEqual({ code: 2, message: 'timeout' }); - expect(rootSpan.ended).toBe(true); - }); - - it('no-ops when startEvalCase was not called', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - // Should not throw - observer.onToolCall('bash', 'ls', 'ok', 10); - observer.onLlmCall('gpt-4', { input: 10, output: 5 }); - observer.finalizeEvalCase(1.0); - - expect(spans).toHaveLength(0); - }); - - it('getStreamCallbacks returns working callbacks', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), false); - - observer.startEvalCase('test-1', 'target'); - const callbacks = observer.getStreamCallbacks(); - - callbacks.onToolCallEnd?.('write_file', null, null, 100, 'tc-2'); - callbacks.onLlmCallEnd?.('gpt-4', { input: 50, output: 25 }); - - // root + tool + llm = 3 spans - expect(spans).toHaveLength(3); - expect(spans[1].name).toBe('execute_tool write_file'); - expect(spans[2].name).toBe('chat gpt-4'); - }); - - it('handles full lifecycle: start → tools → llm → finalize', () => { - const spans: MockSpan[] = []; - const observer = new OtelStreamingObserver(createMockTracer(spans), createMockApi(), true); - - observer.startEvalCase('lifecycle-test', 'claude-target', 'qa-suite'); - observer.onToolCall('search', { q: 'test' }, ['result1'], 200, 'tc-a'); - observer.onLlmCall('claude-sonnet-4-20250514', { input: 500, output: 100 }); - observer.onToolCall('write', { path: 'out.txt' }, 'ok', 50, 'tc-b'); - observer.onLlmCall('claude-sonnet-4-20250514', { input: 600, output: 150 }); - observer.finalizeEvalCase(1.0); - - // root + 2 tools + 2 llm = 5 spans - expect(spans).toHaveLength(5); - expect(spans[0].ended).toBe(true); - expect(spans[0].attributes['agentv.score']).toBe(1.0); - expect(spans.every((s) => s.ended)).toBe(true); - }); -}); diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 2dcba83d7..7b18bdf23 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -14,15 +14,7 @@ export default defineConfig({ }, target: 'node20', tsconfig: './tsconfig.build.json', - external: [ - '@opentelemetry/api', - '@opentelemetry/exporter-trace-otlp-http', - '@opentelemetry/resources', - '@opentelemetry/sdk-trace-node', - '@opentelemetry/semantic-conventions', - '@earendil-works/pi-coding-agent', - '@earendil-works/pi-ai', - ], + external: ['@earendil-works/pi-coding-agent', '@earendil-works/pi-ai'], outExtension({ format }) { return { js: format === 'cjs' ? '.cjs' : '.js',