From bae826f8b059b0ece0ac071b59ddbf7383e157bf Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 30 Apr 2026 12:17:15 +0200 Subject: [PATCH 01/11] test(vercelai): Add span streaming integration tests for event processor migration Closes https://github.com/getsentry/sentry-javascript/issues/20377 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../vercelai/scenario-error-in-tool.mjs | 2 + .../suites/tracing/vercelai/scenario.mjs | 2 + .../instrument-with-pii.mjs} | 1 + .../instrument-with-truncation.mjs} | 0 .../vercelai/span-streaming-v4/instrument.mjs | 11 + .../scenario-error-in-tool.mjs | 42 +++ .../scenario-truncation.mjs} | 0 .../vercelai/span-streaming-v4/scenario.mjs | 82 ++++ .../vercelai/span-streaming-v4/test.ts | 353 ++++++++++++++++++ .../span-streaming-v6/instrument-with-pii.mjs | 12 + .../vercelai/span-streaming-v6/instrument.mjs | 11 + .../scenario-error-in-tool.mjs | 43 +++ .../vercelai/span-streaming-v6/scenario.mjs | 95 +++++ .../vercelai/span-streaming-v6/test.ts | 340 +++++++++++++++++ .../suites/tracing/vercelai/test.ts | 47 --- 15 files changed, 994 insertions(+), 47 deletions(-) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{instrument-streaming.mjs => span-streaming-v4/instrument-with-pii.mjs} (86%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{instrument-streaming-with-truncation.mjs => span-streaming-v4/instrument-with-truncation.mjs} (100%) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs rename dev-packages/node-integration-tests/suites/tracing/vercelai/{scenario-span-streaming.mjs => span-streaming-v4/scenario-truncation.mjs} (100%) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs index 4185d972da4d..501375fecd80 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs @@ -35,6 +35,8 @@ async function run() { prompt: 'What is the weather in San Francisco?', }); }); + + await Sentry.flush(2000); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs index b6abe6fdf673..ef8cb19c3646 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs @@ -75,6 +75,8 @@ async function run() { prompt: 'Where is the third span?', }); }); + + await Sentry.flush(2000); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs similarity index 86% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs index 48a860c510c5..0cc510d5c9e6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs @@ -8,4 +8,5 @@ Sentry.init({ sendDefaultPii: true, transport: loggingTransport, traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming-with-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-truncation.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming-with-truncation.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-truncation.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs new file mode 100644 index 000000000000..cf42383e5c6e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs new file mode 100644 index 000000000000..501375fecd80 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs @@ -0,0 +1,42 @@ +import * as Sentry from '@sentry/node'; +import { generateText } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + Sentry.setTag('test-tag', 'test-value'); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'tool-calls', + usage: { promptTokens: 15, completionTokens: 25 }, + text: 'Tool call completed!', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'getWeather', + args: '{ "location": "San Francisco" }', + }, + ], + }), + }), + tools: { + getWeather: { + parameters: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }, + }, + prompt: 'What is the weather in San Francisco?', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-truncation.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-span-streaming.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-truncation.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs new file mode 100644 index 000000000000..ef8cb19c3646 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs @@ -0,0 +1,82 @@ +import * as Sentry from '@sentry/node'; +import { generateText } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'First span here!', + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'Second span here!', + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'tool-calls', + usage: { promptTokens: 15, completionTokens: 25 }, + text: 'Tool call completed!', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'getWeather', + args: '{ "location": "San Francisco" }', + }, + ], + }), + }), + tools: { + getWeather: { + description: 'Get the current weather for a location', + parameters: z.object({ location: z.string() }), + execute: async args => { + return `Weather in ${args.location}: Sunny, 72°F`; + }, + }, + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'Third span here!', + }), + }), + prompt: 'Where is the third span?', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts new file mode 100644 index 000000000000..66da1bb9f231 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -0,0 +1,353 @@ +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +/** + * Helper to match a typed attribute value in a SerializedStreamedSpan. + * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction + * span `data` which stores values directly. + */ +function attr(value: unknown) { + return expect.objectContaining({ value }); +} + +describe('Vercel AI integration (streaming)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_SPANS_DEFAULT_PII_FALSE = { + items: expect.arrayContaining([ + // First span - invoke_agent for simple generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Second span - generate_content for simple generateText + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText.doGenerate'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Third span - invoke_agent for explicit telemetry generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool + // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_DEFAULT_PII_TRUE = { + items: expect.arrayContaining([ + // First span - invoke_agent with input/output messages (PII enabled) + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: attr(1), + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the first span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Second span - generate_content with input/output messages + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Third span - explicit telemetry invoke_agent with messages + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the second span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent with messages + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{ \\"location\\": \\"San Francisco\\" }"}],"finish_reason":"tool_call"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content with available_tools + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: expect.objectContaining({ + value: expect.stringContaining('getWeather'), + }), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool with description and input/output + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: attr('Get the current weather for a location'), + [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_ERROR_IN_TOOL = { + items: expect.arrayContaining([ + expect.objectContaining({ + name: 'invoke_agent', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: false', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: true', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { + test('normalizes error status in streaming mode', async () => { + await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); + }); + }); + + // Truncation tests (moved from test.ts) + + const streamingLongContent = 'A'.repeat(50_000); + + createEsmAndCjsTests(__dirname, 'scenario-truncation.mjs', 'instrument.mjs', (createRunner, test) => { + test('automatically disables truncation when span streaming is enabled', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + + const chatSpan = spans.find(s => + s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + ); + expect(chatSpan).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario-truncation.mjs', 'instrument-with-truncation.mjs', (createRunner, test) => { + test('respects explicit enableTruncation: true even when span streaming is enabled', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + + // With explicit enableTruncation: true, content should be truncated despite streaming. + const chatSpan = spans.find(s => + s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + ); + expect(chatSpan).toBeDefined(); + expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + streamingLongContent.length, + ); + }, + }) + .start() + .completed(); + }); + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs new file mode 100644 index 000000000000..0cc510d5c9e6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs @@ -0,0 +1,12 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs new file mode 100644 index 000000000000..cf42383e5c6e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs new file mode 100644 index 000000000000..c6d299c3b90e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs @@ -0,0 +1,43 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs new file mode 100644 index 000000000000..bf5e43a32e65 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs @@ -0,0 +1,95 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'First span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Second span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + description: 'Get the current weather for a location', + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Third span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the third span?', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts new file mode 100644 index 000000000000..a69a9ddc1d60 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts @@ -0,0 +1,340 @@ +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +/** + * Helper to match a typed attribute value in a SerializedStreamedSpan. + * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction + * span `data` which stores values directly. + */ +function attr(value: unknown) { + return expect.objectContaining({ value }); +} + +describe('Vercel AI integration (streaming, V6)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_SPANS_DEFAULT_PII_FALSE = { + items: expect.arrayContaining([ + // First span - invoke_agent for simple generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + 'vercel.ai.request.headers.user-agent': expect.objectContaining({ value: expect.any(String) }), + }), + }), + // Second span - generate_content for simple generateText + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText.doGenerate'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Third span - invoke_agent for explicit telemetry generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool + // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_DEFAULT_PII_TRUE = { + items: expect.arrayContaining([ + // First span - invoke_agent with input/output messages (PII enabled) + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: attr(1), + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the first span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Second span - generate_content with input/output messages + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Third span - explicit telemetry invoke_agent with messages + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the second span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent with messages (V6: no text part, only tool_call) + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content with available_tools + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: expect.objectContaining({ + value: expect.stringContaining('getWeather'), + }), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool with description and input/output + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: attr('Get the current weather for a location'), + [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_ERROR_IN_TOOL = { + items: expect.arrayContaining([ + expect.objectContaining({ + name: 'invoke_agent', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: false', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: true', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-error-in-tool.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('normalizes error status in streaming mode', async () => { + await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 5aa1dc8342a5..d75a1faf8ea0 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -983,51 +983,4 @@ describe('Vercel AI integration', () => { }); }, ); - - const streamingLongContent = 'A'.repeat(50_000); - - createEsmAndCjsTests(__dirname, 'scenario-span-streaming.mjs', 'instrument-streaming.mjs', (createRunner, test) => { - test('automatically disables truncation when span streaming is enabled', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items; - - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }); - - createEsmAndCjsTests( - __dirname, - 'scenario-span-streaming.mjs', - 'instrument-streaming-with-truncation.mjs', - (createRunner, test) => { - test('respects explicit enableTruncation: true even when span streaming is enabled', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items; - - // With explicit enableTruncation: true, content should be truncated despite streaming. - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), - ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); - }, - }) - .start() - .completed(); - }); - }, - ); }); From 0ac60620b67a9b3750ec5ae2786a7f74a27caf27 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 30 Apr 2026 13:05:54 +0200 Subject: [PATCH 02/11] renames --- packages/core/src/index.ts | 2 +- packages/core/src/tracing/vercel-ai/index.ts | 55 ++++++++++++++------ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4d80ea02ed33..29ce36b3662c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -166,7 +166,7 @@ export { export * as metrics from './metrics/public-api'; export type { MetricOptions } from './metrics/public-api'; export { createConsolaReporter } from './integrations/consola'; -export { addVercelAiProcessors } from './tracing/vercel-ai'; +export { addVercelAiProcessors, processVercelAiSpanAttributes } from './tracing/vercel-ai'; export { _INTERNAL_getSpanContextForToolCallId, _INTERNAL_cleanupToolCallSpanContext } from './tracing/vercel-ai/utils'; export { toolCallSpanContextMap as _INTERNAL_toolCallSpanContextMap } from './tracing/vercel-ai/constants'; export { instrumentOpenAiClient } from './tracing/openai'; diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 55b53c362612..b8e4aaf0afaa 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -4,7 +4,7 @@ import { getClient } from '../../currentScopes'; import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; import { shouldEnableTruncation } from '../ai/utils'; import type { Event } from '../../types-hoist/event'; -import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON } from '../../types-hoist/span'; +import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON, SpanOrigin, StreamedSpanJSON } from '../../types-hoist/span'; import { spanToJSON } from '../../utils/spanUtils'; import { GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, @@ -233,19 +233,12 @@ function buildOutputMessages(attributes: Record): void { /** * Post-process spans emitted by the Vercel AI SDK. */ -function processEndedVercelAiSpan(span: SpanJSON): void { - const { data: attributes, origin } = span; - - if (origin !== 'auto.vercelai.otel') { - return; - } - - // The Vercel AI SDK sets span status to raw error message strings. - // Any such value should be normalized to a SpanStatusType value. We pick internal_error as it is the most generic. - if (span.status && span.status !== 'ok') { - span.status = 'internal_error'; - } - +/** + * Rename and normalize Vercel AI SDK attributes to OpenTelemetry semantic conventions. + * This is the shared attribute processing logic used by both the legacy event processor + * path (SpanJSON) and the streamed span path (StreamedSpanJSON). + */ +export function processVercelAiSpanAttributes(attributes: Record): void { renameAttributeKey(attributes, AI_USAGE_COMPLETION_TOKENS_ATTRIBUTE, GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE); renameAttributeKey(attributes, AI_USAGE_PROMPT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE); renameAttributeKey(attributes, AI_USAGE_CACHED_INPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_CACHED_ATTRIBUTE); @@ -338,6 +331,31 @@ function processEndedVercelAiSpan(span: SpanJSON): void { } } +function processEndedVercelAiSpan(span: SpanJSON): void { + const { data: attributes, origin } = span; + + if (origin !== 'auto.vercelai.otel') { + return; + } + + // The Vercel AI SDK sets span status to raw error message strings. + // Any such value should be normalized to a SpanStatusType value. We pick internal_error as it is the most generic. + if (span.status && span.status !== 'ok') { + span.status = 'internal_error'; + } + + processVercelAiSpanAttributes(attributes); +} + +function processEndedVercelAiStreamedSpan(span: StreamedSpanJSON): void { + const attributes = span.attributes; + if (!attributes || attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] !== 'auto.vercelai.otel') { + return; + } + + processVercelAiSpanAttributes(attributes); +} + /** * Renames an attribute key in the provided attributes object if the old key exists. * This function safely handles null and undefined values. @@ -427,9 +445,14 @@ export function addVercelAiProcessors(client: Client): void { client.on('spanStart', onVercelAiSpanStart); // Note: We cannot do this on `spanEnd`, because the span cannot be mutated anymore at this point client.addEventProcessor(Object.assign(vercelAiEventProcessor, { id: 'VercelAiEventProcessor' })); + client.on('processSpan', span => { + console.log('processSpan', span); + processEndedVercelAiStreamedSpan(span); + console.log('after processSpan', span); + }); } -function addProviderMetadataToAttributes(attributes: SpanAttributes): void { +function addProviderMetadataToAttributes(attributes: Record): void { const providerMetadata = attributes[AI_RESPONSE_PROVIDER_METADATA_ATTRIBUTE] as string | undefined; if (providerMetadata) { try { @@ -506,7 +529,7 @@ function addProviderMetadataToAttributes(attributes: SpanAttributes): void { /** * Sets an attribute only if the value is not null or undefined. */ -function setAttributeIfDefined(attributes: SpanAttributes, key: string, value: SpanAttributeValue | undefined): void { +function setAttributeIfDefined(attributes: Record, key: string, value: SpanAttributeValue | undefined): void { if (value != null) { attributes[key] = value; } From ad5961aef1e4e0bfba6dc7dc8c120ae9935af4a0 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 5 May 2026 14:25:17 +0200 Subject: [PATCH 03/11] formatting --- packages/core/src/tracing/vercel-ai/index.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index b8e4aaf0afaa..3548a9aad1ad 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -4,7 +4,13 @@ import { getClient } from '../../currentScopes'; import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; import { shouldEnableTruncation } from '../ai/utils'; import type { Event } from '../../types-hoist/event'; -import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON, SpanOrigin, StreamedSpanJSON } from '../../types-hoist/span'; +import type { + Span, + SpanAttributes, + SpanAttributeValue, + SpanJSON, + StreamedSpanJSON, +} from '../../types-hoist/span'; import { spanToJSON } from '../../utils/spanUtils'; import { GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, @@ -349,7 +355,7 @@ function processEndedVercelAiSpan(span: SpanJSON): void { function processEndedVercelAiStreamedSpan(span: StreamedSpanJSON): void { const attributes = span.attributes; - if (!attributes || attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] !== 'auto.vercelai.otel') { + if (attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] !== 'auto.vercelai.otel') { return; } @@ -529,7 +535,11 @@ function addProviderMetadataToAttributes(attributes: Record): v /** * Sets an attribute only if the value is not null or undefined. */ -function setAttributeIfDefined(attributes: Record, key: string, value: SpanAttributeValue | undefined): void { +function setAttributeIfDefined( + attributes: Record, + key: string, + value: SpanAttributeValue | undefined, +): void { if (value != null) { attributes[key] = value; } From 8e1ba0a079b0dbcbd782fb442b70486babc176dc Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 5 May 2026 19:17:10 +0200 Subject: [PATCH 04/11] implement --- .../core/src/tracing/vercel-ai/constants.ts | 4 ++ packages/core/src/tracing/vercel-ai/index.ts | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/packages/core/src/tracing/vercel-ai/constants.ts b/packages/core/src/tracing/vercel-ai/constants.ts index f1d9d3168e84..9215cb3590e0 100644 --- a/packages/core/src/tracing/vercel-ai/constants.ts +++ b/packages/core/src/tracing/vercel-ai/constants.ts @@ -5,6 +5,10 @@ import type { ToolCallSpanContext } from './types'; // without keeping full Span objects (and their potentially large attributes) alive. export const toolCallSpanContextMap = new Map(); +// Maps span_id → Map. +// Populated at spanStart of doGenerate spans, consumed at processSpan of execute_tool spans. +export const toolDescriptionMap = new Map>(); + /** Maps Vercel AI span names to standardized OpenTelemetry operation names. */ export const SPAN_TO_OPERATION_NAME = new Map([ ['ai.generateText', 'invoke_agent'], diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 3548a9aad1ad..af6e42e6ba5d 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -20,6 +20,7 @@ import { GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, GEN_AI_TOOL_INPUT_ATTRIBUTE, GEN_AI_TOOL_NAME_ATTRIBUTE, GEN_AI_TOOL_OUTPUT_ATTRIBUTE, @@ -30,7 +31,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../ai/gen-ai-attributes'; -import { SPAN_TO_OPERATION_NAME, toolCallSpanContextMap } from './constants'; +import { SPAN_TO_OPERATION_NAME, toolCallSpanContextMap, toolDescriptionMap } from './constants'; import type { TokenSummary } from './types'; import { accumulateTokensForParent, @@ -360,6 +361,23 @@ function processEndedVercelAiStreamedSpan(span: StreamedSpanJSON): void { } processVercelAiSpanAttributes(attributes); + + // Look up tool description from the side-channel for execute_tool spans + if (attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP] === 'gen_ai.execute_tool' && span.parent_span_id) { + const descriptions = toolDescriptionMap.get(span.parent_span_id); + if (descriptions) { + const toolName = attributes[GEN_AI_TOOL_NAME_ATTRIBUTE]; + if (typeof toolName === 'string') { + const desc = descriptions.get(toolName); + if (desc) { + attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE] = desc; + } + } + } + } + + // Clean up tool descriptions when the parent span ends + toolDescriptionMap.delete(span.span_id); } /** @@ -442,6 +460,29 @@ function processGenerateSpan(span: Span, name: string, attributes: SpanAttribute if (modelId && operationName) { span.updateName(`${operationName} ${modelId}`); } + + // Store tool descriptions for the side-channel so processSpan can apply them to execute_tool spans + if (attributes[AI_PROMPT_TOOLS_ATTRIBUTE] && Array.isArray(attributes[AI_PROMPT_TOOLS_ATTRIBUTE])) { + const descriptions = new Map(); + for (const toolStr of attributes[AI_PROMPT_TOOLS_ATTRIBUTE] as unknown[]) { + try { + const parsed = typeof toolStr === 'string' ? JSON.parse(toolStr) : toolStr; + if (parsed?.name && parsed?.description) { + descriptions.set(parsed.name as string, parsed.description as string); + } + } catch { + // ignore parse errors + } + } + if (descriptions.size > 0) { + // Tool call spans are siblings of doGenerate (both children of invoke_agent), + // so we key by the parent span ID (the invoke_agent span). + const parentSpanId = spanToJSON(span).parent_span_id; + if (parentSpanId) { + toolDescriptionMap.set(parentSpanId, descriptions); + } + } + } } /** From 597fa3f655e11d0111c3c17bc169bb736a932cdc Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 5 May 2026 19:18:40 +0200 Subject: [PATCH 05/11] yf --- packages/core/src/tracing/vercel-ai/index.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index af6e42e6ba5d..401247b84f5c 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -4,13 +4,7 @@ import { getClient } from '../../currentScopes'; import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; import { shouldEnableTruncation } from '../ai/utils'; import type { Event } from '../../types-hoist/event'; -import type { - Span, - SpanAttributes, - SpanAttributeValue, - SpanJSON, - StreamedSpanJSON, -} from '../../types-hoist/span'; +import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON, StreamedSpanJSON } from '../../types-hoist/span'; import { spanToJSON } from '../../utils/spanUtils'; import { GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, @@ -493,9 +487,7 @@ export function addVercelAiProcessors(client: Client): void { // Note: We cannot do this on `spanEnd`, because the span cannot be mutated anymore at this point client.addEventProcessor(Object.assign(vercelAiEventProcessor, { id: 'VercelAiEventProcessor' })); client.on('processSpan', span => { - console.log('processSpan', span); processEndedVercelAiStreamedSpan(span); - console.log('after processSpan', span); }); } From 8effee826761cb8f67e5ea47a6292aa7d01406f9 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 5 May 2026 19:40:56 +0200 Subject: [PATCH 06/11] feat(vercelai): Migrate event processor to processSpan hook with tool description side-channel Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/core/src/tracing/vercel-ai/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 401247b84f5c..194b0251bef4 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -460,9 +460,12 @@ function processGenerateSpan(span: Span, name: string, attributes: SpanAttribute const descriptions = new Map(); for (const toolStr of attributes[AI_PROMPT_TOOLS_ATTRIBUTE] as unknown[]) { try { - const parsed = typeof toolStr === 'string' ? JSON.parse(toolStr) : toolStr; + const parsed = (typeof toolStr === 'string' ? JSON.parse(toolStr) : toolStr) as { + name?: string; + description?: string; + }; if (parsed?.name && parsed?.description) { - descriptions.set(parsed.name as string, parsed.description as string); + descriptions.set(parsed.name, parsed.description); } } catch { // ignore parse errors From 9006d557ee255e42fdecbde637a1d7714ecbd74d Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 5 May 2026 19:48:14 +0200 Subject: [PATCH 07/11] fix(vercelai): Fix error-in-tool streaming scenario and test expectations The error scenario needs a try/catch so the process stays alive for flush. The invoke_agent span doesn't have token attributes when the tool errors, so remove those from the error test expectations. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../scenario-error-in-tool.mjs | 52 ++++++++-------- .../vercelai/span-streaming-v4/test.ts | 4 -- .../scenario-error-in-tool.mjs | 60 ++++++++++--------- .../vercelai/span-streaming-v6/test.ts | 4 -- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs index 501375fecd80..a305d0bd2dd8 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs @@ -7,33 +7,37 @@ async function run() { Sentry.setTag('test-tag', 'test-value'); await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], + try { + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'tool-calls', + usage: { promptTokens: 15, completionTokens: 25 }, + text: 'Tool call completed!', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'getWeather', + args: '{ "location": "San Francisco" }', + }, + ], + }), }), - }), - tools: { - getWeather: { - parameters: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); + tools: { + getWeather: { + parameters: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, }, }, - }, - prompt: 'What is the weather in San Francisco?', - }); + prompt: 'What is the weather in San Francisco?', + }); + } catch { + // Expected error - we want the spans to still be flushed + } }); await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts index 66da1bb9f231..1869252fdc05 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -252,10 +252,6 @@ describe('Vercel AI integration (streaming)', () => { name: 'invoke_agent', status: 'error', attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs index c6d299c3b90e..51bdae176158 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs @@ -5,36 +5,40 @@ import { z } from 'zod'; async function run() { await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), + try { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, }, - ], - warnings: [], + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), }), - }), - tools: { - getWeather: tool({ - inputSchema: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); + tools: { + getWeather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + } catch { + // Expected error - we want the spans to still be flushed + } }); await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts index a69a9ddc1d60..c6d8d7365ebc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts @@ -253,10 +253,6 @@ describe('Vercel AI integration (streaming, V6)', () => { name: 'invoke_agent', status: 'error', attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), From 761e47e3d642082dc1fa9d44e4550516a744d5ff Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 6 May 2026 09:32:29 +0200 Subject: [PATCH 08/11] fix(vercelai): Don't assert invoke_agent error status in v6 streaming test In Vercel AI v6, the invoke_agent span does not get error status when a tool errors (matching the non-streaming v6 test behavior). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../suites/tracing/vercelai/span-streaming-v6/test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts index c6d8d7365ebc..05226300e160 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts @@ -251,7 +251,6 @@ describe('Vercel AI integration (streaming, V6)', () => { items: expect.arrayContaining([ expect.objectContaining({ name: 'invoke_agent', - status: 'error', attributes: expect.objectContaining({ [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), From 7d4f4c95e9c4e1a8f14088231b2008a34fc22401 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 6 May 2026 09:38:07 +0200 Subject: [PATCH 09/11] chore: Remove stale comment Co-Authored-By: Claude Opus 4.6 (1M context) --- .../suites/tracing/vercelai/span-streaming-v4/test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts index 1869252fdc05..66a96cad2317 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -303,8 +303,6 @@ describe('Vercel AI integration (streaming)', () => { }); }); - // Truncation tests (moved from test.ts) - const streamingLongContent = 'A'.repeat(50_000); createEsmAndCjsTests(__dirname, 'scenario-truncation.mjs', 'instrument.mjs', (createRunner, test) => { From 59d181c514d5a12bcffa1f41281f3a0e9484af22 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 6 May 2026 10:16:54 +0200 Subject: [PATCH 10/11] cleaning up a bit --- packages/core/src/tracing/vercel-ai/constants.ts | 7 +++++-- packages/core/src/tracing/vercel-ai/index.ts | 11 +++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/core/src/tracing/vercel-ai/constants.ts b/packages/core/src/tracing/vercel-ai/constants.ts index 9215cb3590e0..27c2b901554d 100644 --- a/packages/core/src/tracing/vercel-ai/constants.ts +++ b/packages/core/src/tracing/vercel-ai/constants.ts @@ -5,8 +5,11 @@ import type { ToolCallSpanContext } from './types'; // without keeping full Span objects (and their potentially large attributes) alive. export const toolCallSpanContextMap = new Map(); -// Maps span_id → Map. -// Populated at spanStart of doGenerate spans, consumed at processSpan of execute_tool spans. +// Used to make tool descriptions available to execute_tool spans in the span streaming path. +// Streamed spans are processed individually, so execute_tool spans cannot look up descriptions +// from their sibling doGenerate span on span end (as we do for transactions). +// Instead we store descriptions at spanStart and apply them in the processSpan hook. +// Stores parent_span_id -> Map export const toolDescriptionMap = new Map>(); /** Maps Vercel AI span names to standardized OpenTelemetry operation names. */ diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 194b0251bef4..bd851b070f69 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -348,7 +348,7 @@ function processEndedVercelAiSpan(span: SpanJSON): void { processVercelAiSpanAttributes(attributes); } -function processEndedVercelAiStreamedSpan(span: StreamedSpanJSON): void { +function processVercelAiStreamedSpan(span: StreamedSpanJSON): void { const attributes = span.attributes; if (attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] !== 'auto.vercelai.otel') { return; @@ -356,9 +356,10 @@ function processEndedVercelAiStreamedSpan(span: StreamedSpanJSON): void { processVercelAiSpanAttributes(attributes); - // Look up tool description from the side-channel for execute_tool spans + // Look up tool description from the toolDescriptionMap for execute_tool spans if (attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP] === 'gen_ai.execute_tool' && span.parent_span_id) { const descriptions = toolDescriptionMap.get(span.parent_span_id); + if (descriptions) { const toolName = attributes[GEN_AI_TOOL_NAME_ATTRIBUTE]; if (typeof toolName === 'string') { @@ -455,9 +456,11 @@ function processGenerateSpan(span: Span, name: string, attributes: SpanAttribute span.updateName(`${operationName} ${modelId}`); } - // Store tool descriptions for the side-channel so processSpan can apply them to execute_tool spans + // Store tool descriptions in the toolDescriptionMap so processSpan can apply them to execute_tool spans if (attributes[AI_PROMPT_TOOLS_ATTRIBUTE] && Array.isArray(attributes[AI_PROMPT_TOOLS_ATTRIBUTE])) { const descriptions = new Map(); + + // parse tool names and descriptions from tool string array for (const toolStr of attributes[AI_PROMPT_TOOLS_ATTRIBUTE] as unknown[]) { try { const parsed = (typeof toolStr === 'string' ? JSON.parse(toolStr) : toolStr) as { @@ -490,7 +493,7 @@ export function addVercelAiProcessors(client: Client): void { // Note: We cannot do this on `spanEnd`, because the span cannot be mutated anymore at this point client.addEventProcessor(Object.assign(vercelAiEventProcessor, { id: 'VercelAiEventProcessor' })); client.on('processSpan', span => { - processEndedVercelAiStreamedSpan(span); + processVercelAiStreamedSpan(span); }); } From 44cc78961d6507fe48192e2aaa56c38ca6f61040 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 6 May 2026 12:56:57 +0200 Subject: [PATCH 11/11] . --- packages/core/src/index.ts | 2 +- packages/core/src/tracing/vercel-ai/index.ts | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f7ff82168f62..19fc53365bb5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -170,7 +170,7 @@ export { export * as metrics from './metrics/public-api'; export type { MetricOptions } from './metrics/public-api'; export { createConsolaReporter } from './integrations/consola'; -export { addVercelAiProcessors, processVercelAiSpanAttributes } from './tracing/vercel-ai'; +export { addVercelAiProcessors } from './tracing/vercel-ai'; export { _INTERNAL_getSpanContextForToolCallId, _INTERNAL_cleanupToolCallSpanContext } from './tracing/vercel-ai/utils'; export { toolCallSpanContextMap as _INTERNAL_toolCallSpanContextMap } from './tracing/vercel-ai/constants'; export { instrumentOpenAiClient } from './tracing/openai'; diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index bd851b070f69..c6ff4c784dde 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -27,6 +27,7 @@ import { } from '../ai/gen-ai-attributes'; import { SPAN_TO_OPERATION_NAME, toolCallSpanContextMap, toolDescriptionMap } from './constants'; import type { TokenSummary } from './types'; +import { hasSpanStreamingEnabled } from '../spans/hasSpanStreamingEnabled'; import { accumulateTokensForParent, applyAccumulatedTokens, @@ -456,8 +457,15 @@ function processGenerateSpan(span: Span, name: string, attributes: SpanAttribute span.updateName(`${operationName} ${modelId}`); } - // Store tool descriptions in the toolDescriptionMap so processSpan can apply them to execute_tool spans - if (attributes[AI_PROMPT_TOOLS_ATTRIBUTE] && Array.isArray(attributes[AI_PROMPT_TOOLS_ATTRIBUTE])) { + // Store tool descriptions in the toolDescriptionMap so processSpan can apply them to execute_tool spans. + // This is only needed for span streaming (transaction path handles this separately) + const client = getClient(); + if ( + client && + hasSpanStreamingEnabled(client) && + attributes[AI_PROMPT_TOOLS_ATTRIBUTE] && + Array.isArray(attributes[AI_PROMPT_TOOLS_ATTRIBUTE]) + ) { const descriptions = new Map(); // parse tool names and descriptions from tool string array