diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs index 4185d972da4d..501375fecd80 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs @@ -35,6 +35,8 @@ async function run() { prompt: 'What is the weather in San Francisco?', }); }); + + await Sentry.flush(2000); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs index b6abe6fdf673..ef8cb19c3646 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs @@ -75,6 +75,8 @@ async function run() { prompt: 'Where is the third span?', }); }); + + await Sentry.flush(2000); } run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs similarity index 86% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs index 48a860c510c5..0cc510d5c9e6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs @@ -8,4 +8,5 @@ Sentry.init({ sendDefaultPii: true, transport: loggingTransport, traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming-with-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-truncation.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming-with-truncation.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-truncation.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs new file mode 100644 index 000000000000..cf42383e5c6e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs new file mode 100644 index 000000000000..a305d0bd2dd8 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs @@ -0,0 +1,46 @@ +import * as Sentry from '@sentry/node'; +import { generateText } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + Sentry.setTag('test-tag', 'test-value'); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + try { + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'tool-calls', + usage: { promptTokens: 15, completionTokens: 25 }, + text: 'Tool call completed!', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'getWeather', + args: '{ "location": "San Francisco" }', + }, + ], + }), + }), + tools: { + getWeather: { + parameters: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }, + }, + prompt: 'What is the weather in San Francisco?', + }); + } catch { + // Expected error - we want the spans to still be flushed + } + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-truncation.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-span-streaming.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-truncation.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs new file mode 100644 index 000000000000..ef8cb19c3646 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs @@ -0,0 +1,82 @@ +import * as Sentry from '@sentry/node'; +import { generateText } from 'ai'; +import { MockLanguageModelV1 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'First span here!', + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'Second span here!', + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'tool-calls', + usage: { promptTokens: 15, completionTokens: 25 }, + text: 'Tool call completed!', + toolCalls: [ + { + toolCallType: 'function', + toolCallId: 'call-1', + toolName: 'getWeather', + args: '{ "location": "San Francisco" }', + }, + ], + }), + }), + tools: { + getWeather: { + description: 'Get the current weather for a location', + parameters: z.object({ location: z.string() }), + execute: async args => { + return `Weather in ${args.location}: Sunny, 72°F`; + }, + }, + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: 'stop', + usage: { promptTokens: 10, completionTokens: 20 }, + text: 'Third span here!', + }), + }), + prompt: 'Where is the third span?', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts new file mode 100644 index 000000000000..66a96cad2317 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts @@ -0,0 +1,347 @@ +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +/** + * Helper to match a typed attribute value in a SerializedStreamedSpan. + * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction + * span `data` which stores values directly. + */ +function attr(value: unknown) { + return expect.objectContaining({ value }); +} + +describe('Vercel AI integration (streaming)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_SPANS_DEFAULT_PII_FALSE = { + items: expect.arrayContaining([ + // First span - invoke_agent for simple generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Second span - generate_content for simple generateText + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText.doGenerate'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Third span - invoke_agent for explicit telemetry generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool + // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_DEFAULT_PII_TRUE = { + items: expect.arrayContaining([ + // First span - invoke_agent with input/output messages (PII enabled) + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: attr(1), + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the first span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Second span - generate_content with input/output messages + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Third span - explicit telemetry invoke_agent with messages + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the second span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent with messages + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{ \\"location\\": \\"San Francisco\\" }"}],"finish_reason":"tool_call"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content with available_tools + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: expect.objectContaining({ + value: expect.stringContaining('getWeather'), + }), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool with description and input/output + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: attr('Get the current weather for a location'), + [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_ERROR_IN_TOOL = { + items: expect.arrayContaining([ + expect.objectContaining({ + name: 'invoke_agent', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: false', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: true', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { + test('normalizes error status in streaming mode', async () => { + await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); + }); + }); + + const streamingLongContent = 'A'.repeat(50_000); + + createEsmAndCjsTests(__dirname, 'scenario-truncation.mjs', 'instrument.mjs', (createRunner, test) => { + test('automatically disables truncation when span streaming is enabled', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + + const chatSpan = spans.find(s => + s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), + ); + expect(chatSpan).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario-truncation.mjs', 'instrument-with-truncation.mjs', (createRunner, test) => { + test('respects explicit enableTruncation: true even when span streaming is enabled', async () => { + await createRunner() + .expect({ + span: container => { + const spans = container.items; + + // With explicit enableTruncation: true, content should be truncated despite streaming. + const chatSpan = spans.find(s => + s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), + ); + expect(chatSpan).toBeDefined(); + expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( + streamingLongContent.length, + ); + }, + }) + .start() + .completed(); + }); + }); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs new file mode 100644 index 000000000000..0cc510d5c9e6 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs @@ -0,0 +1,12 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs new file mode 100644 index 000000000000..cf42383e5c6e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + traceLifecycle: 'stream', + integrations: [Sentry.vercelAIIntegration()], +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs new file mode 100644 index 000000000000..51bdae176158 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs @@ -0,0 +1,47 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + try { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + } catch { + // Expected error - we want the spans to still be flushed + } + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs new file mode 100644 index 000000000000..bf5e43a32e65 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs @@ -0,0 +1,95 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'First span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Second span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + description: 'Get the current weather for a location', + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Third span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the third span?', + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts new file mode 100644 index 000000000000..05226300e160 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts @@ -0,0 +1,335 @@ +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '@sentry/core'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE, + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +/** + * Helper to match a typed attribute value in a SerializedStreamedSpan. + * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction + * span `data` which stores values directly. + */ +function attr(value: unknown) { + return expect.objectContaining({ value }); +} + +describe('Vercel AI integration (streaming, V6)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_SPANS_DEFAULT_PII_FALSE = { + items: expect.arrayContaining([ + // First span - invoke_agent for simple generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + 'vercel.ai.request.headers.user-agent': expect.objectContaining({ value: expect.any(String) }), + }), + }), + // Second span - generate_content for simple generateText + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText.doGenerate'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Third span - invoke_agent for explicit telemetry generateText + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool + // Note: gen_ai.tool.description is NOT present when sendDefaultPii: false because ai.prompt.tools is not recorded + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_DEFAULT_PII_TRUE = { + items: expect.arrayContaining([ + // First span - invoke_agent with input/output messages (PII enabled) + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: attr(1), + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the first span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + 'vercel.ai.pipeline.name': attr('generateText'), + 'vercel.ai.streaming': attr(false), + }), + }), + // Second span - generate_content with input/output messages + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Third span - explicit telemetry invoke_agent with messages + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr('[{"role":"user","content":"Where is the second span?"}]'), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(10), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(20), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(30), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fourth span - tool call invoke_agent with messages (V6: no text part, only tool_call) + expect.objectContaining({ + name: 'invoke_agent', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ), + [GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]: attr( + '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', + ), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Fifth span - tool call generate_content with available_tools + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: expect.objectContaining({ + value: expect.stringContaining('getWeather'), + }), + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + // Sixth span - execute_tool with description and input/output + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]: attr('Get the current weather for a location'), + [GEN_AI_TOOL_INPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_OUTPUT_ATTRIBUTE]: expect.objectContaining({ value: expect.any(String) }), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + const EXPECTED_SPANS_ERROR_IN_TOOL = { + items: expect.arrayContaining([ + expect.objectContaining({ + name: 'invoke_agent', + attributes: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.invoke_agent'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'generate_content mock-model-id', + status: 'ok', + attributes: expect.objectContaining({ + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: attr('mock-model-id'), + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: attr(15), + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: attr(25), + [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: attr(40), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.generate_content'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + expect.objectContaining({ + name: 'execute_tool getWeather', + status: 'error', + attributes: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), + [GEN_AI_TOOL_NAME_ATTRIBUTE]: attr('getWeather'), + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: attr('function'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: attr('execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr('gen_ai.execute_tool'), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: attr('auto.vercelai.otel'), + }), + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: false', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates ai related spans in streaming mode with sendDefaultPii: true', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-error-in-tool.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('normalizes error status in streaming mode', async () => { + await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 5aa1dc8342a5..d75a1faf8ea0 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -983,51 +983,4 @@ describe('Vercel AI integration', () => { }); }, ); - - const streamingLongContent = 'A'.repeat(50_000); - - createEsmAndCjsTests(__dirname, 'scenario-span-streaming.mjs', 'instrument-streaming.mjs', (createRunner, test) => { - test('automatically disables truncation when span streaming is enabled', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items; - - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.includes(streamingLongContent), - ); - expect(chatSpan).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }); - - createEsmAndCjsTests( - __dirname, - 'scenario-span-streaming.mjs', - 'instrument-streaming-with-truncation.mjs', - (createRunner, test) => { - test('respects explicit enableTruncation: true even when span streaming is enabled', async () => { - await createRunner() - .expect({ - span: container => { - const spans = container.items; - - // With explicit enableTruncation: true, content should be truncated despite streaming. - const chatSpan = spans.find(s => - s.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value?.startsWith('[{"role":"user","content":"AAAA'), - ); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value.length).toBeLessThan( - streamingLongContent.length, - ); - }, - }) - .start() - .completed(); - }); - }, - ); }); diff --git a/packages/core/src/tracing/vercel-ai/constants.ts b/packages/core/src/tracing/vercel-ai/constants.ts index f1d9d3168e84..27c2b901554d 100644 --- a/packages/core/src/tracing/vercel-ai/constants.ts +++ b/packages/core/src/tracing/vercel-ai/constants.ts @@ -5,6 +5,13 @@ import type { ToolCallSpanContext } from './types'; // without keeping full Span objects (and their potentially large attributes) alive. export const toolCallSpanContextMap = new Map(); +// Used to make tool descriptions available to execute_tool spans in the span streaming path. +// Streamed spans are processed individually, so execute_tool spans cannot look up descriptions +// from their sibling doGenerate span on span end (as we do for transactions). +// Instead we store descriptions at spanStart and apply them in the processSpan hook. +// Stores parent_span_id -> Map +export const toolDescriptionMap = new Map>(); + /** Maps Vercel AI span names to standardized OpenTelemetry operation names. */ export const SPAN_TO_OPERATION_NAME = new Map([ ['ai.generateText', 'invoke_agent'], diff --git a/packages/core/src/tracing/vercel-ai/index.ts b/packages/core/src/tracing/vercel-ai/index.ts index 55b53c362612..c6ff4c784dde 100644 --- a/packages/core/src/tracing/vercel-ai/index.ts +++ b/packages/core/src/tracing/vercel-ai/index.ts @@ -4,7 +4,7 @@ import { getClient } from '../../currentScopes'; import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; import { shouldEnableTruncation } from '../ai/utils'; import type { Event } from '../../types-hoist/event'; -import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON } from '../../types-hoist/span'; +import type { Span, SpanAttributes, SpanAttributeValue, SpanJSON, StreamedSpanJSON } from '../../types-hoist/span'; import { spanToJSON } from '../../utils/spanUtils'; import { GEN_AI_EMBEDDINGS_INPUT_ATTRIBUTE, @@ -14,6 +14,7 @@ import { GEN_AI_REQUEST_MODEL_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, GEN_AI_TOOL_INPUT_ATTRIBUTE, GEN_AI_TOOL_NAME_ATTRIBUTE, GEN_AI_TOOL_OUTPUT_ATTRIBUTE, @@ -24,8 +25,9 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../ai/gen-ai-attributes'; -import { SPAN_TO_OPERATION_NAME, toolCallSpanContextMap } from './constants'; +import { SPAN_TO_OPERATION_NAME, toolCallSpanContextMap, toolDescriptionMap } from './constants'; import type { TokenSummary } from './types'; +import { hasSpanStreamingEnabled } from '../spans/hasSpanStreamingEnabled'; import { accumulateTokensForParent, applyAccumulatedTokens, @@ -233,19 +235,12 @@ function buildOutputMessages(attributes: Record): void { /** * Post-process spans emitted by the Vercel AI SDK. */ -function processEndedVercelAiSpan(span: SpanJSON): void { - const { data: attributes, origin } = span; - - if (origin !== 'auto.vercelai.otel') { - return; - } - - // The Vercel AI SDK sets span status to raw error message strings. - // Any such value should be normalized to a SpanStatusType value. We pick internal_error as it is the most generic. - if (span.status && span.status !== 'ok') { - span.status = 'internal_error'; - } - +/** + * Rename and normalize Vercel AI SDK attributes to OpenTelemetry semantic conventions. + * This is the shared attribute processing logic used by both the legacy event processor + * path (SpanJSON) and the streamed span path (StreamedSpanJSON). + */ +export function processVercelAiSpanAttributes(attributes: Record): void { renameAttributeKey(attributes, AI_USAGE_COMPLETION_TOKENS_ATTRIBUTE, GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE); renameAttributeKey(attributes, AI_USAGE_PROMPT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE); renameAttributeKey(attributes, AI_USAGE_CACHED_INPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_CACHED_ATTRIBUTE); @@ -338,6 +333,49 @@ function processEndedVercelAiSpan(span: SpanJSON): void { } } +function processEndedVercelAiSpan(span: SpanJSON): void { + const { data: attributes, origin } = span; + + if (origin !== 'auto.vercelai.otel') { + return; + } + + // The Vercel AI SDK sets span status to raw error message strings. + // Any such value should be normalized to a SpanStatusType value. We pick internal_error as it is the most generic. + if (span.status && span.status !== 'ok') { + span.status = 'internal_error'; + } + + processVercelAiSpanAttributes(attributes); +} + +function processVercelAiStreamedSpan(span: StreamedSpanJSON): void { + const attributes = span.attributes; + if (attributes?.[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN] !== 'auto.vercelai.otel') { + return; + } + + processVercelAiSpanAttributes(attributes); + + // Look up tool description from the toolDescriptionMap for execute_tool spans + if (attributes[SEMANTIC_ATTRIBUTE_SENTRY_OP] === 'gen_ai.execute_tool' && span.parent_span_id) { + const descriptions = toolDescriptionMap.get(span.parent_span_id); + + if (descriptions) { + const toolName = attributes[GEN_AI_TOOL_NAME_ATTRIBUTE]; + if (typeof toolName === 'string') { + const desc = descriptions.get(toolName); + if (desc) { + attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE] = desc; + } + } + } + } + + // Clean up tool descriptions when the parent span ends + toolDescriptionMap.delete(span.span_id); +} + /** * Renames an attribute key in the provided attributes object if the old key exists. * This function safely handles null and undefined values. @@ -418,6 +456,41 @@ function processGenerateSpan(span: Span, name: string, attributes: SpanAttribute if (modelId && operationName) { span.updateName(`${operationName} ${modelId}`); } + + // Store tool descriptions in the toolDescriptionMap so processSpan can apply them to execute_tool spans. + // This is only needed for span streaming (transaction path handles this separately) + const client = getClient(); + if ( + client && + hasSpanStreamingEnabled(client) && + attributes[AI_PROMPT_TOOLS_ATTRIBUTE] && + Array.isArray(attributes[AI_PROMPT_TOOLS_ATTRIBUTE]) + ) { + const descriptions = new Map(); + + // parse tool names and descriptions from tool string array + for (const toolStr of attributes[AI_PROMPT_TOOLS_ATTRIBUTE] as unknown[]) { + try { + const parsed = (typeof toolStr === 'string' ? JSON.parse(toolStr) : toolStr) as { + name?: string; + description?: string; + }; + if (parsed?.name && parsed?.description) { + descriptions.set(parsed.name, parsed.description); + } + } catch { + // ignore parse errors + } + } + if (descriptions.size > 0) { + // Tool call spans are siblings of doGenerate (both children of invoke_agent), + // so we key by the parent span ID (the invoke_agent span). + const parentSpanId = spanToJSON(span).parent_span_id; + if (parentSpanId) { + toolDescriptionMap.set(parentSpanId, descriptions); + } + } + } } /** @@ -427,9 +500,12 @@ export function addVercelAiProcessors(client: Client): void { client.on('spanStart', onVercelAiSpanStart); // Note: We cannot do this on `spanEnd`, because the span cannot be mutated anymore at this point client.addEventProcessor(Object.assign(vercelAiEventProcessor, { id: 'VercelAiEventProcessor' })); + client.on('processSpan', span => { + processVercelAiStreamedSpan(span); + }); } -function addProviderMetadataToAttributes(attributes: SpanAttributes): void { +function addProviderMetadataToAttributes(attributes: Record): void { const providerMetadata = attributes[AI_RESPONSE_PROVIDER_METADATA_ATTRIBUTE] as string | undefined; if (providerMetadata) { try { @@ -506,7 +582,11 @@ function addProviderMetadataToAttributes(attributes: SpanAttributes): void { /** * Sets an attribute only if the value is not null or undefined. */ -function setAttributeIfDefined(attributes: SpanAttributes, key: string, value: SpanAttributeValue | undefined): void { +function setAttributeIfDefined( + attributes: Record, + key: string, + value: SpanAttributeValue | undefined, +): void { if (value != null) { attributes[key] = value; }