diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs deleted file mode 100644 index c0abcf114ae1..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs +++ /dev/null @@ -1,12 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - // inputs and outputs are enabeld by default when opting into dataCollection - dataCollection: {}, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs deleted file mode 100644 index a305d0bd2dd8..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs +++ /dev/null @@ -1,46 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText } from 'ai'; -import { MockLanguageModelV1 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - Sentry.setTag('test-tag', 'test-value'); - - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - try { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], - }), - }), - tools: { - getWeather: { - parameters: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }, - }, - prompt: 'What is the weather in San Francisco?', - }); - } catch { - // Expected error - we want the spans to still be flushed - } - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs deleted file mode 100644 index ef8cb19c3646..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs +++ /dev/null @@ -1,82 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText } from 'ai'; -import { MockLanguageModelV1 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'First span here!', - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'Second span here!', - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], - }), - }), - tools: { - getWeather: { - description: 'Get the current weather for a location', - parameters: z.object({ location: z.string() }), - execute: async args => { - return `Weather in ${args.location}: Sunny, 72°F`; - }, - }, - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'Third span here!', - }), - }), - prompt: 'Where is the third span?', - }); - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts deleted file mode 100644 index 50385f0e0c40..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - GEN_AI_TOOL_DEFINITIONS, - GEN_AI_TOOL_DESCRIPTION, - GEN_AI_TOOL_NAME, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -/** - * Helper to match a typed attribute value in a SerializedStreamedSpan. - * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction - * span `data` which stores values directly. - */ -function attr(value: unknown) { - return expect.objectContaining({ value }); -} - -const expectedOrigin = 'auto.vercelai.channel'; - -// The channel path serializes the SDK-parsed args object (compact). -const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; - -describe('Vercel AI integration (streaming v4)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - const EXPECTED_SPANS_DEFAULT_PII_FALSE = { - items: expect.arrayContaining([ - // First span - invoke_agent for simple generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Second span - generate_content for simple generateText - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Third span - invoke_agent for explicit telemetry generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fourth span - tool call invoke_agent - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fifth span - tool call generate_content - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Sixth span - execute_tool - // Note: gen_ai.tool.description is NOT present when genAI recording disabled because ai.prompt.tools is not recorded - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_DEFAULT_PII_TRUE = { - items: expect.arrayContaining([ - // First span - invoke_agent with input/output messages (PII enabled) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the first span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Second span - generate_content with input/output messages - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Third span - explicit telemetry invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the second span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fourth span - tool call invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"What is the weather in San Francisco?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fifth span - tool call generate_content with tool definitions - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_DEFINITIONS]: expect.objectContaining({ - value: expect.stringContaining('getWeather'), - }), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Sixth span - execute_tool with description and input/output - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_DESCRIPTION]: attr('Get the current weather for a location'), - [GEN_AI_TOOL_CALL_ARGUMENTS]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_TOOL_CALL_RESULT]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_ERROR_IN_TOOL = { - items: expect.arrayContaining([ - expect.objectContaining({ - name: 'invoke_agent', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording disabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording enabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { - test('normalizes error status in streaming mode', async () => { - await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); - }); - }); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs deleted file mode 100644 index 51bdae176158..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - try { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - inputSchema: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - } catch { - // Expected error - we want the spans to still be flushed - } - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs deleted file mode 100644 index bf5e43a32e65..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'First span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Second span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - description: 'Get the current weather for a location', - inputSchema: z.object({ location: z.string() }), - execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Third span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the third span?', - }); - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts deleted file mode 100644 index 68c88b131783..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - GEN_AI_TOOL_DEFINITIONS, - GEN_AI_TOOL_DESCRIPTION, - GEN_AI_TOOL_NAME, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -/** - * Helper to match a typed attribute value in a SerializedStreamedSpan. - * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction - * span `data` which stores values directly. - */ -function attr(value: unknown) { - return expect.objectContaining({ value }); -} - -describe('Vercel AI integration (streaming, v6)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - const origin = 'auto.vercelai.channel'; - - const EXPECTED_SPANS_DEFAULT_PII_FALSE = { - items: expect.arrayContaining([ - // First span - invoke_agent for simple generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Second span - generate_content for simple generateText - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Third span - invoke_agent for explicit telemetry generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fourth span - tool call invoke_agent - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fifth span - tool call generate_content - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Sixth span - execute_tool - // Note: gen_ai.tool.description is NOT present when genAI recording disabled because ai.prompt.tools is not recorded - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_DEFAULT_PII_TRUE = { - items: expect.arrayContaining([ - // First span - invoke_agent with input/output messages (PII enabled) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the first span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Second span - generate_content with input/output messages - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Third span - explicit telemetry invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the second span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fourth span - tool call invoke_agent with messages (V6: no text part, only tool_call) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"What is the weather in San Francisco?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fifth span - tool call generate_content with tool definitions - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_DEFINITIONS]: expect.objectContaining({ - value: expect.stringContaining('getWeather'), - }), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Sixth span - execute_tool with description and input/output - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_DESCRIPTION]: attr('Get the current weather for a location'), - [GEN_AI_TOOL_CALL_ARGUMENTS]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_TOOL_CALL_RESULT]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_ERROR_IN_TOOL = { - items: expect.arrayContaining([ - expect.objectContaining({ - name: 'invoke_agent', - attributes: expect.objectContaining({ - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording disabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording enabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - function spanWithMetadata(name: string, op: string, requestId: string, extra: Record = {}) { - return expect.objectContaining({ - name, - attributes: expect.objectContaining({ - 'vercel.ai.telemetry.metadata.requestId': attr(requestId), - 'vercel.ai.telemetry.metadata.tenantId': attr('acme'), - ...extra, - [SENTRY_OP]: attr(op), - }), - }); - } - - const EXPECTED_SPANS_TELEMETRY_METADATA = { - items: expect.arrayContaining([ - spanWithMetadata('invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate', { - 'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' }, - }), - spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate', { - 'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' }, - }), - spanWithMetadata('invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'), - spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'), - spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'), - spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'), - ]), - }; - - createEsmAndCjsTests( - __dirname, - 'scenario-telemetry-metadata.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('records experimental_telemetry.metadata as span attributes', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_TELEMETRY_METADATA }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-error-in-tool.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('normalizes error status in streaming mode', async () => { - await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs similarity index 82% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs index bdf24d6bbd0e..43ea2c5b8d86 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs @@ -6,6 +6,6 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, + dataCollection: {}, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-conversation-id.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-conversation-id.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-conversation-id.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-conversation-id.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-embeddings.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-embeddings.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool-express.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool-express.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool-express.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool-express.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-generate-object.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-generate-object.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-generate-object.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-generate-object.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-stream-text.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-stream-text.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-system-instructions.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-system-instructions.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-system-instructions.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-system-instructions.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts similarity index 57% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts index e6ec70fa35ff..99604cb8a599 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts @@ -1,8 +1,10 @@ import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, @@ -16,12 +18,15 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; -import { getStringAttributeValue } from '../../../utils'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; +import { getStringAttributeValue } from '../../../../utils'; const expectedOrigin = 'auto.vercelai.channel'; +const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; describe('Vercel AI integration (v4)', () => { afterAll(() => { @@ -42,17 +47,17 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); const firstGenerateContentSpan = container.items.find( span => @@ -61,16 +66,33 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); const secondInvokeAgentSpan = container.items.find( span => @@ -78,13 +100,29 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); @@ -94,21 +132,25 @@ describe('Vercel AI integration (v4)', () => { span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Second span here!'); const toolInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); @@ -117,17 +159,37 @@ describe('Vercel AI integration (v4)', () => { span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -150,14 +212,28 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); @@ -167,15 +243,45 @@ describe('Vercel AI integration (v4)', () => { span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('First span here!'); + expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]?.value).toEqual(expect.any(String)); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + }); const secondInvokeAgentSpan = container.items.find( span => @@ -183,23 +289,42 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + }); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolInvokeAgentSpan = container.items.find( span => @@ -207,38 +332,76 @@ describe('Vercel AI integration (v4)', () => { span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 40 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 25 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 15 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeDefined(); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, + }); const toolGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_TOOL_DEFINITIONS]?.value)?.includes('getWeather'), ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toContain('getWeather'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ + type: 'string', + value: expect.stringContaining('getWeather'), + }); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'call-1', + }); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', ); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]).toBeDefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_RESULT]).toBeDefined(); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toEqual(expect.any(String)); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toEqual(expect.any(String)); }, }) .start() @@ -266,25 +429,37 @@ describe('Vercel AI integration (v4)', () => { const invokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.status === 'error', ); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.status).toBe('error'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 40 }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 25 }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 15 }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -296,10 +471,8 @@ describe('Vercel AI integration (v4)', () => { .start() .completed(); - expect(transactionEvent).toBeDefined(); expect(transactionEvent!.transaction).toBe('main'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError`. The @@ -339,23 +512,16 @@ describe('Vercel AI integration (v4)', () => { const invokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.status === 'error', ); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.status).toBe('error'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -369,11 +535,9 @@ describe('Vercel AI integration (v4)', () => { await runner.makeRequest('get', '/test/error-in-tool', { expectError: true }); await runner.completed(); - expect(transactionEvent).toBeDefined(); expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); - expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); @@ -411,17 +575,13 @@ describe('Vercel AI integration (v4)', () => { span: container => { expect(container.items).toHaveLength(2); const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); }, }) .start() @@ -438,21 +598,17 @@ describe('Vercel AI integration (v4)', () => { span: container => { expect(container.items).toHaveLength(2); const embedSpan = container.items.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); - expect(embedSpan).toBeDefined(); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); - expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); expect(embedSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); - expect(embedSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); const embedManySpan = container.items.find( span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20, ); - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); + expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); }, }) .start() @@ -470,20 +626,16 @@ describe('Vercel AI integration (v4)', () => { const embedSpan = container.items.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); - expect(embedSpan).toBeDefined(); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); - expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT].value).toBe('Embedding test!'); + expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); const embedManySpan = container.items.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_EMBEDDINGS_INPUT].value).toBe('["First input","Second input"]'); + expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); }, }) .start() @@ -499,16 +651,12 @@ describe('Vercel AI integration (v4)', () => { span: container => { expect(container.items).toHaveLength(2); const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); }, }) .start() @@ -525,10 +673,9 @@ describe('Vercel AI integration (v4)', () => { expect(container.items).toHaveLength(2); const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText'); expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); // Aggregated over the drained stream: v4 reports `promptTokens`/`completionTokens`, which the @@ -539,10 +686,9 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Stream response!'); const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText.doStream'); }, }) @@ -561,31 +707,29 @@ describe('Vercel AI integration (v4)', () => { // generateObject (invoke_agent) const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(invokeAgentSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(invokeAgentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(invokeAgentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(invokeAgentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); // generateObject.doGenerate (generate_content) const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(generateContentSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); - expect(generateContentSpan!.attributes['gen_ai.provider.name'].value).toBe('mock-provider'); - expect(generateContentSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(generateContentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(generateContentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs index 7ca11a34673e..7a021162097c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index 00430e451229..b608a1f717fc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -14,13 +15,13 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { getStringAttributeValue } from '../../../../utils'; -const expectedOrigin = 'auto.vercelai.channel'; - describe('Vercel AI integration (v5)', () => { afterAll(() => { cleanupChildProcesses(); @@ -33,99 +34,76 @@ describe('Vercel AI integration (v5)', () => { (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes( 'Second span here!', ), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); expect(toolInvokeAgentSpan!.status).toBe('ok'); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(toolGenerateContentSpan!.status).toBe('ok'); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -148,99 +126,80 @@ describe('Vercel AI integration (v5)', () => { (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('First span here!'); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes( 'Second span here!', ), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', @@ -266,39 +225,17 @@ describe('Vercel AI integration (v5)', () => { 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; await createRunner() // The tool error is captured while the tool is running (mid-transaction), so the error event - // and the transaction/span envelopes can arrive in either order — assert content, not wire order. + // and the span containers can arrive in either order — assert content, not wire order. .unordered() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); - expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .expect({ @@ -309,20 +246,29 @@ describe('Vercel AI integration (v5)', () => { .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'main'); + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); + expect(toolSpan!.status).toBe('error'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); + expect(errorEvent!.tags).toEqual({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + // Trace id should be the same for the segment and error event + expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs index 7ca11a34673e..7a021162097c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-telemetry-metadata.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-telemetry-metadata.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 805a65313a66..893cea265ebb 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -22,6 +23,8 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner'; @@ -47,104 +50,239 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', )!; - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstInvokeAgentSpan.status).toBe('ok'); - expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('First span here!'), )!; - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan.status).toBe('ok'); - expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateText.doGenerate', ); - expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(firstGenerateContentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('First span here!'); - - const secondInvokeAgentSpan = container.items.find( + expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toEqual(expect.any(String)); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + }); + + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', )!; - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 183, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 92, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 91, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Where is the second span?"}]', + }); expect(secondInvokeAgentSpan.status).toBe('ok'); - expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('Second span here!'), )!; - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan.status).toBe('ok'); - expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', )!; - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', + }); expect(toolInvokeAgentSpan.status).toBe('ok'); - expect(toolInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, )!; - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan.status).toBe('ok'); - expect(toolGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); + expect(toolGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ + type: 'string', + value: expect.stringContaining('getWeather'), + }); expect(toolGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'call-1', + }); expect(toolExecutionSpan.status).toBe('ok'); - expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_DESCRIPTION]?.value).toBe( 'Get the current weather for a location', ); - expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]).toBeDefined(); - expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_RESULT]).toBeDefined(); + expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toEqual(expect.any(String)); + expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toEqual(expect.any(String)); }, }) .start() @@ -165,62 +303,104 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10, )!; - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(firstInvokeAgentSpan.status).toBe('ok'); - expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(firstInvokeAgentSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - expect(firstInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10, )!; - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan.status).toBe('ok'); - expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( - 'ai.generateText.doGenerate', - ); + expect(firstGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('mock-provider'); - expect(firstGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 183, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 92, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 91, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(secondInvokeAgentSpan.status).toBe('ok'); - expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( '[{"role":"user","content":"Where is the second span?"}]', ); @@ -228,16 +408,14 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan.status).toBe('ok'); - expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string).toContain( 'Where is the second span?', @@ -246,27 +424,79 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'Second span here!', ); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, )!; - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan.status).toBe('ok'); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, )!; - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan.status).toBe('ok'); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); expect(toolExecutionSpan.status).toBe('ok'); - expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); }, @@ -288,39 +518,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; await createRunner() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) + .unordered() .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent')!; - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan.name).toBe('invoke_agent'); - expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id')!; - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan.status).toBe('ok'); - expect(generateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan).toBeDefined(); - expect(toolSpan.name).toBe('execute_tool getWeather'); - expect(toolSpan.status).toBe('error'); - expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); - expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .expect({ @@ -331,20 +537,58 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'main'); + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent')!; + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id')!; + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(generateContentSpan.status).toBe('ok'); + expect(generateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); + + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); + expect(toolSpan.status).toBe('error'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); - - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + expect(errorEvent!.tags).toEqual({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); + + // Trace id should be the same for the segment and error event + expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); }); }, { @@ -361,27 +605,25 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai related spans', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const invokeAgentSpans = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const invokeAgentSpans = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent', ); expect(invokeAgentSpans).toHaveLength(3); - const generateContentSpans = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContentSpans = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(generateContentSpans).toHaveLength(3); - const toolSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.execute_tool', - ); + const toolSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool'); expect(toolSpan).toBeDefined(); }, }) @@ -403,46 +645,40 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates spans for ToolLoopAgent with tool calls', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(4); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan.name).toBe('invoke_agent weather_agent'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(4); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(invokeAgentSpan.status).toBe('ok'); - expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); - const toolCallsGenerateContentSpan = container.items.find( + const toolCallsGenerateContentSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(toolCallsGenerateContentSpan).toBeDefined(); expect(toolCallsGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(toolCallsGenerateContentSpan.status).toBe('ok'); - expect(toolCallsGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolCallsGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan).toBeDefined(); - expect(toolSpan.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); - const finalGenerateContentSpan = container.items.find( + const finalGenerateContentSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', )!; - expect(finalGenerateContentSpan).toBeDefined(); expect(finalGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(finalGenerateContentSpan.status).toBe('ok'); - expect(finalGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(finalGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(finalGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(finalGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); }, @@ -465,18 +701,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('parents concurrent calls that share one model instance correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const invokeAgents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const generateContents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); // Two concurrent operations -> two invoke_agent + two generate_content spans. @@ -516,14 +750,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // collapse onto whichever operation resolved the shared model last. test('parents concurrent streamText calls that share one model instance correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const generateContents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); // Two concurrent operations -> two invoke_agent + two generate_content spans. @@ -560,23 +796,19 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates streamText spans with the model call parented to invoke_agent', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText.doStream'); @@ -619,13 +851,14 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('captures usage, tool calls and output across a multi-step streamText', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); // Usage is summed across the two streamed model calls (10+15, 20+25, 30+40). @@ -633,8 +866,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(45); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(70); - const generateContents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContents = spans.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(generateContents).toHaveLength(2); generateContents.forEach(span => expect(span.parent_span_id).toBe(invokeAgent.span_id)); @@ -643,7 +876,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const toolStep = generateContents.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(toolStep).toBeDefined(); expect(toolStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolStep.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); const toolStepOutput = toolStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; @@ -654,15 +886,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const textStep = generateContents.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', )!; - expect(textStep).toBeDefined(); expect(textStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(textStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('Sunny, 72°F.'); // A tool span is emitted for the streamed tool call. Its parent and recorded input/output // vary by path during stream consumption (tool i/o is covered by the non-stream scenario // and the v7 path), so here we just assert the span exists with the right name/status. - const executeTool = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(executeTool).toBeDefined(); + const executeTool = spans.find(span => span.name === 'execute_tool getWeather')!; expect(executeTool.status).toBe('ok'); expect(executeTool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); }, @@ -685,21 +915,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('captures streamed structured output (streamText with experimental_output)', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(12); expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(18); @@ -729,25 +959,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('finishes spans with an error status when the operation rejects', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); // The model throws, so the operation rejects. The spans must still be *finished* (and // therefore present in the transaction) with an error status — not left open. - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('error'); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.status).toBe('error'); }, }) @@ -769,17 +995,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('derives provider-metadata token breakdown and system instructions', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); // Cache/reasoning token breakdown is derived from the model's `providerMetadata` by the // channel subscriber, which v6 reaches through the orchestrion adapter, so the shape is the @@ -789,9 +1014,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // The per-response `responseId` is not a conversation id and must not be recorded as one. expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]).toBeUndefined(); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent).toBeDefined(); // The system prompt is supplied via the v7-only `instructions` option. Only the channel @@ -821,17 +1044,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('derives gen_ai.conversation.id from the OpenAI `conversation` provider option', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); const genAiSpans = container.items.filter(s => - String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.'), + String(s.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), ); const conversationIdOf = (span: (typeof genAiSpans)[number]) => span.attributes[GEN_AI_CONVERSATION_ID]?.value; - const invokeAgentSpans = genAiSpans.filter( - s => s.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - ); + const invokeAgentSpans = genAiSpans.filter(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); expect(invokeAgentSpans).toHaveLength(4); const [firstTurn, secondTurn, chainedTurn, apiTurn] = invokeAgentSpans.sort( (a, b) => a.start_timestamp - b.start_timestamp, @@ -850,7 +1071,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // Model-call and tool spans carry their operation's id, even though their start events // do not carry `providerOptions`. const modelCallSpans = genAiSpans.filter( - s => s.attributes['sentry.op']?.value === 'gen_ai.generate_content', + s => s.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(modelCallSpans.map(conversationIdOf).sort()).toEqual([ 'conv-from-api', @@ -858,7 +1079,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'conv_azure', undefined, ]); - const toolSpan = genAiSpans.find(s => s.attributes['sentry.op']?.value === 'gen_ai.execute_tool')!; + const toolSpan = genAiSpans.find(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool')!; expect(toolSpan).toBeDefined(); expect(conversationIdOf(toolSpan)).toBe('conv_azure'); }, @@ -881,21 +1102,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('reads cache token counts from the SDK usage object', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const generateContent = spans.find( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(120); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); }, @@ -924,32 +1145,30 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe test('creates embeddings spans for embed and embedMany', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); - const embedSpan = container.items.find( + const embedSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', )!; - expect(embedSpan).toBeDefined(); expect(embedSpan.name).toBe('embeddings mock-model-id'); expect(embedSpan.status).toBe('ok'); - expect(embedSpan.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); + expect(embedSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); expect(embedSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(embedSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - const embedManySpan = container.items.find( + const embedManySpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); if (embedManyInstrumented) { - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); + expect(embedManySpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(20); } else { expect(embedManySpan).toBeUndefined(); @@ -978,28 +1197,26 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // channel directly — v6 is exercised here, v4 in the base suite. test.skipIf(version === '7')('creates spans for generateObject', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateObject.doGenerate', ); @@ -1032,9 +1249,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .expect({ span: container => { const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent', )!; - expect(invokeAgent).toBeDefined(); expect(invokeAgent.status).toBe('error'); expect(invokeAgent.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(invokeAgent.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); @@ -1042,9 +1258,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.status).toBe('error'); }, }) @@ -1072,16 +1287,14 @@ describe('Vercel AI integration experimental_evaluate', () => { (createRunner, test) => { test('creates an evaluate span', async () => { await createRunner() - .unordered() .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', )!; - expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); - expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(evaluateSpan.attributes[SENTRY_ORIGIN]?.value).toBe('auto.vercelai.channel'); expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway'); expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev'); @@ -1145,13 +1358,11 @@ describe('Vercel AI integration experimental_evaluate', () => { (createRunner, test) => { test('does not record inputs or outputs when recording is off', async () => { await createRunner() - .unordered() .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', )!; - expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); // State, questions and answers must not come back through another attribute. Only the attributes @@ -1172,3 +1383,52 @@ describe('Vercel AI integration experimental_evaluate', () => { }, ); }); + +describe('Vercel AI telemetry metadata (v6)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario-telemetry-metadata.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('records experimental_telemetry.metadata as span attributes', async () => { + await createRunner() + .expect({ + span: container => { + for (const [name, op, requestId] of [ + ['invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate'], + ['generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate'], + ['invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'], + ['generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'], + ['embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'], + ['embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'], + ]) { + const span = container.items.find( + span => + span.name === name && + span.attributes['vercel.ai.telemetry.metadata.requestId']?.value === requestId, + ); + expect(span?.attributes[SENTRY_OP]).toEqual({ type: 'string', value: op }); + expect(span?.attributes['vercel.ai.telemetry.metadata.tenantId']).toEqual({ + type: 'string', + value: 'acme', + }); + } + const generateSpans = container.items.filter( + span => span.attributes['vercel.ai.telemetry.metadata.requestId']?.value === 'req_generate', + ); + for (const span of generateSpans) { + expect(span.attributes['vercel.ai.telemetry.metadata.attempt']).toEqual({ type: 'string', value: '2' }); + } + }, + }) + .start() + .completed(); + }); + }, + { additionalDependencies: { ai: '^6.0.0' } }, + ); +});