From 69fbdef994dc32270cfb80c7acf9442cd46d5e6f Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:40:35 +0200 Subject: [PATCH 1/8] test(node): Port Vercel AI tests to span streaming Co-Authored-By: GPT-6 --- .../tracing/vercelai/instrument-with-pii.mjs | 3 +- .../suites/tracing/vercelai/instrument.mjs | 1 - .../span-streaming-v4/instrument-with-pii.mjs | 12 - .../vercelai/span-streaming-v4/instrument.mjs | 11 - .../scenario-error-in-tool.mjs | 46 -- .../vercelai/span-streaming-v4/scenario.mjs | 82 --- .../vercelai/span-streaming-v4/test.ts | 297 -------- .../span-streaming-v6/instrument-with-pii.mjs | 11 - .../vercelai/span-streaming-v6/instrument.mjs | 11 - .../scenario-error-in-tool.mjs | 47 -- .../vercelai/span-streaming-v6/scenario.mjs | 95 --- .../vercelai/span-streaming-v6/test.ts | 366 ---------- .../suites/tracing/vercelai/test.ts | 512 ++++++++----- .../vercelai/v5/instrument-with-pii.mjs | 1 - .../suites/tracing/vercelai/v5/instrument.mjs | 1 - .../suites/tracing/vercelai/v5/test.ts | 134 ++-- .../vercelai/v6_v7/instrument-with-pii.mjs | 1 - .../tracing/vercelai/v6_v7/instrument.mjs | 1 - .../scenario-telemetry-metadata.mjs | 0 .../suites/tracing/vercelai/v6_v7/test.ts | 691 ++++++++++++------ 20 files changed, 855 insertions(+), 1468 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts rename dev-packages/node-integration-tests/suites/tracing/vercelai/{span-streaming-v6 => v6_v7}/scenario-telemetry-metadata.mjs (100%) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs index bdf24d6bbd0e..852786895514 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs @@ -2,10 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, + dataCollection: {}, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs deleted file mode 100644 index c0abcf114ae1..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs +++ /dev/null @@ -1,12 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - // inputs and outputs are enabeld by default when opting into dataCollection - dataCollection: {}, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs deleted file mode 100644 index a305d0bd2dd8..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs +++ /dev/null @@ -1,46 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText } from 'ai'; -import { MockLanguageModelV1 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - Sentry.setTag('test-tag', 'test-value'); - - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - try { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], - }), - }), - tools: { - getWeather: { - parameters: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }, - }, - prompt: 'What is the weather in San Francisco?', - }); - } catch { - // Expected error - we want the spans to still be flushed - } - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs deleted file mode 100644 index ef8cb19c3646..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario.mjs +++ /dev/null @@ -1,82 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText } from 'ai'; -import { MockLanguageModelV1 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'First span here!', - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'Second span here!', - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'tool-calls', - usage: { promptTokens: 15, completionTokens: 25 }, - text: 'Tool call completed!', - toolCalls: [ - { - toolCallType: 'function', - toolCallId: 'call-1', - toolName: 'getWeather', - args: '{ "location": "San Francisco" }', - }, - ], - }), - }), - tools: { - getWeather: { - description: 'Get the current weather for a location', - parameters: z.object({ location: z.string() }), - execute: async args => { - return `Weather in ${args.location}: Sunny, 72°F`; - }, - }, - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV1({ - doGenerate: async () => ({ - rawCall: { rawPrompt: null, rawSettings: {} }, - finishReason: 'stop', - usage: { promptTokens: 10, completionTokens: 20 }, - text: 'Third span here!', - }), - }), - prompt: 'Where is the third span?', - }); - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts deleted file mode 100644 index 50385f0e0c40..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/test.ts +++ /dev/null @@ -1,297 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - GEN_AI_TOOL_DEFINITIONS, - GEN_AI_TOOL_DESCRIPTION, - GEN_AI_TOOL_NAME, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -/** - * Helper to match a typed attribute value in a SerializedStreamedSpan. - * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction - * span `data` which stores values directly. - */ -function attr(value: unknown) { - return expect.objectContaining({ value }); -} - -const expectedOrigin = 'auto.vercelai.channel'; - -// The channel path serializes the SDK-parsed args object (compact). -const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; - -describe('Vercel AI integration (streaming v4)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - const EXPECTED_SPANS_DEFAULT_PII_FALSE = { - items: expect.arrayContaining([ - // First span - invoke_agent for simple generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Second span - generate_content for simple generateText - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Third span - invoke_agent for explicit telemetry generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fourth span - tool call invoke_agent - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fifth span - tool call generate_content - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Sixth span - execute_tool - // Note: gen_ai.tool.description is NOT present when genAI recording disabled because ai.prompt.tools is not recorded - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_DEFAULT_PII_TRUE = { - items: expect.arrayContaining([ - // First span - invoke_agent with input/output messages (PII enabled) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the first span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Second span - generate_content with input/output messages - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Third span - explicit telemetry invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the second span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fourth span - tool call invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"What is the weather in San Francisco?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Fifth span - tool call generate_content with tool definitions - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_DEFINITIONS]: expect.objectContaining({ - value: expect.stringContaining('getWeather'), - }), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - // Sixth span - execute_tool with description and input/output - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_DESCRIPTION]: attr('Get the current weather for a location'), - [GEN_AI_TOOL_CALL_ARGUMENTS]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_TOOL_CALL_RESULT]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_ERROR_IN_TOOL = { - items: expect.arrayContaining([ - expect.objectContaining({ - name: 'invoke_agent', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(expectedOrigin), - }), - }), - ]), - }; - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording disabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording enabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); - }); - }); - - createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { - test('normalizes error status in streaming mode', async () => { - await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); - }); - }); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument-with-pii.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs deleted file mode 100644 index 5e0b6fb5592f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs deleted file mode 100644 index 51bdae176158..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-error-in-tool.mjs +++ /dev/null @@ -1,47 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - try { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - inputSchema: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - } catch { - // Expected error - we want the spans to still be flushed - } - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs deleted file mode 100644 index bf5e43a32e65..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'First span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Second span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - description: 'Get the current weather for a location', - inputSchema: z.object({ location: z.string() }), - execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Third span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the third span?', - }); - }); - - await Sentry.flush(2000); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts deleted file mode 100644 index 68c88b131783..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES, - GEN_AI_OPERATION_NAME, - GEN_AI_OUTPUT_MESSAGES, - GEN_AI_REQUEST_MODEL, - GEN_AI_RESPONSE_MODEL, - GEN_AI_TOOL_CALL_ARGUMENTS, - GEN_AI_TOOL_CALL_RESULT, - GEN_AI_TOOL_DEFINITIONS, - GEN_AI_TOOL_DESCRIPTION, - GEN_AI_TOOL_NAME, - GEN_AI_USAGE_INPUT_TOKENS, - GEN_AI_USAGE_OUTPUT_TOKENS, - GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_OP, - SENTRY_ORIGIN, -} from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -/** - * Helper to match a typed attribute value in a SerializedStreamedSpan. - * Streamed span attributes are `{ value: X, type: Y }` objects, unlike transaction - * span `data` which stores values directly. - */ -function attr(value: unknown) { - return expect.objectContaining({ value }); -} - -describe('Vercel AI integration (streaming, v6)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - const origin = 'auto.vercelai.channel'; - - const EXPECTED_SPANS_DEFAULT_PII_FALSE = { - items: expect.arrayContaining([ - // First span - invoke_agent for simple generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Second span - generate_content for simple generateText - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Third span - invoke_agent for explicit telemetry generateText - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fourth span - tool call invoke_agent - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fifth span - tool call generate_content - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Sixth span - execute_tool - // Note: gen_ai.tool.description is NOT present when genAI recording disabled because ai.prompt.tools is not recorded - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_DEFAULT_PII_TRUE = { - items: expect.arrayContaining([ - // First span - invoke_agent with input/output messages (PII enabled) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the first span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Second span - generate_content with input/output messages - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_RESPONSE_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Third span - explicit telemetry invoke_agent with messages - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"Where is the second span?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(10), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(20), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(30), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fourth span - tool call invoke_agent with messages (V6: no text part, only tool_call) - expect.objectContaining({ - name: 'invoke_agent', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_INPUT_MESSAGES]: attr('[{"role":"user","content":"What is the weather in San Francisco?"}]'), - [GEN_AI_OUTPUT_MESSAGES]: attr( - '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', - ), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Fifth span - tool call generate_content with tool definitions - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_DEFINITIONS]: expect.objectContaining({ - value: expect.stringContaining('getWeather'), - }), - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - // Sixth span - execute_tool with description and input/output - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_DESCRIPTION]: attr('Get the current weather for a location'), - [GEN_AI_TOOL_CALL_ARGUMENTS]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_TOOL_CALL_RESULT]: expect.objectContaining({ value: expect.any(String) }), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - const EXPECTED_SPANS_ERROR_IN_TOOL = { - items: expect.arrayContaining([ - expect.objectContaining({ - name: 'invoke_agent', - attributes: expect.objectContaining({ - [GEN_AI_OPERATION_NAME]: attr('invoke_agent'), - [SENTRY_OP]: attr('gen_ai.invoke_agent'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - expect.objectContaining({ - name: 'generate_content mock-model-id', - status: 'ok', - attributes: expect.objectContaining({ - [GEN_AI_REQUEST_MODEL]: attr('mock-model-id'), - [GEN_AI_USAGE_INPUT_TOKENS]: attr(15), - [GEN_AI_USAGE_OUTPUT_TOKENS]: attr(25), - [GEN_AI_USAGE_TOTAL_TOKENS]: attr(40), - [GEN_AI_OPERATION_NAME]: attr('generate_content'), - [SENTRY_OP]: attr('gen_ai.generate_content'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - expect.objectContaining({ - name: 'execute_tool getWeather', - status: 'error', - attributes: expect.objectContaining({ - [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: attr('call-1'), - [GEN_AI_TOOL_NAME]: attr('getWeather'), - [GEN_AI_OPERATION_NAME]: attr('execute_tool'), - [SENTRY_OP]: attr('gen_ai.execute_tool'), - [SENTRY_ORIGIN]: attr(origin), - }), - }), - ]), - }; - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording disabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_FALSE }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('creates ai related spans in streaming mode with genAI recording enabled', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_DEFAULT_PII_TRUE }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - function spanWithMetadata(name: string, op: string, requestId: string, extra: Record = {}) { - return expect.objectContaining({ - name, - attributes: expect.objectContaining({ - 'vercel.ai.telemetry.metadata.requestId': attr(requestId), - 'vercel.ai.telemetry.metadata.tenantId': attr('acme'), - ...extra, - [SENTRY_OP]: attr(op), - }), - }); - } - - const EXPECTED_SPANS_TELEMETRY_METADATA = { - items: expect.arrayContaining([ - spanWithMetadata('invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate', { - 'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' }, - }), - spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate', { - 'vercel.ai.telemetry.metadata.attempt': { type: 'string', value: '2' }, - }), - spanWithMetadata('invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'), - spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'), - spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'), - spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'), - ]), - }; - - createEsmAndCjsTests( - __dirname, - 'scenario-telemetry-metadata.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('records experimental_telemetry.metadata as span attributes', async () => { - await createRunner().expect({ span: EXPECTED_SPANS_TELEMETRY_METADATA }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-error-in-tool.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('normalizes error status in streaming mode', async () => { - await createRunner().ignore('event').expect({ span: EXPECTED_SPANS_ERROR_IN_TOOL }).start().completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 75945701e92f..1491641455e3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpan, SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -16,12 +17,15 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_ORIGIN, + GEN_AI_OPERATION_NAME, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; import { getStringAttributeValue } from '../../../utils'; const expectedOrigin = 'auto.vercelai.channel'; +const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; describe('Vercel AI integration (v4)', () => { afterAll(() => { @@ -31,101 +35,162 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(firstInvokeAgentSpan!.status).toBe('ok'); expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan!.status).toBe('ok'); expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(secondInvokeAgentSpan!.status).toBe('ok'); expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Second span here!'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan!.status).toBe('ok'); expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan!.status).toBe('ok'); expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); expect(toolExecutionSpan!.status).toBe('ok'); expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); @@ -140,97 +205,202 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 30 }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 10 }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstInvokeAgentSpan!.status).toBe('ok'); expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan!.status).toBe('ok'); expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('First span here!'); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + }); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + }); expect(secondInvokeAgentSpan!.status).toBe('ok'); expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 40 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 25 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 15 }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan!.status).toBe('ok'); expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeDefined(); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, + }); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_TOOL_DEFINITIONS]?.value)?.includes('getWeather'), ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan!.status).toBe('ok'); expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toContain('getWeather'); + expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ + type: 'string', + value: expect.stringContaining('getWeather'), + }); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'call-1', + }); expect(toolExecutionSpan!.status).toBe('ok'); expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); @@ -248,43 +418,51 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let segment: SerializedStreamedSpan | undefined; let errorEvent: Event | undefined; await createRunner() // In orchestrion mode the tool error is captured mid-transaction, so the error and - // transaction/span envelopes can arrive in either order — assert content, not wire order. + // span containers can arrive in either order — assert content, not wire order. .unordered() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find( - span => span.name === 'invoke_agent' && span.status === 'error', + segment = container.items.find(span => span.is_segment); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.status).toBe('error'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent' && span.status === 'error'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ type: 'integer', value: 40 }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ type: 'integer', value: 25 }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ type: 'integer', value: 15 }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -296,10 +474,8 @@ describe('Vercel AI integration (v4)', () => { .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + expect(segment!.name).toBe('main'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError`. The @@ -315,45 +491,37 @@ describe('Vercel AI integration (v4)', () => { ); // Both paths stamp the operation's call-site span onto the bubbled error, so the global // unhandled-rejection handler restores it and correlates the report to the transaction's root span. - expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); - expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); + expect(errorEvent!.contexts!.trace!.trace_id).toBe(segment!.trace_id); + expect(errorEvent!.contexts!.trace!.span_id).toBe(segment!.span_id); }); }); createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool in express server', async () => { - let transactionEvent: Event | undefined; + let segment: SerializedStreamedSpan | undefined; + let spanItems: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; const runner = createRunner() - // The error and transaction/span envelopes can arrive in either order, so assert content, not order. + // The error and span containers can arrive in either order, so assert content, not order. .unordered() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find( - span => span.name === 'invoke_agent' && span.status === 'error', + spanItems = container.items; + segment = container.items.find(span => span.is_segment); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.status).toBe('error'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent' && span.status === 'error'); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); @@ -369,13 +537,10 @@ describe('Vercel AI integration (v4)', () => { await runner.makeRequest('get', '/test/error-in-tool', { expectError: true }); await runner.completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); - expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); + expect(segment!.name).toBe('GET /test/error-in-tool'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); - expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); + expect(errorEvent!.contexts!.trace!.trace_id).toBe(segment!.trace_id); // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError` and is // captured once by the express error handler — the channel subscriber deliberately doesn't @@ -388,13 +553,8 @@ describe('Vercel AI integration (v4)', () => { }), ]), ); - // The error is captured on the Express layer span (a child of the request transaction), so its - // `span_id` is one of the transaction's spans rather than the transaction's own (root) span. - const transactionSpanIds = [ - transactionEvent!.contexts!.trace!.span_id, - ...(transactionEvent!.spans ?? []).map(span => span.span_id), - ]; - expect(transactionSpanIds).toContain(errorEvent!.contexts!.trace!.span_id); + // The Express error handler captures the error on a child span of the request. + expect(spanItems.map(span => span.span_id)).toContain(errorEvent!.contexts!.trace!.span_id); }); }); @@ -406,21 +566,21 @@ describe('Vercel AI integration (v4)', () => { test('extracts system instructions from messages', async () => { await createRunner() .ignore('event') - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(2); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); }, }) @@ -433,26 +593,24 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates embedding related spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const embedSpan = container.items.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); - expect(embedSpan).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(2); + const embedSpan = spans.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); expect(embedSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); - expect(embedSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - const embedManySpan = container.items.find( - span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20, - ); - expect(embedManySpan).toBeDefined(); + const embedManySpan = spans.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); }, }) .start() @@ -463,27 +621,25 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates embedding related spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const embedSpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - expect(embedSpan).toBeDefined(); + expect(spans).toHaveLength(2); + const embedSpan = spans.find(span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!'); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT].value).toBe('Embedding test!'); - const embedManySpan = container.items.find( + const embedManySpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); - expect(embedManySpan!.attributes[GEN_AI_EMBEDDINGS_INPUT].value).toBe('["First input","Second input"]'); }, }) .start() @@ -494,19 +650,19 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('does not overwrite conversation id set via Sentry.setConversationId with responseId from provider metadata', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(2); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); }, @@ -519,16 +675,18 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-stream-text.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai spans for streamText (doStream)', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(2); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText'); expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); // Aggregated over the drained stream: v4 reports `promptTokens`/`completionTokens`, which the @@ -538,11 +696,9 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); expect(invokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Stream response!'); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText.doStream'); }, }) @@ -554,18 +710,20 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-generate-object.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('captures generateObject spans with schema attributes', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(2); // generateObject (invoke_agent) - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); - expect(invokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(invokeAgentSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); expect(invokeAgentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); expect(invokeAgentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); @@ -573,12 +731,10 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); // generateObject.doGenerate (generate_content) - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); - expect(generateContentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(generateContentSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); expect(generateContentSpan!.attributes['gen_ai.provider.name'].value).toBe('mock-provider'); expect(generateContentSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs index 7ca11a34673e..7a021162097c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index 00430e451229..ee41c108b0af 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpan } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -14,13 +15,12 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { getStringAttributeValue } from '../../../../utils'; -const expectedOrigin = 'auto.vercelai.channel'; - describe('Vercel AI integration (v5)', () => { afterAll(() => { cleanupChildProcesses(); @@ -33,99 +33,76 @@ describe('Vercel AI integration (v5)', () => { (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(firstGenerateContentSpan!.status).toBe('ok'); expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); expect(secondInvokeAgentSpan!.status).toBe('ok'); expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes( 'Second span here!', ), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); expect(toolInvokeAgentSpan!.status).toBe('ok'); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(toolGenerateContentSpan!.status).toBe('ok'); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -148,35 +125,32 @@ describe('Vercel AI integration (v5)', () => { (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); expect(firstInvokeAgentSpan!.status).toBe('ok'); expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(firstGenerateContentSpan!.status).toBe('ok'); expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( @@ -185,60 +159,44 @@ describe('Vercel AI integration (v5)', () => { expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('First span here!'); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); expect(secondInvokeAgentSpan!.status).toBe('ok'); expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes( 'Second span here!', ), ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan!.status).toBe('ok'); expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); expect(toolGenerateContentSpan!.status).toBe('ok'); expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); @@ -266,38 +224,30 @@ describe('Vercel AI integration (v5)', () => { 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let segment: SerializedStreamedSpan | undefined; let errorEvent: Event | undefined; await createRunner() // The tool error is captured while the tool is running (mid-transaction), so the error event - // and the transaction/span envelopes can arrive in either order — assert content, not wire order. + // and the span containers can arrive in either order — assert content, not wire order. .unordered() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); + segment = container.items.find(span => span.is_segment); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes['sentry.origin'].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -309,10 +259,8 @@ describe('Vercel AI integration (v5)', () => { .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + expect(segment!.name).toBe('main'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.level).toBe('error'); expect(errorEvent!.tags).toEqual( expect.objectContaining({ @@ -321,8 +269,8 @@ describe('Vercel AI integration (v5)', () => { }), ); - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + // Trace id should be the same for the segment and error event + expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); }); }, { diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs index 7ca11a34673e..7a021162097c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs index 42052d281304..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-telemetry-metadata.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-telemetry-metadata.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index f8a26a410833..294c66a1acb6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpan } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -22,6 +23,8 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_ORIGIN, + SENTRY_OP, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner'; @@ -47,96 +50,231 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the first span?"}]', )!; - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstInvokeAgentSpan.status).toBe('ok'); expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); expect(firstInvokeAgentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('First span here!'), )!; - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 10, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan.status).toBe('ok'); expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateText.doGenerate', ); expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(firstGenerateContentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('First span here!'); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + }); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"Where is the second span?"}]', )!; - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 183, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 92, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 91, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({ + type: 'string', + value: '[{"role":"user","content":"Where is the second span?"}]', + }); expect(secondInvokeAgentSpan.status).toBe('ok'); expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('Second span here!'), )!; - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan.status).toBe('ok'); expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"What is the weather in San Francisco?"}]', )!; - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ + type: 'string', + value: + '[{"role":"assistant","parts":[{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"{\\"location\\":\\"San Francisco\\"}"}],"finish_reason":"tool_call"}]', + }); expect(toolInvokeAgentSpan.status).toBe('ok'); - expect(toolInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, )!; - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan.status).toBe('ok'); expect(toolGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); + expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ + type: 'string', + value: expect.stringContaining('getWeather'), + }); expect(toolGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ + type: 'string', + value: 'call-1', + }); expect(toolExecutionSpan.status).toBe('ok'); expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); @@ -165,60 +303,102 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const firstInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10, )!; - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(firstInvokeAgentSpan.status).toBe('ok'); expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); expect(firstInvokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(firstInvokeAgentSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - expect(firstInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const firstGenerateContentSpan = container.items.find( + const firstGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_INPUT_MESSAGES] === undefined && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10, )!; - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 30, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 20, + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(firstGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(firstGenerateContentSpan.status).toBe('ok'); expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( - 'ai.generateText.doGenerate', - ); expect(firstGenerateContentSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('mock-provider'); - expect(firstGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); - const secondInvokeAgentSpan = container.items.find( + const secondInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 183, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 92, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 91, + }); + expect(secondInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(secondInvokeAgentSpan.status).toBe('ok'); expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( @@ -228,14 +408,12 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = container.items.find( + const secondGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(secondGenerateContentSpan.status).toBe('ok'); expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); @@ -246,25 +424,77 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'Second span here!', ); - const toolInvokeAgentSpan = container.items.find( + const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, )!; - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolInvokeAgentSpan.status).toBe('ok'); - const toolGenerateContentSpan = container.items.find( + const toolGenerateContentSpan = spans.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, )!; - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ + type: 'string', + value: expectedOrigin, + }); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP]).toEqual({ + type: 'string', + value: 'gen_ai.generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(toolGenerateContentSpan.status).toBe('ok'); - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'execute_tool', + }); expect(toolExecutionSpan.status).toBe('ok'); expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); @@ -288,36 +518,55 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let transactionEvent: Event | undefined; + let segment: SerializedStreamedSpan | undefined; let errorEvent: Event | undefined; await createRunner() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) + .unordered() .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent')!; - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan.name).toBe('invoke_agent'); + segment = container.items.find(span => span.is_segment); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent')!; + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id')!; - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan.name).toBe('generate_content mock-model-id'); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id')!; + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); expect(generateContentSpan.status).toBe('ok'); expect(generateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan).toBeDefined(); - expect(toolSpan.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan.status).toBe('error'); expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); @@ -331,10 +580,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .start() .completed(); - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); + expect(segment!.name).toBe('main'); - expect(errorEvent).toBeDefined(); expect(errorEvent!.level).toBe('error'); expect(errorEvent!.tags).toEqual( expect.objectContaining({ @@ -343,8 +590,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }), ); - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + // Trace id should be the same for the segment and error event + expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); }); }, { @@ -361,27 +608,25 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates ai related spans', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(7); - const invokeAgentSpans = container.items.filter( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(7); + const invokeAgentSpans = spans.filter( span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', ); expect(invokeAgentSpans).toHaveLength(3); - const generateContentSpans = container.items.filter( + const generateContentSpans = spans.filter( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', ); expect(generateContentSpans).toHaveLength(3); - const toolSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.execute_tool', - ); + const toolSpan = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.execute_tool'); expect(toolSpan).toBeDefined(); }, }) @@ -403,43 +648,37 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates spans for ToolLoopAgent with tool calls', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - expect(container.items).toHaveLength(4); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan.name).toBe('invoke_agent weather_agent'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + expect(spans).toHaveLength(4); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(invokeAgentSpan.status).toBe('ok'); expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); - const toolCallsGenerateContentSpan = container.items.find( + const toolCallsGenerateContentSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(toolCallsGenerateContentSpan).toBeDefined(); expect(toolCallsGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(toolCallsGenerateContentSpan.status).toBe('ok'); expect(toolCallsGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan).toBeDefined(); - expect(toolSpan.name).toBe('execute_tool getWeather'); + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; expect(toolSpan.status).toBe('ok'); expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); - const finalGenerateContentSpan = container.items.find( + const finalGenerateContentSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', )!; - expect(finalGenerateContentSpan).toBeDefined(); expect(finalGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(finalGenerateContentSpan.status).toBe('ok'); expect(finalGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); @@ -465,17 +704,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('parents concurrent calls that share one model instance correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const invokeAgents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const generateContents = container.items.filter( + const invokeAgents = spans.filter(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', ); @@ -516,13 +753,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // collapse onto whichever operation resolved the shared model last. test('parents concurrent streamText calls that share one model instance correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgents = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const generateContents = container.items.filter( + const invokeAgents = spans.filter(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent'); + const generateContents = spans.filter( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', ); @@ -560,23 +799,19 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('creates streamText spans with the model call parented to invoke_agent', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); - const generateContent = container.items.find( + const generateContent = spans.find( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText.doStream'); @@ -619,13 +854,14 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('captures usage, tool calls and output across a multi-step streamText', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); // Usage is summed across the two streamed model calls (10+15, 20+25, 30+40). @@ -633,7 +869,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(45); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(70); - const generateContents = container.items.filter( + const generateContents = spans.filter( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', ); expect(generateContents).toHaveLength(2); @@ -643,7 +879,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const toolStep = generateContents.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["tool-calls"]', )!; - expect(toolStep).toBeDefined(); expect(toolStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolStep.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); const toolStepOutput = toolStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string; @@ -654,15 +889,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const textStep = generateContents.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["stop"]', )!; - expect(textStep).toBeDefined(); expect(textStep.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(textStep.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toContain('Sunny, 72°F.'); // A tool span is emitted for the streamed tool call. Its parent and recorded input/output // vary by path during stream consumption (tool i/o is covered by the non-stream scenario // and the v7 path), so here we just assert the span exists with the right name/status. - const executeTool = container.items.find(span => span.name === 'execute_tool getWeather')!; - expect(executeTool).toBeDefined(); + const executeTool = spans.find(span => span.name === 'execute_tool getWeather')!; expect(executeTool.status).toBe('ok'); expect(executeTool.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); }, @@ -685,21 +918,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('captures streamed structured output (streamText with experimental_output)', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); - const generateContent = container.items.find( + const generateContent = spans.find( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(12); expect(generateContent.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(18); @@ -729,25 +962,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('finishes spans with an error status when the operation rejects', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); // The model throws, so the operation rejects. The spans must still be *finished* (and // therefore present in the transaction) with an error status — not left open. - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('error'); - const generateContent = container.items.find( + const generateContent = spans.find( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.status).toBe('error'); }, }) @@ -769,17 +998,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('derives provider-metadata token breakdown, conversation id and system instructions', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); - const generateContent = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const generateContent = spans.find( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); // Cache/reasoning token breakdown and conversation id are derived from the model's // `providerMetadata` — by the OTel processor on v6 and by the channel subscriber on v7, @@ -788,9 +1016,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(generateContent.attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]?.value).toBe(7); expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe('resp_abc123'); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; + const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent).toBeDefined(); // The system prompt is supplied via the v7-only `instructions` option. Only the channel @@ -820,21 +1046,21 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('reads cache token counts from the SDK usage object', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const generateContent = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); + const generateContent = spans.find( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(120); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); - const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - )!; - expect(invokeAgent).toBeDefined(); + const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); }, @@ -863,29 +1089,27 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe test('creates embeddings spans for embed and embedMany', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); - const embedSpan = container.items.find( + const embedSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', )!; - expect(embedSpan).toBeDefined(); expect(embedSpan.name).toBe('embeddings mock-model-id'); expect(embedSpan.status).toBe('ok'); expect(embedSpan.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); expect(embedSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(embedSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); - const embedManySpan = container.items.find( + const embedManySpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); if (embedManyInstrumented) { - expect(embedManySpan).toBeDefined(); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); expect(embedManySpan!.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); @@ -917,16 +1141,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // channel directly — v6 is exercised here, v4 in the base suite. test.skipIf(version === '7')('creates spans for generateObject', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - // Every emitted gen_ai span carries the version-appropriate origin. - container.items - .filter(s => String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.')) - .forEach(s => expect(s.attributes['sentry.origin']?.value).toBe(expectedOrigin)); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + ); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); expect(invokeAgentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); @@ -935,8 +1158,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(40); - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( @@ -973,7 +1195,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const invokeAgent = container.items.find( span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', )!; - expect(invokeAgent).toBeDefined(); expect(invokeAgent.status).toBe('error'); expect(invokeAgent.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(invokeAgent.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); @@ -983,7 +1204,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const generateContent = container.items.find( span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', )!; - expect(generateContent).toBeDefined(); expect(generateContent.status).toBe('error'); }, }) @@ -1017,7 +1237,6 @@ describe('Vercel AI integration experimental_evaluate', () => { const evaluateSpan = container.items.find( span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', )!; - expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); @@ -1090,7 +1309,6 @@ describe('Vercel AI integration experimental_evaluate', () => { const evaluateSpan = container.items.find( span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', )!; - expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); // State, questions and answers must not come back through another attribute. Only the attributes @@ -1111,3 +1329,52 @@ describe('Vercel AI integration experimental_evaluate', () => { }, ); }); + +describe('Vercel AI telemetry metadata (v6)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario-telemetry-metadata.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('records experimental_telemetry.metadata as span attributes', async () => { + await createRunner() + .expect({ + span: container => { + for (const [name, op, requestId] of [ + ['invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate'], + ['generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate'], + ['invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'], + ['generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'], + ['embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'], + ['embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'], + ]) { + const span = container.items.find( + span => + span.name === name && + span.attributes['vercel.ai.telemetry.metadata.requestId']?.value === requestId, + ); + expect(span?.attributes[SENTRY_OP]).toEqual({ type: 'string', value: op }); + expect(span?.attributes['vercel.ai.telemetry.metadata.tenantId']).toEqual({ + type: 'string', + value: 'acme', + }); + } + const generateSpans = container.items.filter( + span => span.attributes['vercel.ai.telemetry.metadata.requestId']?.value === 'req_generate', + ); + for (const span of generateSpans) { + expect(span.attributes['vercel.ai.telemetry.metadata.attempt']).toEqual({ type: 'string', value: '2' }); + } + }, + }) + .start() + .completed(); + }); + }, + { additionalDependencies: { ai: '^6.0.0' } }, + ); +}); From 7725a78f050dacf9b5bcf095e8f81905a086aeba Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 12:07:03 +0200 Subject: [PATCH 2/8] test(node): Use the SENTRY_ORIGIN convention in vercelai tests Co-Authored-By: GPT-6 --- .../suites/tracing/vercelai/v6_v7/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 294c66a1acb6..788746973da5 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1239,7 +1239,7 @@ describe('Vercel AI integration experimental_evaluate', () => { )!; expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); - expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(evaluateSpan.attributes[SENTRY_ORIGIN]?.value).toBe('auto.vercelai.channel'); expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway'); expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev'); From 45a2f0bdbc48d51107ac1e42514b1aedeb4e30f4 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 12:13:00 +0200 Subject: [PATCH 3/8] test(node): Use convention constants for vercelai span attributes Co-Authored-By: GPT-6 --- .../suites/tracing/vercelai/test.ts | 96 ++++++++++--------- .../suites/tracing/vercelai/v5/test.ts | 29 +++--- .../suites/tracing/vercelai/v6_v7/test.ts | 92 +++++++++--------- 3 files changed, 110 insertions(+), 107 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 1491641455e3..5ed94f104da7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -2,8 +2,10 @@ import type { SerializedStreamedSpan, SerializedStreamedSpanContainer } from '@s import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, @@ -17,8 +19,8 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, SENTRY_ORIGIN, - GEN_AI_OPERATION_NAME, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -56,7 +58,7 @@ describe('Vercel AI integration (v4)', () => { value: 'invoke_agent', }); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); @@ -94,7 +96,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); const secondInvokeAgentSpan = spans.find( @@ -125,7 +127,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); @@ -136,7 +138,7 @@ describe('Vercel AI integration (v4)', () => { getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Second span here!'); @@ -153,7 +155,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); @@ -183,7 +185,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); @@ -192,7 +194,7 @@ describe('Vercel AI integration (v4)', () => { value: 'execute_tool', }); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -239,7 +241,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', @@ -279,7 +281,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); @@ -323,7 +325,7 @@ describe('Vercel AI integration (v4)', () => { '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', }); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); const secondGenerateContentSpan = spans.find( span => @@ -331,7 +333,7 @@ describe('Vercel AI integration (v4)', () => { getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), ); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolInvokeAgentSpan = spans.find( span => @@ -352,7 +354,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(toolInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ type: 'string', value: `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, @@ -384,7 +386,7 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ type: 'string', value: expect.stringContaining('getWeather'), @@ -402,7 +404,7 @@ describe('Vercel AI integration (v4)', () => { value: 'call-1', }); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', @@ -438,7 +440,7 @@ describe('Vercel AI integration (v4)', () => { type: 'string', value: 'invoke_agent', }); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); @@ -455,14 +457,14 @@ describe('Vercel AI integration (v4)', () => { value: 'mock-model-id', }); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -514,16 +516,16 @@ describe('Vercel AI integration (v4)', () => { ); expect(spans).toHaveLength(3); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent' && span.status === 'error'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -575,13 +577,13 @@ describe('Vercel AI integration (v4)', () => { ); expect(spans).toHaveLength(2); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); }, }) .start() @@ -604,13 +606,13 @@ describe('Vercel AI integration (v4)', () => { const embedSpan = spans.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); - expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); expect(embedSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); const embedManySpan = spans.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); }, }) .start() @@ -632,14 +634,14 @@ describe('Vercel AI integration (v4)', () => { const embedSpan = spans.find(span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!'); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); - expect(embedSpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); const embedManySpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op'].value).toBe('gen_ai.embeddings'); + expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); }, }) .start() @@ -659,12 +661,12 @@ describe('Vercel AI integration (v4)', () => { ); expect(spans).toHaveLength(2); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(generateContentSpan!.attributes['gen_ai.conversation.id'].value).toBe('conv-a'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); }, }) .start() @@ -686,7 +688,7 @@ describe('Vercel AI integration (v4)', () => { const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText'); expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); // Aggregated over the drained stream: v4 reports `promptTokens`/`completionTokens`, which the @@ -698,7 +700,7 @@ describe('Vercel AI integration (v4)', () => { const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText.doStream'); }, }) @@ -722,26 +724,26 @@ describe('Vercel AI integration (v4)', () => { // generateObject (invoke_agent) const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); - expect(invokeAgentSpan!.attributes['gen_ai.operation.name'].value).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(invokeAgentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(invokeAgentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(invokeAgentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); // generateObject.doGenerate (generate_content) const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); - expect(generateContentSpan!.attributes['gen_ai.operation.name'].value).toBe('generate_content'); - expect(generateContentSpan!.attributes['gen_ai.provider.name'].value).toBe('mock-provider'); - expect(generateContentSpan!.attributes['gen_ai.request.model'].value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes['gen_ai.response.model'].value).toBe('mock-model-id'); - expect(generateContentSpan!.attributes['gen_ai.usage.input_tokens'].value).toBe(15); - expect(generateContentSpan!.attributes['gen_ai.usage.output_tokens'].value).toBe(25); - expect(generateContentSpan!.attributes['gen_ai.usage.total_tokens'].value).toBe(40); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); + expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index ee41c108b0af..34250d07fc6c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -15,6 +15,7 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; @@ -49,7 +50,7 @@ describe('Vercel AI integration (v5)', () => { span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); @@ -63,7 +64,7 @@ describe('Vercel AI integration (v5)', () => { span.attributes[GEN_AI_USAGE_INPUT_TOKENS].value === 10, ); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); const secondInvokeAgentSpan = spans.find( @@ -73,7 +74,7 @@ describe('Vercel AI integration (v5)', () => { '[{"role":"user","content":"Where is the second span?"}]', ); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); @@ -86,7 +87,7 @@ describe('Vercel AI integration (v5)', () => { ), ); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolInvokeAgentSpan = spans.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, @@ -102,7 +103,7 @@ describe('Vercel AI integration (v5)', () => { const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); }, @@ -140,7 +141,7 @@ describe('Vercel AI integration (v5)', () => { '[{"role":"user","content":"Where is the first span?"}]', ); expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', @@ -152,7 +153,7 @@ describe('Vercel AI integration (v5)', () => { getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), ); expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); @@ -166,7 +167,7 @@ describe('Vercel AI integration (v5)', () => { '[{"role":"user","content":"Where is the second span?"}]', ); expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); const secondGenerateContentSpan = spans.find( span => @@ -176,7 +177,7 @@ describe('Vercel AI integration (v5)', () => { ), ); expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolInvokeAgentSpan = spans.find( span => @@ -192,13 +193,13 @@ describe('Vercel AI integration (v5)', () => { span.attributes[GEN_AI_TOOL_DEFINITIONS] !== undefined, ); expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toBeDefined(); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', @@ -239,15 +240,15 @@ describe('Vercel AI integration (v5)', () => { ); expect(spans).toHaveLength(3); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 788746973da5..740138a65519 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -23,8 +23,8 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, - SENTRY_ORIGIN, SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests, createEsmTests } from '../../../../utils/runner'; @@ -93,7 +93,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'mock-model-id', }); expect(firstInvokeAgentSpan.status).toBe('ok'); - expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes['vercel.ai.operationId']?.value).toBe('ai.generateText'); expect(firstInvokeAgentSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value).toBe( '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', @@ -133,7 +133,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'mock-model-id', }); expect(firstGenerateContentSpan.status).toBe('ok'); - expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateText.doGenerate', ); @@ -184,7 +184,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: '[{"role":"user","content":"Where is the second span?"}]', }); expect(secondInvokeAgentSpan.status).toBe('ok'); - expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); const secondGenerateContentSpan = spans.find( span => @@ -192,7 +192,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string | undefined)?.includes('Second span here!'), )!; expect(secondGenerateContentSpan.status).toBe('ok'); - expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); const toolInvokeAgentSpan = spans.find( span => @@ -258,7 +258,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'mock-model-id', }); expect(toolGenerateContentSpan.status).toBe('ok'); - expect(toolGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(toolGenerateContentSpan!.attributes[GEN_AI_TOOL_DEFINITIONS]).toEqual({ type: 'string', value: expect.stringContaining('getWeather'), @@ -276,7 +276,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'call-1', }); expect(toolExecutionSpan.status).toBe('ok'); - expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_DESCRIPTION]?.value).toBe( 'Get the current weather for a location', @@ -327,7 +327,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'invoke_agent', }); expect(firstInvokeAgentSpan.status).toBe('ok'); - expect(firstInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(firstInvokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(firstInvokeAgentSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); expect(firstInvokeAgentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); @@ -365,7 +365,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'mock-model-id', }); expect(firstGenerateContentSpan.status).toBe('ok'); - expect(firstGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('mock-provider'); const secondInvokeAgentSpan = spans.find( @@ -400,7 +400,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'mock-model-id', }); expect(secondInvokeAgentSpan.status).toBe('ok'); - expect(secondInvokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(secondInvokeAgentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe( '[{"role":"user","content":"Where is the second span?"}]', ); @@ -415,7 +415,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 91, )!; expect(secondGenerateContentSpan.status).toBe('ok'); - expect(secondGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(secondGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string).toContain( 'Where is the second span?', @@ -496,7 +496,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'execute_tool', }); expect(toolExecutionSpan.status).toBe('ok'); - expect(toolExecutionSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); }, @@ -536,7 +536,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe type: 'string', value: 'invoke_agent', }); - expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id')!; expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); @@ -561,14 +561,14 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe value: 'mock-model-id', }); expect(generateContentSpan.status).toBe('ok'); - expect(generateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(generateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; expect(toolSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan.status).toBe('error'); - expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); }, }) @@ -617,16 +617,16 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe ); expect(spans).toHaveLength(7); const invokeAgentSpans = spans.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent', ); expect(invokeAgentSpans).toHaveLength(3); const generateContentSpans = spans.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(generateContentSpans).toHaveLength(3); - const toolSpan = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.execute_tool'); + const toolSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool'); expect(toolSpan).toBeDefined(); }, }) @@ -658,7 +658,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(spans).toHaveLength(4); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent weather_agent')!; expect(invokeAgentSpan.status).toBe('ok'); - expect(invokeAgentSpan.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); const toolCallsGenerateContentSpan = spans.find( @@ -666,13 +666,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe )!; expect(toolCallsGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(toolCallsGenerateContentSpan.status).toBe('ok'); - expect(toolCallsGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolCallsGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); expect(toolCallsGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; expect(toolSpan.status).toBe('ok'); - expect(toolSpan.attributes['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(toolSpan.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); @@ -681,7 +681,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe )!; expect(finalGenerateContentSpan.name).toBe('generate_content mock-model-id'); expect(finalGenerateContentSpan.status).toBe('ok'); - expect(finalGenerateContentSpan.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(finalGenerateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(finalGenerateContentSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); expect(finalGenerateContentSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(25); }, @@ -711,9 +711,9 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const invokeAgents = spans.filter(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent'); + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); const generateContents = spans.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); // Two concurrent operations -> two invoke_agent + two generate_content spans. @@ -760,9 +760,9 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const invokeAgents = spans.filter(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent'); + const invokeAgents = spans.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); const generateContents = spans.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); // Two concurrent operations -> two invoke_agent + two generate_content spans. @@ -806,11 +806,11 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); const generateContent = spans.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText.doStream'); @@ -861,7 +861,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); // Usage is summed across the two streamed model calls (10+15, 20+25, 30+40). @@ -870,7 +870,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(70); const generateContents = spans.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(generateContents).toHaveLength(2); generateContents.forEach(span => expect(span.parent_span_id).toBe(invokeAgent.span_id)); @@ -925,13 +925,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const spans = container.items.filter( span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); - const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('ok'); expect(invokeAgent.attributes['vercel.ai.operationId']?.value).toBe('ai.streamText'); expect(invokeAgent.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(30); const generateContent = spans.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; expect(generateContent.parent_span_id).toBe(invokeAgent.span_id); expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(12); @@ -971,11 +971,11 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe ); // The model throws, so the operation rejects. The spans must still be *finished* (and // therefore present in the transaction) with an error status — not left open. - const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.status).toBe('error'); const generateContent = spans.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; expect(generateContent.status).toBe('error'); }, @@ -1006,7 +1006,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); const generateContent = spans.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; // Cache/reasoning token breakdown and conversation id are derived from the model's @@ -1016,7 +1016,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(generateContent.attributes[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]?.value).toBe(7); expect(generateContent.attributes[GEN_AI_CONVERSATION_ID]?.value).toBe('resp_abc123'); - const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent).toBeDefined(); // The system prompt is supplied via the v7-only `instructions` option. Only the channel @@ -1054,13 +1054,13 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', ); const generateContent = spans.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; expect(generateContent.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(120); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(generateContent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); - const invokeAgent = spans.find(span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent')!; + const invokeAgent = spans.find(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent')!; expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]?.value).toBe(80); expect(invokeAgent.attributes[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]?.value).toBe(20); }, @@ -1102,7 +1102,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe )!; expect(embedSpan.name).toBe('embeddings mock-model-id'); expect(embedSpan.status).toBe('ok'); - expect(embedSpan.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); + expect(embedSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); expect(embedSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); expect(embedSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10); @@ -1112,7 +1112,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe if (embedManyInstrumented) { expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); - expect(embedManySpan!.attributes['sentry.op']?.value).toBe('gen_ai.embeddings'); + expect(embedManySpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.embeddings'); expect(embedManySpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(20); } else { expect(embedManySpan).toBeUndefined(); @@ -1151,7 +1151,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId']?.value).toBe('ai.generateObject'); expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('mock-model-id'); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(15); @@ -1160,7 +1160,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateObject.doGenerate', ); @@ -1193,7 +1193,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .expect({ span: container => { const invokeAgent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent', )!; expect(invokeAgent.status).toBe('error'); expect(invokeAgent.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('mock-model-id'); @@ -1202,7 +1202,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(invokeAgent.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); const generateContent = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.generate_content', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', )!; expect(generateContent.status).toBe('error'); }, @@ -1235,7 +1235,7 @@ describe('Vercel AI integration experimental_evaluate', () => { .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', )!; expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); @@ -1307,7 +1307,7 @@ describe('Vercel AI integration experimental_evaluate', () => { .expect({ span: container => { const evaluateSpan = container.items.find( - span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', )!; expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); From 52d54962be463239c1e7b1d0e1d4bb4ee62a9e55 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 14:15:04 +0200 Subject: [PATCH 4/8] test(node): Retain Vercel AI v4 transaction coverage Co-Authored-By: GPT-6 --- .../{ => v4-static}/instrument-with-pii.mjs | 1 + .../vercelai/{ => v4-static}/instrument.mjs | 1 + .../scenario-conversation-id.mjs | 0 .../{ => v4-static}/scenario-embeddings.mjs | 0 .../scenario-error-in-tool-express.mjs | 0 .../scenario-error-in-tool.mjs | 0 .../scenario-generate-object.mjs | 0 .../{ => v4-static}/scenario-stream-text.mjs | 0 .../scenario-system-instructions.mjs | 0 .../vercelai/{ => v4-static}/scenario.mjs | 0 .../tracing/vercelai/{ => v4-static}/test.ts | 194 ++++++++---------- 11 files changed, 92 insertions(+), 104 deletions(-) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/instrument-with-pii.mjs (90%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/instrument.mjs (92%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-conversation-id.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-embeddings.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-error-in-tool-express.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-error-in-tool.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-generate-object.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-stream-text.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario-system-instructions.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/scenario.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{ => v4-static}/test.ts (81%) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs similarity index 90% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs index 852786895514..43ea2c5b8d86 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument-with-pii.mjs @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ + traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs similarity index 92% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs index cc192fb89834..42052d281304 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/instrument.mjs @@ -2,6 +2,7 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ + traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-conversation-id.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-conversation-id.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-conversation-id.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-conversation-id.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-embeddings.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-embeddings.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-embeddings.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool-express.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool-express.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool-express.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool-express.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-error-in-tool.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-generate-object.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-generate-object.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-generate-object.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-generate-object.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-stream-text.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-stream-text.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-stream-text.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-system-instructions.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-system-instructions.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-system-instructions.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario-system-instructions.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/scenario.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts similarity index 81% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts index 5ed94f104da7..b3a159746121 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts @@ -1,4 +1,3 @@ -import type { SerializedStreamedSpan, SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -22,9 +21,9 @@ import { SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; -import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; -import { getStringAttributeValue } from '../../../utils'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; +import { getStringAttributeValue } from '../../../../utils'; const expectedOrigin = 'auto.vercelai.channel'; const toolCallArgs = '{\\"location\\":\\"San Francisco\\"}'; @@ -37,15 +36,11 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai spans when dataCollection.genAi has inputs and outputs disabled', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(7); - const firstInvokeAgentSpan = spans.find( + expect(container.items).toHaveLength(7); + const firstInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && @@ -64,7 +59,7 @@ describe('Vercel AI integration (v4)', () => { expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(20); expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); - const firstGenerateContentSpan = spans.find( + const firstGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && @@ -99,7 +94,7 @@ describe('Vercel AI integration (v4)', () => { expect(firstGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(firstGenerateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); - const secondInvokeAgentSpan = spans.find( + const secondInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === @@ -132,7 +127,7 @@ describe('Vercel AI integration (v4)', () => { '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', ); - const secondGenerateContentSpan = spans.find( + const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), @@ -142,7 +137,7 @@ describe('Vercel AI integration (v4)', () => { expect(secondGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(secondGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Second span here!'); - const toolInvokeAgentSpan = spans.find( + const toolInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, ); expect(toolInvokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); @@ -159,7 +154,7 @@ describe('Vercel AI integration (v4)', () => { expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(25); expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); - const toolGenerateContentSpan = spans.find( + const toolGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 15, @@ -187,7 +182,7 @@ describe('Vercel AI integration (v4)', () => { expect(toolGenerateContentSpan!.status).toBe('ok'); expect(toolGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -207,15 +202,11 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates ai spans for dataCollection defaults', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(7); - const firstInvokeAgentSpan = spans.find( + expect(container.items).toHaveLength(7); + const firstInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === @@ -247,7 +238,7 @@ describe('Vercel AI integration (v4)', () => { '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', ); - const firstGenerateContentSpan = spans.find( + const firstGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('First span here!'), @@ -292,7 +283,7 @@ describe('Vercel AI integration (v4)', () => { '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', }); - const secondInvokeAgentSpan = spans.find( + const secondInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === @@ -327,7 +318,7 @@ describe('Vercel AI integration (v4)', () => { expect(secondInvokeAgentSpan!.status).toBe('ok'); expect(secondInvokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); - const secondGenerateContentSpan = spans.find( + const secondGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_OUTPUT_MESSAGES]?.value)?.includes('Second span here!'), @@ -335,7 +326,7 @@ describe('Vercel AI integration (v4)', () => { expect(secondGenerateContentSpan!.status).toBe('ok'); expect(secondGenerateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolInvokeAgentSpan = spans.find( + const toolInvokeAgentSpan = container.items.find( span => span.name === 'invoke_agent' && span.attributes[GEN_AI_INPUT_MESSAGES]?.value === @@ -360,7 +351,7 @@ describe('Vercel AI integration (v4)', () => { value: `[{"role":"assistant","parts":[{"type":"text","content":"Tool call completed!"},{"type":"tool_call","id":"call-1","name":"getWeather","arguments":"${toolCallArgs}"}],"finish_reason":"tool_call"}]`, }); - const toolGenerateContentSpan = spans.find( + const toolGenerateContentSpan = container.items.find( span => span.name === 'generate_content mock-model-id' && getStringAttributeValue(span.attributes[GEN_AI_TOOL_DEFINITIONS]?.value)?.includes('getWeather'), @@ -393,7 +384,7 @@ describe('Vercel AI integration (v4)', () => { }); expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); - const toolExecutionSpan = spans.find(span => span.name === 'execute_tool getWeather'); + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); expect(toolExecutionSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); expect(toolExecutionSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -420,21 +411,24 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-error-in-tool.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let segment: SerializedStreamedSpan | undefined; + let transactionEvent: Event | undefined; let errorEvent: Event | undefined; await createRunner() // In orchestrion mode the tool error is captured mid-transaction, so the error and - // span containers can arrive in either order — assert content, not wire order. + // transaction/span envelopes can arrive in either order — assert content, not wire order. .unordered() + .expect({ + transaction: transaction => { + transactionEvent = transaction; + }, + }) .expect({ span: container => { - segment = container.items.find(span => span.is_segment); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + expect(container.items).toHaveLength(3); + const invokeAgentSpan = container.items.find( + span => span.name === 'invoke_agent' && span.status === 'error', ); - expect(spans).toHaveLength(3); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent' && span.status === 'error'); expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -443,7 +437,7 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', @@ -460,11 +454,12 @@ describe('Vercel AI integration (v4)', () => { expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); - const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); }, }) @@ -476,7 +471,7 @@ describe('Vercel AI integration (v4)', () => { .start() .completed(); - expect(segment!.name).toBe('main'); + expect(transactionEvent!.transaction).toBe('main'); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); @@ -493,37 +488,38 @@ describe('Vercel AI integration (v4)', () => { ); // Both paths stamp the operation's call-site span onto the bubbled error, so the global // unhandled-rejection handler restores it and correlates the report to the transaction's root span. - expect(errorEvent!.contexts!.trace!.trace_id).toBe(segment!.trace_id); - expect(errorEvent!.contexts!.trace!.span_id).toBe(segment!.span_id); + expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); + expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); }); }); createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool in express server', async () => { - let segment: SerializedStreamedSpan | undefined; - let spanItems: SerializedStreamedSpanContainer['items'] = []; + let transactionEvent: Event | undefined; let errorEvent: Event | undefined; const runner = createRunner() - // The error and span containers can arrive in either order, so assert content, not order. + // The error and transaction/span envelopes can arrive in either order, so assert content, not order. .unordered() + .expect({ + transaction: transaction => { + transactionEvent = transaction; + }, + }) .expect({ span: container => { - spanItems = container.items; - segment = container.items.find(span => span.is_segment); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + expect(container.items).toHaveLength(3); + const invokeAgentSpan = container.items.find( + span => span.name === 'invoke_agent' && span.status === 'error', ); - expect(spans).toHaveLength(3); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent' && span.status === 'error'); expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); expect(toolSpan!.status).toBe('error'); expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); @@ -539,10 +535,11 @@ describe('Vercel AI integration (v4)', () => { await runner.makeRequest('get', '/test/error-in-tool', { expectError: true }); await runner.completed(); - expect(segment!.name).toBe('GET /test/error-in-tool'); + expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); + expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); - expect(errorEvent!.contexts!.trace!.trace_id).toBe(segment!.trace_id); + expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError` and is // captured once by the express error handler — the channel subscriber deliberately doesn't @@ -555,8 +552,13 @@ describe('Vercel AI integration (v4)', () => { }), ]), ); - // The Express error handler captures the error on a child span of the request. - expect(spanItems.map(span => span.span_id)).toContain(errorEvent!.contexts!.trace!.span_id); + // The error is captured on the Express layer span (a child of the request transaction), so its + // `span_id` is one of the transaction's spans rather than the transaction's own (root) span. + const transactionSpanIds = [ + transactionEvent!.contexts!.trace!.span_id, + ...(transactionEvent!.spans ?? []).map(span => span.span_id), + ]; + expect(transactionSpanIds).toContain(errorEvent!.contexts!.trace!.span_id); }); }); @@ -568,21 +570,17 @@ describe('Vercel AI integration (v4)', () => { test('extracts system instructions from messages', async () => { await createRunner() .ignore('event') + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(2); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + expect(container.items).toHaveLength(2); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); }, }) @@ -595,21 +593,19 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates embedding related spans with genAI recording disabled', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(2); - const embedSpan = spans.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); + expect(container.items).toHaveLength(2); + const embedSpan = container.items.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 10); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); expect(embedSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); - const embedManySpan = spans.find(span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20); + const embedManySpan = container.items.find( + span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value === 20, + ); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); expect(embedManySpan!.status).toBe('ok'); expect(embedManySpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); @@ -623,20 +619,18 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates embedding related spans with genAI recording enabled', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', + expect(container.items).toHaveLength(2); + const embedSpan = container.items.find( + span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!', ); - expect(spans).toHaveLength(2); - const embedSpan = spans.find(span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Embedding test!'); expect(embedSpan!.name).toBe('embeddings mock-model-id'); expect(embedSpan!.status).toBe('ok'); expect(embedSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.embeddings'); - const embedManySpan = spans.find( + const embedManySpan = container.items.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === '["First input","Second input"]', ); expect(embedManySpan!.name).toBe('embeddings mock-model-id'); @@ -652,19 +646,15 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('does not overwrite conversation id set via Sentry.setConversationId with responseId from provider metadata', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(2); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + expect(container.items).toHaveLength(2); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes[GEN_AI_CONVERSATION_ID].value).toBe('conv-a'); }, @@ -677,18 +667,15 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-stream-text.mjs', 'instrument.mjs', (createRunner, test) => { test('creates ai spans for streamText (doStream)', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(2); + expect(container.items).toHaveLength(2); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText'); expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); // Aggregated over the drained stream: v4 reports `promptTokens`/`completionTokens`, which the @@ -698,9 +685,10 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30); expect(invokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES].value).toContain('Stream response!'); - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.streamText.doStream'); }, }) @@ -712,20 +700,17 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-generate-object.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('captures generateObject spans with schema attributes', async () => { await createRunner() + .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(2); + expect(container.items).toHaveLength(2); // generateObject (invoke_agent) - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); expect(invokeAgentSpan!.status).toBe('ok'); expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); expect(invokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject'); + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(invokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model-id'); expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(15); @@ -733,10 +718,11 @@ describe('Vercel AI integration (v4)', () => { expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(40); // generateObject.doGenerate (generate_content) - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); expect(generateContentSpan!.status).toBe('ok'); expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); expect(generateContentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateObject.doGenerate'); + expect(generateContentSpan!.attributes[SENTRY_ORIGIN].value).toBe(expectedOrigin); expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('generate_content'); expect(generateContentSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('mock-provider'); expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('mock-model-id'); From f5fd775d81879fa8d4519da6accd2c44854e54d8 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 09:59:43 +0200 Subject: [PATCH 5/8] test(node): Report Vercel AI assertions without unordered retries Co-Authored-By: GPT-6 --- .../suites/tracing/vercelai/v5/test.ts | 37 +++---- .../suites/tracing/vercelai/v6_v7/test.ts | 97 +++++++++---------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index 34250d07fc6c..c3a1efe546de 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -1,4 +1,4 @@ -import type { SerializedStreamedSpan } from '@sentry/core'; +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -225,7 +225,7 @@ describe('Vercel AI integration (v5)', () => { 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let segment: SerializedStreamedSpan | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; await createRunner() @@ -234,22 +234,8 @@ describe('Vercel AI integration (v5)', () => { .unordered() .expect({ span: container => { - segment = container.items.find(span => span.is_segment); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(3); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); - - const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .expect({ @@ -260,6 +246,21 @@ describe('Vercel AI integration (v5)', () => { .start() .completed(); + const segment = receivedSpans.find(span => span.is_segment); + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.generate_content'); + + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather'); + expect(toolSpan!.status).toBe('error'); + expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); + expect(segment!.name).toBe('main'); expect(errorEvent!.level).toBe('error'); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 740138a65519..bd6eb0809811 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1,4 +1,4 @@ -import type { SerializedStreamedSpan } from '@sentry/core'; +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import type { Event } from '@sentry/node'; import { afterAll, describe, expect } from 'vitest'; import { @@ -518,58 +518,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'instrument.mjs', (createRunner, test) => { test('captures error in tool', async () => { - let segment: SerializedStreamedSpan | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; let errorEvent: Event | undefined; await createRunner() .unordered() .expect({ span: container => { - segment = container.items.find(span => span.is_segment); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel', - ); - expect(spans).toHaveLength(3); - const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent')!; - expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); - expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'invoke_agent', - }); - expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id')!; - expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); - expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ - type: 'string', - value: 'generate_content', - }); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ - type: 'integer', - value: 40, - }); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ - type: 'integer', - value: 25, - }); - expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ - type: 'integer', - value: 15, - }); - expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ - type: 'string', - value: 'mock-model-id', - }); - expect(generateContentSpan.status).toBe('ok'); - expect(generateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); - - const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; - expect(toolSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); - expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); - expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); - expect(toolSpan.status).toBe('error'); - expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); - expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .expect({ @@ -580,6 +537,50 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .start() .completed(); + const segment = receivedSpans.find(span => span.is_segment); + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); + expect(spans).toHaveLength(3); + const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent')!; + expect(invokeAgentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'invoke_agent', + }); + expect(invokeAgentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = spans.find(span => span.name === 'generate_content mock-model-id')!; + expect(generateContentSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(generateContentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ + type: 'string', + value: 'generate_content', + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({ + type: 'integer', + value: 40, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({ + type: 'integer', + value: 25, + }); + expect(generateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({ + type: 'integer', + value: 15, + }); + expect(generateContentSpan!.attributes[GEN_AI_REQUEST_MODEL]).toEqual({ + type: 'string', + value: 'mock-model-id', + }); + expect(generateContentSpan.status).toBe('ok'); + expect(generateContentSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.generate_content'); + + const toolSpan = spans.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan!.attributes[SENTRY_ORIGIN]).toEqual({ type: 'string', value: expectedOrigin }); + expect(toolSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({ type: 'string', value: 'execute_tool' }); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]).toEqual({ type: 'string', value: 'call-1' }); + expect(toolSpan.status).toBe('error'); + expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); + expect(segment!.name).toBe('main'); expect(errorEvent!.level).toBe('error'); @@ -1231,7 +1232,6 @@ describe('Vercel AI integration experimental_evaluate', () => { (createRunner, test) => { test('creates an evaluate span', async () => { await createRunner() - .unordered() .expect({ span: container => { const evaluateSpan = container.items.find( @@ -1303,7 +1303,6 @@ describe('Vercel AI integration experimental_evaluate', () => { (createRunner, test) => { test('does not record inputs or outputs when recording is off', async () => { await createRunner() - .unordered() .expect({ span: container => { const evaluateSpan = container.items.find( From 68ac39e11d994f26530bd581132cf193ab1c3de5 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 15:32:40 +0200 Subject: [PATCH 6/8] test(node): Clean up Vercel AI streaming assertions Co-Authored-By: GPT-6 --- .../suites/tracing/vercelai/v5/test.ts | 4 +--- .../suites/tracing/vercelai/v6_v7/test.ts | 16 ++++++---------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index c3a1efe546de..d08b1e545362 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -246,7 +246,7 @@ describe('Vercel AI integration (v5)', () => { .start() .completed(); - const segment = receivedSpans.find(span => span.is_segment); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'main'); const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); expect(spans).toHaveLength(3); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent'); @@ -261,8 +261,6 @@ describe('Vercel AI integration (v5)', () => { expect(toolSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.execute_tool'); expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); - expect(segment!.name).toBe('main'); - expect(errorEvent!.level).toBe('error'); expect(errorEvent!.tags).toEqual( expect.objectContaining({ diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index e7b3de00f59c..7de580aea1d1 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -537,7 +537,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe .start() .completed(); - const segment = receivedSpans.find(span => span.is_segment); + const segment = receivedSpans.find(span => span.is_segment && span.name === 'main'); const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.vercelai.channel'); expect(spans).toHaveLength(3); const invokeAgentSpan = spans.find(span => span.name === 'invoke_agent')!; @@ -581,8 +581,6 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(toolSpan.attributes[SENTRY_OP]?.value).toBe('gen_ai.execute_tool'); expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); - expect(segment!.name).toBe('main'); - expect(errorEvent!.level).toBe('error'); expect(errorEvent!.tags).toEqual( expect.objectContaining({ @@ -1048,17 +1046,15 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe (createRunner, test) => { test('derives gen_ai.conversation.id from the OpenAI `conversation` provider option', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); const genAiSpans = container.items.filter(s => - String(s.attributes['sentry.op']?.value ?? '').startsWith('gen_ai.'), + String(s.attributes[SENTRY_OP]?.value ?? '').startsWith('gen_ai.'), ); const conversationIdOf = (span: (typeof genAiSpans)[number]) => span.attributes[GEN_AI_CONVERSATION_ID]?.value; - const invokeAgentSpans = genAiSpans.filter( - s => s.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - ); + const invokeAgentSpans = genAiSpans.filter(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent'); expect(invokeAgentSpans).toHaveLength(4); const [firstTurn, secondTurn, chainedTurn, apiTurn] = invokeAgentSpans.sort( (a, b) => a.start_timestamp - b.start_timestamp, @@ -1077,7 +1073,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe // Model-call and tool spans carry their operation's id, even though their start events // do not carry `providerOptions`. const modelCallSpans = genAiSpans.filter( - s => s.attributes['sentry.op']?.value === 'gen_ai.generate_content', + s => s.attributes[SENTRY_OP]?.value === 'gen_ai.generate_content', ); expect(modelCallSpans.map(conversationIdOf).sort()).toEqual([ 'conv-from-api', @@ -1085,7 +1081,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe 'conv_azure', undefined, ]); - const toolSpan = genAiSpans.find(s => s.attributes['sentry.op']?.value === 'gen_ai.execute_tool')!; + const toolSpan = genAiSpans.find(s => s.attributes[SENTRY_OP]?.value === 'gen_ai.execute_tool')!; expect(toolSpan).toBeDefined(); expect(conversationIdOf(toolSpan)).toBe('conv_azure'); }, From 7c8da72e832e8ddb5cbe77494e53db8695807b29 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 15:43:43 +0200 Subject: [PATCH 7/8] test(node): Preserve Vercel AI string attribute checks Co-Authored-By: GPT-6 --- .../suites/tracing/vercelai/v4-static/test.ts | 6 +++--- .../suites/tracing/vercelai/v6_v7/test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts index 6ed07eb586c1..99604cb8a599 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v4-static/test.ts @@ -276,7 +276,7 @@ describe('Vercel AI integration (v4)', () => { expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( 'ai.generateText.doGenerate', ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES]?.value).toEqual(expect.any(String)); expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ type: 'string', value: @@ -400,8 +400,8 @@ describe('Vercel AI integration (v4)', () => { expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION].value).toBe( 'Get the current weather for a location', ); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]).toBeDefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_RESULT]).toBeDefined(); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toEqual(expect.any(String)); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toEqual(expect.any(String)); }, }) .start() diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 7de580aea1d1..208062418686 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -137,7 +137,7 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(firstGenerateContentSpan.attributes['vercel.ai.operationId']?.value).toBe( 'ai.generateText.doGenerate', ); - expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(firstGenerateContentSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value).toEqual(expect.any(String)); expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]).toEqual({ type: 'string', value: @@ -281,8 +281,8 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(toolExecutionSpan.attributes[GEN_AI_TOOL_DESCRIPTION]?.value).toBe( 'Get the current weather for a location', ); - expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]).toBeDefined(); - expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_RESULT]).toBeDefined(); + expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_ARGUMENTS]?.value).toEqual(expect.any(String)); + expect(toolExecutionSpan.attributes[GEN_AI_TOOL_CALL_RESULT]?.value).toEqual(expect.any(String)); }, }) .start() From 438afa8d3ef48d626d98147e189eb908c962ef65 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Oct 2026 14:04:50 +0200 Subject: [PATCH 8/8] test(node): exactly match Vercel AI tool error tags Co-Authored-By: GPT-6 --- .../suites/tracing/vercelai/v5/test.ts | 10 ++++------ .../suites/tracing/vercelai/v6_v7/test.ts | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts index d08b1e545362..b608a1f717fc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v5/test.ts @@ -262,12 +262,10 @@ describe('Vercel AI integration (v5)', () => { expect(toolSpan!.attributes[GEN_AI_TOOL_NAME].value).toBe('getWeather'); expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); + expect(errorEvent!.tags).toEqual({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); // Trace id should be the same for the segment and error event expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 208062418686..893cea265ebb 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -582,12 +582,10 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe expect(toolSpan.attributes[GEN_AI_TOOL_NAME]?.value).toBe('getWeather'); expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); + expect(errorEvent!.tags).toEqual({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }); // Trace id should be the same for the segment and error event expect(segment!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id);