diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs index e60b44f864d7..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/embeddings')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs index c81bb4dc1322..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/embeddings')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 290cf38bcba6..48767041abf9 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -18,6 +18,9 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + LANGCHAIN_CHAIN_NAME, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_EMBEDDINGS } from '@sentry/conventions/op'; import { @@ -35,21 +38,21 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat claude-3-5-sonnet-20241022', 'chat claude-3-opus-20240229', 'chat error-model', ]); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(sonnetSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); @@ -62,11 +65,10 @@ describe('LangChain integration', () => { expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); expect(opusSpan!.status).toBe('ok'); - expect(opusSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(opusSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(opusSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(opusSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); @@ -76,11 +78,10 @@ describe('LangChain integration', () => { expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }, @@ -88,44 +89,26 @@ describe('LangChain integration', () => { .start() .completed(); }); - - test('does not create duplicate spans from double module patching', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - // The scenario makes 3 LangChain calls (2 successful + 1 error). - // Without the dedup guard, the file-level and module-level hooks - // both patch the same prototype, producing 6 spans instead of 3. - expect(container.items).toHaveLength(3); - for (const span of container.items) { - expect(span.attributes['sentry.op'].value).toBe('gen_ai.chat'); - } - }, - }) - .start() - .completed(); - }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat claude-3-5-sonnet-20241022', 'chat claude-3-opus-20240229', 'chat error-model', ]); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); @@ -139,8 +122,7 @@ describe('LangChain integration', () => { expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); expect(opusSpan!.status).toBe('ok'); expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); @@ -153,8 +135,7 @@ describe('LangChain integration', () => { expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan!.status).toBe('error'); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); @@ -169,17 +150,18 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain spans with tool calls', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] chat with tool_use stop reason expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(firstSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); @@ -198,31 +180,23 @@ describe('LangChain integration', () => { createEsmTests(__dirname, 'scenario-openai-before-langchain.mjs', 'instrument.mjs', (createRunner, test) => { test('demonstrates timing issue with duplicate spans', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(2); - const anthropicSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.anthropic', - ); - expect(anthropicSpan).toBeDefined(); - expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - - // LangChain call is instrumented by LangChain. - const langchainSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.langchain', - ); - expect(langchainSpan).toBeDefined(); - expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - - // Third call (not present): Direct Anthropic call made AFTER LangChain import - // is NOT instrumented, which demonstrates the skip mechanism works for NEW - // clients. We should only have ONE Anthropic span (the first one), not two. - }, - }) - .start() - .completed(); + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain' || + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(2); + const anthropicSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.anthropic'); + expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + + // LangChain call is instrumented by LangChain. + const langchainSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.langchain'); + expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); }); }); @@ -233,15 +207,18 @@ describe('LangChain integration', () => { (createRunner, test) => { test('extracts system instructions from messages', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] chat with extracted system instructions expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe( JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]), ); @@ -256,42 +233,41 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario-chain.mjs', 'instrument.mjs', (createRunner, test) => { test('uses runName for chain spans instead of unknown_chain', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chain format_prompt', - 'chain parse_output', - 'chain unknown_chain', + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(4); + expect(spans.map(span => span.name).sort()).toEqual([ 'chat claude-3-5-sonnet-20241022', + 'invoke_agent', + 'invoke_agent format_prompt', + 'invoke_agent parse_output', ]); - const formatPromptSpan = container.items.find(span => span.name === 'chain format_prompt'); - expect(formatPromptSpan).toBeDefined(); - expect(formatPromptSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(formatPromptSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + const formatPromptSpan = spans.find(span => span.name === 'invoke_agent format_prompt'); + expect(formatPromptSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(formatPromptSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(formatPromptSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(formatPromptSpan!.attributes[GEN_AI_PIPELINE_NAME].value).toBe('format_prompt'); - expect(formatPromptSpan!.attributes['langchain.chain.name']).toBeUndefined(); + expect(formatPromptSpan!.attributes[LANGCHAIN_CHAIN_NAME]).toBeUndefined(); - const chatSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(chatSpan).toBeDefined(); - expect(chatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(chatSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + const chatSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); + expect(chatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(chatSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); - const parseOutputSpan = container.items.find(span => span.name === 'chain parse_output'); - expect(parseOutputSpan).toBeDefined(); - expect(parseOutputSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(parseOutputSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + const parseOutputSpan = spans.find(span => span.name === 'invoke_agent parse_output'); + expect(parseOutputSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(parseOutputSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(parseOutputSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(parseOutputSpan!.attributes[GEN_AI_PIPELINE_NAME].value).toBe('parse_output'); - expect(parseOutputSpan!.attributes['langchain.chain.name']).toBeUndefined(); + expect(parseOutputSpan!.attributes[LANGCHAIN_CHAIN_NAME]).toBeUndefined(); - const unknownChainSpan = container.items.find(span => span.name === 'chain unknown_chain'); - expect(unknownChainSpan).toBeDefined(); - expect(unknownChainSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + const unknownChainSpan = spans.find(span => span.name === 'invoke_agent'); + expect(unknownChainSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.invoke_agent'); + expect(unknownChainSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('invoke_agent'); expect(unknownChainSpan!.attributes[GEN_AI_PIPELINE_NAME]).toBeUndefined(); - expect(unknownChainSpan!.attributes['langchain.chain.name']).toBeUndefined(); + expect(unknownChainSpan!.attributes[LANGCHAIN_CHAIN_NAME]).toBeUndefined(); }, }) .start() @@ -306,89 +282,69 @@ describe('LangChain integration', () => { createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument.mjs', (createRunner, test) => { test('creates embedding spans with genAI recording disabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'embeddings error-model', 'embeddings text-embedding-3-small', 'embeddings text-embedding-3-small', ]); - const successfulSpans = container.items.filter( + const successfulSpans = spans.filter( span => span.name === 'embeddings text-embedding-3-small' && span.status === 'ok', ); expect(successfulSpans).toHaveLength(2); for (const span of successfulSpans) { - expect(span.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS); - expect(span.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); + expect(span.attributes[SENTRY_OP].value).toBe(GEN_AI_EMBEDDINGS); + expect(span.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('embeddings'); expect(span.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('text-embedding-3-small'); expect(span.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE].value).toBe(1536); } - const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe(GEN_AI_EMBEDDINGS); expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); }, }) .start() .completed(); }); - - test('does not create duplicate embedding spans from double module patching', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - // The scenario makes 3 embedding calls (2 successful + 1 error). - expect(container.items).toHaveLength(3); - for (const span of container.items) { - expect(span.attributes['sentry.op'].value).toBe(GEN_AI_EMBEDDINGS); - } - }, - }) - .start() - .completed(); - }); }); createEsmAndCjsTests(__dirname, 'scenario-embeddings.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates embedding spans with genAI recording enabled', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ 'embeddings error-model', 'embeddings text-embedding-3-small', 'embeddings text-embedding-3-small', ]); - const querySpan = container.items.find( - span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Hello world', - ); - expect(querySpan).toBeDefined(); + const querySpan = spans.find(span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === 'Hello world'); expect(querySpan!.name).toBe('embeddings text-embedding-3-small'); expect(querySpan!.status).toBe('ok'); expect(querySpan!.attributes[GEN_AI_REQUEST_DIMENSIONS_ATTRIBUTE].value).toBe(1536); - const documentsSpan = container.items.find( + const documentsSpan = spans.find( span => span.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value === JSON.stringify(['First document', 'Second document']), ); - expect(documentsSpan).toBeDefined(); expect(documentsSpan!.name).toBe('embeddings text-embedding-3-small'); expect(documentsSpan!.status).toBe('ok'); - const errorSpan = container.items.find(span => span.name === 'embeddings error-model'); - expect(errorSpan).toBeDefined(); + const errorSpan = spans.find(span => span.name === 'embeddings error-model'); expect(errorSpan!.status).toBe('error'); }, }) @@ -397,44 +353,25 @@ describe('LangChain integration', () => { }); }); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates langchain related spans with span streaming enabled', async () => { - await createRunner() - .expect({ - span: container => { - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); - expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(sonnetSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }); - createEsmAndCjsTests(__dirname, 'scenario-conversation-id.mjs', 'instrument.mjs', (createRunner, test) => { test('derives gen_ai.conversation.id from the invoke config and lets a scope id win', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { // Four invocations, each producing the outer chain span, the lambda step span and the chat span. - expect(container.items).toHaveLength(12); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain'); + expect(spans).toHaveLength(12); const idsByName = (name: string): (string | undefined)[] => - container.items + spans .filter(span => span.name === name) .sort((a, b) => a.start_timestamp - b.start_timestamp) .map(span => span.attributes[GEN_AI_CONVERSATION_ID]?.value as string | undefined); // The id reaches the chain span, the child lambda span and the child chat span alike. const expected = ['thread_from_config', 'session_from_config', 'conversation_from_scope', undefined]; - expect(idsByName('chain weather_chain')).toEqual(expected); - expect(idsByName('chain format_prompt')).toEqual(expected); + expect(idsByName('invoke_agent weather_chain')).toEqual(expected); + expect(idsByName('invoke_agent format_prompt')).toEqual(expected); expect(idsByName('chat claude-3-5-sonnet-20241022')).toEqual(expected); }, }) @@ -442,39 +379,4 @@ describe('LangChain integration', () => { .completed(); }); }); - - createEsmAndCjsTests(__dirname, 'scenario-chain.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('leads chain span names with the operation when span streaming is enabled', async () => { - await createRunner() - .ignore('event') - .expect({ - span: container => { - const chainSpans = container.items.filter( - span => span.attributes['sentry.op']?.value === 'gen_ai.invoke_agent', - ); - // The `unknown_chain` sentinel is dropped, so that span falls back to the bare operation. - expect(chainSpans.map(span => span.name).sort()).toEqual([ - 'invoke_agent', - 'invoke_agent format_prompt', - 'invoke_agent parse_output', - ]); - for (const span of chainSpans) { - expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('invoke_agent'); - } - expect( - chainSpans - .map(span => span.attributes[GEN_AI_PIPELINE_NAME]?.value) - .filter(Boolean) - .sort(), - ).toEqual(['format_prompt', 'parse_output']); - const unnamedChainSpan = chainSpans.find(span => span.name === 'invoke_agent'); - expect(unnamedChainSpan).toBeDefined(); - expect(unnamedChainSpan!.attributes[GEN_AI_PIPELINE_NAME]).toBeUndefined(); - expect(unnamedChainSpan!.attributes['langchain.chain.name']).toBeUndefined(); - }, - }) - .start() - .completed(); - }); - }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs index a220d64b148a..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs index 8a2ba440b758..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 669ac68afa34..41158cb85db1 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -15,6 +15,8 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; @@ -31,60 +33,56 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording disabled', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat claude-3-5-sonnet-20241022', - 'chat claude-3-opus-20240229', - 'chat error-model', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); - expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(sonnetSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + await runner.start().completed(); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); - expect(opusSpan!.status).toBe('ok'); - expect(opusSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(opusSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); - expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat claude-3-5-sonnet-20241022', + 'chat claude-3-opus-20240229', + 'chat error-model', + ]); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - }, - }) - .start() - .completed(); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.status).toBe('ok'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); + expect(opusSpan!.status).toBe('ok'); + expect(opusSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(opusSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); + expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }); }, { @@ -102,60 +100,56 @@ describe('LangChain integration (v1)', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('creates langchain related spans with genAI recording enabled', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat claude-3-5-sonnet-20241022', - 'chat claude-3-opus-20240229', - 'chat error-model', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); - const sonnetSpan = container.items.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); - expect(sonnetSpan).toBeDefined(); - expect(sonnetSpan!.status).toBe('ok'); - expect(sonnetSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(sonnetSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(sonnetSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual([ + 'chat claude-3-5-sonnet-20241022', + 'chat claude-3-opus-20240229', + 'chat error-model', + ]); - const opusSpan = container.items.find(span => span.name === 'chat claude-3-opus-20240229'); - expect(opusSpan).toBeDefined(); - expect(opusSpan!.status).toBe('ok'); - expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); - expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); - expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); - expect(opusSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(opusSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + const sonnetSpan = spans.find(span => span.name === 'chat claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.status).toBe('ok'); + expect(sonnetSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(sonnetSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(sonnetSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(sonnetSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(sonnetSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE]).toBeDefined(); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(sonnetSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - }, - }) - .start() - .completed(); + const opusSpan = spans.find(span => span.name === 'chat claude-3-opus-20240229'); + expect(opusSpan!.status).toBe('ok'); + expect(opusSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-opus-20240229'); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.9); + expect(opusSpan!.attributes[GEN_AI_REQUEST_TOP_P].value).toBe(0.95); + expect(opusSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(200); + expect(opusSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(opusSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); + expect(opusSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(opusSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(opusSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + expect(errorSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); }); }, { @@ -173,32 +167,31 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('creates langchain spans with tool calls', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; - // [0] chat with tool_use stop reason - expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(firstSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(firstSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); - expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); - expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); - expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(30); - expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(50); - expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('tool_use'); - expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS]).toBeDefined(); - }, - }) - .start() - .completed(); + // [0] chat with tool_use stop reason + expect(firstSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(firstSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-5-sonnet-20241022'); + expect(firstSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(firstSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(150); + expect(firstSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20); + expect(firstSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(30); + expect(firstSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(50); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('tool_use'); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS]).toBeDefined(); }); }, { @@ -216,20 +209,19 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('derives gen_ai.conversation.id from the invoke config and lets a scope id win', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(4); - const ids = container.items - .sort((a, b) => a.start_timestamp - b.start_timestamp) - .map(span => span.attributes[GEN_AI_CONVERSATION_ID]?.value); - expect(ids).toEqual(['thread_from_config', 'session_from_config', 'conversation_from_scope', undefined]); - }, - }) - .start() - .completed(); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(4); + const ids = spans + .sort((a, b) => a.start_timestamp - b.start_timestamp) + .map(span => span.attributes[GEN_AI_CONVERSATION_ID]?.value); + expect(ids).toEqual(['thread_from_config', 'session_from_config', 'conversation_from_scope', undefined]); }); }, { @@ -247,30 +239,22 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('demonstrates timing issue with duplicate spans', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(2); - const anthropicSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.anthropic', - ); - expect(anthropicSpan).toBeDefined(); - expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); - const langchainSpan = container.items.find( - span => span.attributes['sentry.origin'].value === 'auto.ai.langchain', - ); - expect(langchainSpan).toBeDefined(); - expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); + const spans = (await spansPromise).filter( + span => + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain' || + span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(2); + const anthropicSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.anthropic'); + expect(anthropicSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); - // Third call (not present): Direct Anthropic call made AFTER LangChain import - // is NOT instrumented, demonstrating the skip mechanism works for NEW clients. - }, - }) - .start() - .completed(); + const langchainSpan = spans.find(span => span.attributes[SENTRY_ORIGIN].value === 'auto.ai.langchain'); + expect(langchainSpan!.name).toBe('chat claude-3-5-sonnet-20241022'); }); }, { @@ -288,60 +272,52 @@ describe('LangChain integration (v1)', () => { 'instrument.mjs', (createRunner, test) => { test('creates langchain spans using initChatModel with OpenAI', async () => { - await createRunner() - .ignore('event') - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - expect(container.items.map(span => span.name).sort()).toEqual([ - 'chat error-model', - 'chat gpt-3.5-turbo', - 'chat gpt-4o', - ]); + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = (await spansPromise).filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain', + ); + expect(spans).toHaveLength(3); + expect(spans.map(span => span.name).sort()).toEqual(['chat error-model', 'chat gpt-3.5-turbo', 'chat gpt-4o']); - const gpt4oSpan = container.items.find(span => span.name === 'chat gpt-4o'); - expect(gpt4oSpan).toBeDefined(); - expect(gpt4oSpan!.status).toBe('ok'); - expect(gpt4oSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(gpt4oSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(gpt4oSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(gpt4oSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o'); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(gpt4oSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o'); - expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + const gpt4oSpan = spans.find(span => span.name === 'chat gpt-4o'); + expect(gpt4oSpan!.status).toBe('ok'); + expect(gpt4oSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(gpt4oSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(gpt4oSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(gpt4oSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-4o'); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(gpt4oSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(gpt4oSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_ID]).toBeDefined(); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-4o'); + expect(gpt4oSpan!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); - const gpt35Span = container.items.find(span => span.name === 'chat gpt-3.5-turbo'); - expect(gpt35Span).toBeDefined(); - expect(gpt35Span!.status).toBe('ok'); - expect(gpt35Span!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(gpt35Span!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(gpt35Span!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(gpt35Span!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-3.5-turbo'); - expect(gpt35Span!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); - expect(gpt35Span!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); - expect(gpt35Span!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); - expect(gpt35Span!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); - expect(gpt35Span!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-3.5-turbo'); - expect(gpt35Span!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); + const gpt35Span = spans.find(span => span.name === 'chat gpt-3.5-turbo'); + expect(gpt35Span!.status).toBe('ok'); + expect(gpt35Span!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(gpt35Span!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(gpt35Span!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(gpt35Span!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('gpt-3.5-turbo'); + expect(gpt35Span!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.5); + expect(gpt35Span!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(8); + expect(gpt35Span!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(12); + expect(gpt35Span!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(20); + expect(gpt35Span!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('gpt-3.5-turbo'); + expect(gpt35Span!.attributes[GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE].value).toBe('stop'); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(errorSpan!.attributes['sentry.origin'].value).toBe('auto.ai.langchain'); - expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - }, - }) - .start() - .completed(); + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(errorSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('openai'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); }); }, {