diff --git a/dev-packages/bun-integration-tests/node-suites/excludes.ts b/dev-packages/bun-integration-tests/node-suites/excludes.ts index d27790a79a0f..a8bba26f4713 100644 --- a/dev-packages/bun-integration-tests/node-suites/excludes.ts +++ b/dev-packages/bun-integration-tests/node-suites/excludes.ts @@ -100,7 +100,7 @@ export const NO_AUTO_INSTRUMENTATION = [ 'suites/tracing/knex/**', 'suites/tracing/koa/test.ts', 'suites/tracing/langchain/**', - 'suites/tracing/langgraph/test.ts', + 'suites/tracing/langgraph/**', 'suites/tracing/lru-memoizer/test.ts', 'suites/tracing/mastra/test.ts', 'suites/tracing/mcp-handler-exact-once/test.ts', @@ -244,6 +244,7 @@ const BUN_BUILD_NOT_TRIAGED = [ 'suites/tracing/google-genai-v2/test.ts', 'suites/tracing/google-genai/test.ts', 'suites/tracing/langchain/v1/test.ts', + 'suites/tracing/langgraph/v1/test.ts', 'suites/tracing/mastra/test.ts', 'suites/tracing/mcp-handler-exact-once/test.ts', 'suites/tracing/mcp-server-streamed/test.ts', diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs new file mode 100644 index 000000000000..d9d56910381b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-model-router.mjs @@ -0,0 +1,69 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import { modelRouterMiddleware } from '@langchain/typesafe/middleware'; +import * as Sentry from '@sentry/node'; +import express from 'express'; +import { createAgent } from 'langchain'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + res.json({ + id: 'msg_router_test', + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'A refund is on its way.' }], + model: req.body.model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 5 }, + }); + }); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: 'jev-1.13', + answers: { + model_route: { type: 'choice', choice: 'fast', probabilities: { fast: 0.9, smart: 0.1 }, confidence: 0.8 }, + }, + usage: { input_tokens: 40, output_tokens: 3 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + const model = name => new ChatAnthropic({ model: name, apiKey: 'mock-api-key', clientOptions: { baseURL: baseUrl } }); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const agent = createAgent({ + name: 'support_agent', + model: model('claude-3-5-sonnet-20241022'), + middleware: [ + modelRouterMiddleware({ + instructions: 'Pick the model for this request.', + choices: { + fast: { model: model('claude-3-5-haiku-20241022'), criteria: 'Simple requests' }, + smart: { model: model('claude-3-5-sonnet-20241022'), criteria: 'Complex requests' }, + }, + classifierOptions: { apiKey: 'mock-api-key', baseUrl }, + }), + ], + }); + + await agent.invoke({ messages: [{ role: 'user', content: 'Where is my refund?' }] }); + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index ae4f7a1554ef..f14ccb93169d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -1,5 +1,6 @@ import { afterAll, describe, expect } from 'vitest'; import { + GEN_AI_AGENT_NAME, GEN_AI_CONVERSATION_ID, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, @@ -317,6 +318,65 @@ describe('LangChain integration (v1)', () => { }, ); + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-model-router.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records the Jev model router call inside a createAgent run', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const evaluateSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)!; + const chatSpan = spans.find(span => span.name === 'chat claude-3-5-haiku-20241022')!; + + // The router classifies in `beforeAgent`, inside the agent run, where other chain steps are skipped. + expect(evaluateSpan.name).toBe('evaluate jev-latest'); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent'); + expect(evaluateSpan.parent_span_id).toBe(chatSpan.parent_span_id); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + // The router passes the latest human message, which the classifier sends as a transcript line. + state: 'user: Where is my refund?', + questions: { + model_route: { + type: 'choice', + instructions: 'Pick the model for this request.', + criteria: { fast: 'Simple requests', smart: 'Complex requests' }, + }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + answers: { + model_route: { + type: 'choice', + choice: 'fast', + probabilities: { fast: 0.9, smart: 0.1 }, + confidence: 0.8, + }, + }, + }, + ]); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); + createEsmTests( __dirname, 'scenario-openai-before-langchain.mjs', diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs new file mode 100644 index 000000000000..1c507ba84b9c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument-with-pii.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: true, outputs: true } }, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs new file mode 100644 index 000000000000..cc192fb89834 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/instrument.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: false, outputs: false } }, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs new file mode 100644 index 000000000000..33256a644def --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/scenario-typesafe-classifier-node.mjs @@ -0,0 +1,37 @@ +import { Annotation, END, START, StateGraph } from '@langchain/langgraph'; +import { TypeSafeClassifier } from '@langchain/typesafe'; +import * as Sentry from '@sentry/node'; + +// Answer in-process through the classifier's `fetch` option, so no mock server is needed. +async function mockTypeSafeFetch() { + return new Response( + JSON.stringify({ + model: 'jev-1.13', + answers: { urgent: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 30, output_tokens: 2 }, + }), + { headers: { 'content-type': 'application/json' } }, + ); +} + +async function run() { + const classifier = new TypeSafeClassifier({ + apiKey: 'mock-api-key', + fetch: mockTypeSafeFetch, + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }); + + const State = Annotation.Root({ ticket: Annotation(), urgent: Annotation() }); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const graph = new StateGraph(State) + .addNode('triage', async state => ({ urgent: (await classifier.invoke(state.ticket)).nouls.urgent.noul })) + .addEdge(START, 'triage') + .addEdge('triage', END) + .compile({ name: 'triage_graph' }); + + await graph.invoke({ ticket: 'My payouts have been failing.' }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts new file mode 100644 index 000000000000..c6a71539de1a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langgraph/v1/test.ts @@ -0,0 +1,88 @@ +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OUTPUT_MESSAGES, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe('LangGraph integration (v1)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier-node.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records a TypeSafeClassifier call made inside a StateGraph node', async () => { + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const rootSpan = spans.find(span => span.is_segment && span.name === 'main')!; + const agentSpan = spans.find(span => span.name === 'invoke_agent triage_graph')!; + const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!; + + expect(agentSpan.parent_span_id).toBe(rootSpan.span_id); + expect(evaluateSpan.parent_span_id).toBe(agentSpan.span_id); + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph'); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: 'My payouts have been failing.', + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, + ]); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/langgraph': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier-node.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('omits the classifier input and output when genAI recording is off', async () => { + const runner = createRunner(); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!; + + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/langgraph': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + }, + }, + ); +});