Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ export const NO_AUTO_INSTRUMENTATION = [
'suites/tracing/knex/**',
'suites/tracing/koa/test.ts',
'suites/tracing/langchain/**',
'suites/tracing/langgraph/test.ts',
'suites/tracing/langgraph/**',
'suites/tracing/lru-memoizer/test.ts',
'suites/tracing/mastra/test.ts',
'suites/tracing/mcp-handler-exact-once/test.ts',
Expand Down Expand Up @@ -244,6 +244,7 @@ const BUN_BUILD_NOT_TRIAGED = [
'suites/tracing/google-genai-v2/test.ts',
'suites/tracing/google-genai/test.ts',
'suites/tracing/langchain/v1/test.ts',
'suites/tracing/langgraph/v1/test.ts',
Comment thread
cursor[bot] marked this conversation as resolved.
'suites/tracing/mastra/test.ts',
'suites/tracing/mcp-handler-exact-once/test.ts',
'suites/tracing/mcp-server-streamed/test.ts',
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import { ChatAnthropic } from '@langchain/anthropic';
import { modelRouterMiddleware } from '@langchain/typesafe/middleware';
import * as Sentry from '@sentry/node';
import express from 'express';
import { createAgent } from 'langchain';

function startMockServer() {
const app = express();
app.use(express.json());

app.post('/v1/messages', (req, res) => {
res.json({
id: 'msg_router_test',
type: 'message',
role: 'assistant',
content: [{ type: 'text', text: 'A refund is on its way.' }],
model: req.body.model,
stop_reason: 'end_turn',
stop_sequence: null,
usage: { input_tokens: 10, output_tokens: 5 },
});
});

app.post('/v1/systemone', (req, res) => {
res.json({
model: 'jev-1.13',
answers: {
model_route: { type: 'choice', choice: 'fast', probabilities: { fast: 0.9, smart: 0.1 }, confidence: 0.8 },
},
usage: { input_tokens: 40, output_tokens: 3 },
});
});

return new Promise(resolve => {
const server = app.listen(0, () => {
resolve(server);
});
});
}

async function run() {
const server = await startMockServer();
const baseUrl = `http://localhost:${server.address().port}`;

const model = name => new ChatAnthropic({ model: name, apiKey: 'mock-api-key', clientOptions: { baseURL: baseUrl } });

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const agent = createAgent({
name: 'support_agent',
model: model('claude-3-5-sonnet-20241022'),
middleware: [
modelRouterMiddleware({
instructions: 'Pick the model for this request.',
choices: {
fast: { model: model('claude-3-5-haiku-20241022'), criteria: 'Simple requests' },
smart: { model: model('claude-3-5-sonnet-20241022'), criteria: 'Complex requests' },
},
classifierOptions: { apiKey: 'mock-api-key', baseUrl },
}),
],
});

await agent.invoke({ messages: [{ role: 'user', content: 'Where is my refund?' }] });
});

server.close();
}

run();
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { afterAll, describe, expect } from 'vitest';
import {
GEN_AI_AGENT_NAME,
GEN_AI_CONVERSATION_ID,
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
Expand Down Expand Up @@ -317,6 +318,65 @@ describe('LangChain integration (v1)', () => {
},
);

createEsmAndCjsTests(
__dirname,
'scenario-typesafe-model-router.mjs',
'instrument-with-pii.mjs',
(createRunner, test) => {
test('records the Jev model router call inside a createAgent run', async () => {
const runner = createRunner().ignore('event');
const spansPromise = runner.collectStreamedSpansUntilSegment('main');

await runner.start().completed();

const spans = await spansPromise;
const evaluateSpan = spans.find(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)!;
const chatSpan = spans.find(span => span.name === 'chat claude-3-5-haiku-20241022')!;

// The router classifies in `beforeAgent`, inside the agent run, where other chain steps are skipped.
expect(evaluateSpan.name).toBe('evaluate jev-latest');
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('support_agent');
expect(evaluateSpan.parent_span_id).toBe(chatSpan.parent_span_id);
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
{
type: 'evaluation',
// The router passes the latest human message, which the classifier sends as a transcript line.
state: 'user: Where is my refund?',
questions: {
model_route: {
type: 'choice',
instructions: 'Pick the model for this request.',
criteria: { fast: 'Simple requests', smart: 'Complex requests' },
},
},
},
]);
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
{
type: 'evaluation',
answers: {
model_route: {
type: 'choice',
choice: 'fast',
probabilities: { fast: 0.9, smart: 0.1 },
confidence: 0.8,
},
},
},
]);
});
},
{
additionalDependencies: {
langchain: '^1.0.0',
'@langchain/core': '^1.0.0',
'@langchain/anthropic': '^1.0.0',
'@langchain/typesafe': '^0.0.2',
},
},
);

createEsmTests(
__dirname,
'scenario-openai-before-langchain.mjs',
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: true, outputs: true } },
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import * as Sentry from '@sentry/node';
import { loggingTransport } from '@sentry-internal/node-integration-tests';

Sentry.init({
dsn: 'https://public@dsn.ingest.sentry.io/1337',
release: '1.0',
tracesSampleRate: 1.0,
dataCollection: { genAI: { inputs: false, outputs: false } },
transport: loggingTransport,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { Annotation, END, START, StateGraph } from '@langchain/langgraph';
import { TypeSafeClassifier } from '@langchain/typesafe';
import * as Sentry from '@sentry/node';

// Answer in-process through the classifier's `fetch` option, so no mock server is needed.
async function mockTypeSafeFetch() {
return new Response(
JSON.stringify({
model: 'jev-1.13',
answers: { urgent: { type: 'noul', noul: 0.9 } },
usage: { input_tokens: 30, output_tokens: 2 },
}),
{ headers: { 'content-type': 'application/json' } },
);
}

async function run() {
const classifier = new TypeSafeClassifier({
apiKey: 'mock-api-key',
fetch: mockTypeSafeFetch,
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
});

const State = Annotation.Root({ ticket: Annotation(), urgent: Annotation() });

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const graph = new StateGraph(State)
.addNode('triage', async state => ({ urgent: (await classifier.invoke(state.ticket)).nouls.urgent.noul }))
.addEdge(START, 'triage')
.addEdge('triage', END)
.compile({ name: 'triage_graph' });

await graph.invoke({ ticket: 'My payouts have been failing.' });
});
}

run();
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { afterAll, describe, expect } from 'vitest';
import {
GEN_AI_AGENT_NAME,
GEN_AI_INPUT_MESSAGES,
GEN_AI_OUTPUT_MESSAGES,
SENTRY_OP,
SENTRY_ORIGIN,
} from '@sentry/conventions/attributes';
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';

describe('LangGraph integration (v1)', () => {
afterAll(() => {
cleanupChildProcesses();
});

createEsmAndCjsTests(
__dirname,
'scenario-typesafe-classifier-node.mjs',
'instrument-with-pii.mjs',
(createRunner, test) => {
test('records a TypeSafeClassifier call made inside a StateGraph node', async () => {
const runner = createRunner();
const spansPromise = runner.collectStreamedSpansUntilSegment('main');

await runner.start().completed();

const spans = await spansPromise;
const rootSpan = spans.find(span => span.is_segment && span.name === 'main')!;
const agentSpan = spans.find(span => span.name === 'invoke_agent triage_graph')!;
const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!;

expect(agentSpan.parent_span_id).toBe(rootSpan.span_id);
expect(evaluateSpan.parent_span_id).toBe(agentSpan.span_id);
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
expect(evaluateSpan.attributes[GEN_AI_AGENT_NAME].value).toBe('triage_graph');
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
{
type: 'evaluation',
state: 'My payouts have been failing.',
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
},
]);
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
]);
});
},
{
additionalDependencies: {
langchain: '^1.0.0',
'@langchain/core': '^1.0.0',
'@langchain/langgraph': '^1.0.0',
'@langchain/typesafe': '^0.0.2',
},
},
);

createEsmAndCjsTests(
__dirname,
'scenario-typesafe-classifier-node.mjs',
'instrument.mjs',
(createRunner, test) => {
test('omits the classifier input and output when genAI recording is off', async () => {
const runner = createRunner();
const spansPromise = runner.collectStreamedSpansUntilSegment('main');

await runner.start().completed();

const spans = await spansPromise;
const evaluateSpan = spans.find(span => span.name === 'evaluate jev-latest')!;

expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
});
},
{
additionalDependencies: {
langchain: '^1.0.0',
'@langchain/core': '^1.0.0',
'@langchain/langgraph': '^1.0.0',
'@langchain/typesafe': '^0.0.2',
},
},
);
});
Loading