From 11e88d2939ca730593c82081e1262c9e4c2ae703 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 11:12:43 +0200 Subject: [PATCH 1/7] feat(server-utils): Record LangChain TypeSafeClassifier runs as gen_ai.evaluate spans Co-Authored-By: Claude Opus 5.5 --- .../langchain/v1/instrument-with-pii.mjs | 6 +- .../v1/scenario-typesafe-classifier.mjs | 42 +++++++++++++ .../suites/tracing/langchain/v1/test.ts | 52 ++++++++++++++++ .../server-utils/src/ai/langchain/index.ts | 23 ++++++- .../server-utils/src/ai/langchain/types.ts | 2 +- .../src/ai/langchain/typesafe-classifier.ts | 62 +++++++++++++++++++ .../src/integrations/langchain.ts | 12 +++- .../config/channel-integration-definitions.ts | 1 + .../src/orchestrion/config/langchain.ts | 15 ++++- .../tracing/langchain-invoke-agent.test.ts | 21 ++++++- 10 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs create mode 100644 packages/server-utils/src/ai/langchain/typesafe-classifier.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs index a220d64b148a..92bf7cb8d001 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs @@ -10,7 +10,11 @@ Sentry.init({ transport: loggingTransport, beforeSendTransaction: event => { // Filter out mock express server transactions - if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { + if ( + event.transaction.includes('/v1/messages') || + event.transaction.includes('/v1/chat/completions') || + event.transaction.includes('/v1/systemone') + ) { return null; } return event; diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs new file mode 100644 index 000000000000..5d5e35bb30a5 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs @@ -0,0 +1,42 @@ +import { TypeSafeClassifier } from '@langchain/typesafe'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockTypeSafeServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: 'jev-1.13', + answers: { urgent: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 30, output_tokens: 2 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockTypeSafeServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const classifier = new TypeSafeClassifier({ + apiKey: 'mock-api-key', + baseUrl, + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }); + + await classifier.invoke('My payouts have been failing.'); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 669ac68afa34..49fafc100461 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -3,6 +3,7 @@ import { GEN_AI_CONVERSATION_ID, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, @@ -15,7 +16,10 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests'; @@ -241,6 +245,54 @@ describe('LangChain integration (v1)', () => { }, ); + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); + + const evaluateSpan = container.items[0]!; + expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); + expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest'); + expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: 'My payouts have been failing.', + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ + { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, + ]); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/typesafe': '0.0.2', + }, + }, + ); + createEsmTests( __dirname, 'scenario-openai-before-langchain.mjs', diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index 9088e24cc523..6d82b879fb8a 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -15,6 +15,11 @@ import { import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op'; import { resolveAIRecordingOptions } from '../core/utils'; import { LANGCHAIN_ORIGIN } from './constants'; +import { + addTypeSafeClassifierResponseAttributes, + isTypeSafeClassifier, + startTypeSafeClassifierSpan, +} from './typesafe-classifier'; import type { LangChainCallbackHandler, LangChainLLMResult, @@ -43,6 +48,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Internal state - single instance tracks all spans const spanMap = new Map(); + const evaluateRunIds = new Set(); /** * Exit a span and clean up @@ -53,6 +59,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): span.end(); spanMap.delete(runId); } + evaluateRunIds.delete(runId); }; /** @@ -206,7 +213,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Chain Start Handler handleChainStart( - chain: { name?: string }, + chain: LangChainSerialized, inputs: Record, runId: string, _parentRunId?: string, @@ -215,6 +222,15 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): _runType?: string, runName?: string, ) { + // A Jev call is a real model call, so it is recorded inside an agent too. + if (isTypeSafeClassifier(chain)) { + const span = startTypeSafeClassifierSpan(chain, inputs, recordInputs); + span.setAttributes({ ...getAgentNameFromMetadata(metadata), ...getConversationIdFromMetadata(metadata) }); + spanMap.set(runId, span); + evaluateRunIds.add(runId); + return; + } + // Skip chain spans when inside an agent context (createReactAgent). // The agent already creates an invoke_agent span; internal chain steps // (ChannelWrite, Branch, prompt, etc.) are noise. @@ -264,8 +280,9 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChainEnd(outputs: unknown, runId: string) { const span = spanMap.get(runId); if (span?.isRecording()) { - // Add outputs if recordOutputs is enabled - if (recordOutputs) { + if (evaluateRunIds.has(runId)) { + addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs); + } else if (recordOutputs) { span.setAttributes({ 'langchain.chain.outputs': JSON.stringify(outputs), }); diff --git a/packages/server-utils/src/ai/langchain/types.ts b/packages/server-utils/src/ai/langchain/types.ts index 352aecdea295..2556ec83aa98 100644 --- a/packages/server-utils/src/ai/langchain/types.ts +++ b/packages/server-utils/src/ai/langchain/types.ts @@ -165,7 +165,7 @@ export interface LangChainCallbackHandler { extraParams?: Record, ) => Promise | unknown; handleChainStart?: ( - chain: { name?: string }, + chain: LangChainSerialized, inputs: Record, runId: string, parentRunId?: string, diff --git a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts new file mode 100644 index 000000000000..57b46e37e9b3 --- /dev/null +++ b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts @@ -0,0 +1,62 @@ +import { SENTRY_ORIGIN } from '@sentry/conventions/attributes'; +import type { Span } from '@sentry/core'; +import { isObjectLike } from '@sentry/core'; +import { addResponseAttributes, startEvaluateSpan } from '../typesafe'; +import { LANGCHAIN_ORIGIN } from './constants'; +import type { LangChainSerialized } from './types'; + +// `TypeSafeClassifier` from `@langchain/typesafe` calls Jev with `fetch`, not through `@typesafe-ai/sdk`, +// so the TypeSafe integration does not see it. Its serialized id is `[...lc_namespace, lc_name()]`. +const TYPESAFE_CLASSIFIER_ID = 'langchain/classifiers/typesafe/TypeSafeClassifier'; + +/** The package's default, used when the classifier is constructed without a `model`. */ +const DEFAULT_TYPESAFE_CLASSIFIER_MODEL = 'jev-latest'; + +export function isTypeSafeClassifier(chain: LangChainSerialized): boolean { + return chain.id?.join('/') === TYPESAFE_CLASSIFIER_ID; +} + +/** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */ +export function startTypeSafeClassifierSpan( + chain: LangChainSerialized, + inputs: Record, + recordInputs: boolean, +): Span { + const kwargs = chain.kwargs ?? {}; + const model = typeof kwargs.model === 'string' ? kwargs.model : DEFAULT_TYPESAFE_CLASSIFIER_MODEL; + + const span = startEvaluateSpan( + { model, state: getState(inputs), questions: kwargs.questions }, + undefined, + recordInputs, + ); + span.setAttribute(SENTRY_ORIGIN, LANGCHAIN_ORIGIN); + return span; +} + +/** The classifier result reports usage in camelCase, the TypeSafe helpers read the API's snake_case. */ +export function addTypeSafeClassifierResponseAttributes(span: Span, outputs: unknown, recordOutputs: boolean): void { + if (!isObjectLike(outputs)) { + return; + } + + const usage = isObjectLike(outputs.usage) ? outputs.usage : {}; + addResponseAttributes( + span, + { + model: outputs.model, + answers: outputs.answers, + usage: { input_tokens: usage.inputTokens, output_tokens: usage.outputTokens }, + }, + recordOutputs, + ); +} + +/** LangChain hands a string or array input to callbacks wrapped as `{ input }`. */ +function getState(inputs: Record): unknown { + const keys = Object.keys(inputs); + const input = inputs.input; + return keys.length === 1 && keys[0] === 'input' && (typeof input === 'string' || Array.isArray(input)) + ? input + : inputs; +} diff --git a/packages/server-utils/src/integrations/langchain.ts b/packages/server-utils/src/integrations/langchain.ts index e623bbbed9f0..dac3145a8b66 100644 --- a/packages/server-utils/src/integrations/langchain.ts +++ b/packages/server-utils/src/integrations/langchain.ts @@ -77,8 +77,6 @@ function instrumentChatModels(options: LangChainOptions): void { // callback dispatch then creates the spans, exactly as in the OTel path, so no span is opened // here — a `start` subscriber (which also makes orchestrion wrap the function) is enough. const injectHandler = (message: unknown): void => { - markProvidersSkipped(); - const args = (message as RunnableChannelContext).arguments; if (!Array.isArray(args)) { return; @@ -99,8 +97,16 @@ function instrumentChatModels(options: LangChainOptions): void { }; for (const channelName of [CHANNELS.LANGCHAIN_CHAT_MODEL_INVOKE, CHANNELS.LANGCHAIN_CHAT_MODEL_STREAM]) { - diagnosticsChannel.tracingChannel(channelName).start.subscribe(injectHandler); + diagnosticsChannel.tracingChannel(channelName).start.subscribe(message => { + markProvidersSkipped(); + injectHandler(message); + }); } + + // `TypeSafeClassifier` calls Jev with `fetch`, not through a provider SDK, so nothing needs skipping. + diagnosticsChannel + .tracingChannel(CHANNELS.LANGCHAIN_TYPESAFE_CLASSIFIER_INVOKE) + .start.subscribe(injectHandler); } // Embeddings don't use the callback system. Wrap the method in its own span. diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 51d25ab28328..4a133a2d89a7 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -43,6 +43,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ '@langchain/google-common', '@langchain/google-genai', '@langchain/mistralai', + '@langchain/typesafe', ], }, { exportName: 'langGraphIntegration', modules: ['@langchain/langgraph'] }, diff --git a/packages/server-utils/src/orchestrion/config/langchain.ts b/packages/server-utils/src/orchestrion/config/langchain.ts index 4697613a9448..f6d2684b86a3 100644 --- a/packages/server-utils/src/orchestrion/config/langchain.ts +++ b/packages/server-utils/src/orchestrion/config/langchain.ts @@ -61,7 +61,19 @@ const embeddingsConfig = EMBEDDINGS_PROVIDERS.flatMap(({ name, versionRange, met ), ); -export const langchainConfig = [...chatModelConfig, ...embeddingsConfig] satisfies InstrumentationConfig[]; +// `TypeSafeClassifier` (`@langchain/typesafe`) is a plain `Runnable`, not a chat model, so the chat-model +// hooks never add the Sentry callback handler to its runs. +const typesafeClassifierConfig = ['dist/classifier.cjs', 'dist/classifier.js'].map(filePath => ({ + channelName: 'typesafeClassifierInvoke', + module: { name: '@langchain/typesafe', versionRange: '>=0.0.1 <1.0.0', filePath }, + functionQuery: { className: 'TypeSafeClassifier', methodName: 'invoke', kind: 'Async' as const }, +})); + +export const langchainConfig = [ + ...chatModelConfig, + ...embeddingsConfig, + ...typesafeClassifierConfig, +] satisfies InstrumentationConfig[]; // The embeddings channel strings the subscriber binds to, derived from the provider list above so that // adding a provider is a single edit that both instruments it and subscribes the listener to it. @@ -74,4 +86,5 @@ export const langchainModuleNames = getModuleNames(langchainConfig); export const langchainChannels = { LANGCHAIN_CHAT_MODEL_INVOKE: 'orchestrion:@langchain/core:chatModelInvoke', LANGCHAIN_CHAT_MODEL_STREAM: 'orchestrion:@langchain/core:chatModelStream', + LANGCHAIN_TYPESAFE_CLASSIFIER_INVOKE: 'orchestrion:@langchain/typesafe:typesafeClassifierInvoke', } as const; diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts index 8e316a69a763..7ebf937e463c 100644 --- a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { GEN_AI_OPERATION_NAME, GEN_AI_PIPELINE_NAME } from '@sentry/conventions/attributes'; +import { GEN_AI_OPERATION_NAME, GEN_AI_PIPELINE_NAME, SENTRY_OP } from '@sentry/conventions/attributes'; +import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; @@ -95,4 +96,22 @@ describe('LangChain invoke_agent span names', () => { expect(span.data?.[GEN_AI_PIPELINE_NAME]).toBeUndefined(); expect(span.data?.['langchain.chain.name']).toBeUndefined(); }); + + it('records a TypeSafeClassifier run inside an agent, where other chain steps are skipped', () => { + const endedSpans = setupClient('stream'); + const handler = createLangChainCallbackHandler(); + const classifier = { id: ['langchain', 'classifiers', 'typesafe', 'TypeSafeClassifier'], kwargs: {} }; + + handler.handleChainStart?.(classifier, { input: 'My payouts have been failing.' }, 'run-1', undefined, undefined, { + __sentry_langgraph__: true, + }); + handler.handleChainEnd?.({ model: 'jev-1.13', answers: {}, usage: {} }, 'run-1'); + + expect(endedSpans.map(span => spanToStaticSpanJSON(span))).toEqual([ + expect.objectContaining({ + description: 'evaluate jev-latest', + data: expect.objectContaining({ [SENTRY_OP]: GEN_AI_EVALUATE }), + }), + ]); + }); }); From 973ab3e656f77614dfc8f32954eaa7b9322825eb Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 11:49:10 +0200 Subject: [PATCH 2/7] Set classifier span attributes at start and restore handler type --- .../server-utils/src/ai/langchain/index.ts | 6 ++--- .../server-utils/src/ai/langchain/types.ts | 2 +- .../src/ai/langchain/typesafe-classifier.ts | 24 ++++++++--------- .../server-utils/src/ai/typesafe/index.ts | 10 +++++-- .../tracing/langchain-invoke-agent.test.ts | 27 ++++++++++++++----- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index 6d82b879fb8a..265e6c27136d 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -213,7 +213,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Chain Start Handler handleChainStart( - chain: LangChainSerialized, + chain: { name?: string }, inputs: Record, runId: string, _parentRunId?: string, @@ -224,9 +224,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): ) { // A Jev call is a real model call, so it is recorded inside an agent too. if (isTypeSafeClassifier(chain)) { - const span = startTypeSafeClassifierSpan(chain, inputs, recordInputs); - span.setAttributes({ ...getAgentNameFromMetadata(metadata), ...getConversationIdFromMetadata(metadata) }); - spanMap.set(runId, span); + spanMap.set(runId, startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs)); evaluateRunIds.add(runId); return; } diff --git a/packages/server-utils/src/ai/langchain/types.ts b/packages/server-utils/src/ai/langchain/types.ts index 2556ec83aa98..352aecdea295 100644 --- a/packages/server-utils/src/ai/langchain/types.ts +++ b/packages/server-utils/src/ai/langchain/types.ts @@ -165,7 +165,7 @@ export interface LangChainCallbackHandler { extraParams?: Record, ) => Promise | unknown; handleChainStart?: ( - chain: LangChainSerialized, + chain: { name?: string }, inputs: Record, runId: string, parentRunId?: string, diff --git a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts index 57b46e37e9b3..5d477a10cb8a 100644 --- a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts +++ b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts @@ -3,7 +3,7 @@ import type { Span } from '@sentry/core'; import { isObjectLike } from '@sentry/core'; import { addResponseAttributes, startEvaluateSpan } from '../typesafe'; import { LANGCHAIN_ORIGIN } from './constants'; -import type { LangChainSerialized } from './types'; +import { getAgentNameFromMetadata, getConversationIdFromMetadata } from './utils'; // `TypeSafeClassifier` from `@langchain/typesafe` calls Jev with `fetch`, not through `@typesafe-ai/sdk`, // so the TypeSafe integration does not see it. Its serialized id is `[...lc_namespace, lc_name()]`. @@ -12,26 +12,26 @@ const TYPESAFE_CLASSIFIER_ID = 'langchain/classifiers/typesafe/TypeSafeClassifie /** The package's default, used when the classifier is constructed without a `model`. */ const DEFAULT_TYPESAFE_CLASSIFIER_MODEL = 'jev-latest'; -export function isTypeSafeClassifier(chain: LangChainSerialized): boolean { - return chain.id?.join('/') === TYPESAFE_CLASSIFIER_ID; +// Typed loosely: LangChain's `Serialized` union does not match our handler's chain type. +export function isTypeSafeClassifier(chain: unknown): boolean { + return isObjectLike(chain) && Array.isArray(chain.id) && chain.id.join('/') === TYPESAFE_CLASSIFIER_ID; } /** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */ export function startTypeSafeClassifierSpan( - chain: LangChainSerialized, + chain: unknown, inputs: Record, + metadata: Record | undefined, recordInputs: boolean, ): Span { - const kwargs = chain.kwargs ?? {}; + const kwargs = isObjectLike(chain) && isObjectLike(chain.kwargs) ? chain.kwargs : {}; const model = typeof kwargs.model === 'string' ? kwargs.model : DEFAULT_TYPESAFE_CLASSIFIER_MODEL; - const span = startEvaluateSpan( - { model, state: getState(inputs), questions: kwargs.questions }, - undefined, - recordInputs, - ); - span.setAttribute(SENTRY_ORIGIN, LANGCHAIN_ORIGIN); - return span; + return startEvaluateSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, { + [SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + ...getAgentNameFromMetadata(metadata), + ...getConversationIdFromMetadata(metadata), + }); } /** The classifier result reports usage in camelCase, the TypeSafe helpers read the API's snake_case. */ diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index af4115b0137d..9552e5e96a4d 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -21,7 +21,12 @@ import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; * Start the span for a `systemOne(request)` call. The request model falls back to the client's * `defaultModel`, the same way the SDK resolves it. */ -export function startEvaluateSpan(request: unknown, client: unknown, recordInputs: boolean): Span { +export function startEvaluateSpan( + request: unknown, + client: unknown, + recordInputs: boolean, + attributes?: SpanAttributes, +): Span { const params = isObjectLike(request) ? request : {}; const defaultModel = isObjectLike(client) ? client.defaultModel : undefined; const model = @@ -30,7 +35,8 @@ export function startEvaluateSpan(request: unknown, client: unknown, recordInput return startInactiveSpan({ name: model ? `evaluate ${model}` : 'evaluate', op: GEN_AI_EVALUATE, - attributes: getRequestAttributes(params, model, recordInputs), + // Extra attributes (e.g. another integration's origin) are set at start, so samplers see them. + attributes: { ...getRequestAttributes(params, model, recordInputs), ...attributes }, }); } diff --git a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts index 7ebf937e463c..3a36432167ea 100644 --- a/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts +++ b/packages/server-utils/test/ai/lib/tracing/langchain-invoke-agent.test.ts @@ -1,7 +1,13 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { GEN_AI_OPERATION_NAME, GEN_AI_PIPELINE_NAME, SENTRY_OP } from '@sentry/conventions/attributes'; +import { + GEN_AI_AGENT_NAME, + GEN_AI_OPERATION_NAME, + GEN_AI_PIPELINE_NAME, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; import { GEN_AI_EVALUATE } from '@sentry/conventions/op'; -import { getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; +import { getClient, getMainCarrier, setCurrentClient, spanToStaticSpanJSON } from '@sentry/core'; import type { Span } from '@sentry/core'; import { createLangChainCallbackHandler } from '../../../../src/ai/langchain'; import { getDefaultTestClientOptions, TestClient } from '../../../mocks/client'; @@ -97,21 +103,28 @@ describe('LangChain invoke_agent span names', () => { expect(span.data?.['langchain.chain.name']).toBeUndefined(); }); - it('records a TypeSafeClassifier run inside an agent, where other chain steps are skipped', () => { + it('records a TypeSafeClassifier run inside an agent, with its attributes set at span start', () => { const endedSpans = setupClient('stream'); + const startAttributes: Record[] = []; + getClient()!.on('spanStart', span => startAttributes.push(spanToStaticSpanJSON(span).data)); + const handler = createLangChainCallbackHandler(); - const classifier = { id: ['langchain', 'classifiers', 'typesafe', 'TypeSafeClassifier'], kwargs: {} }; + // LangChain passes the classifier serialized as `[...lc_namespace, lc_name()]`. + const classifier = { name: undefined, id: ['langchain', 'classifiers', 'typesafe', 'TypeSafeClassifier'] }; handler.handleChainStart?.(classifier, { input: 'My payouts have been failing.' }, 'run-1', undefined, undefined, { __sentry_langgraph__: true, + lc_agent_name: 'support_agent', }); handler.handleChainEnd?.({ model: 'jev-1.13', answers: {}, usage: {} }, 'run-1'); - expect(endedSpans.map(span => spanToStaticSpanJSON(span))).toEqual([ + expect(startAttributes).toEqual([ expect.objectContaining({ - description: 'evaluate jev-latest', - data: expect.objectContaining({ [SENTRY_OP]: GEN_AI_EVALUATE }), + [SENTRY_OP]: GEN_AI_EVALUATE, + [SENTRY_ORIGIN]: 'auto.ai.langchain', + [GEN_AI_AGENT_NAME]: 'support_agent', }), ]); + expect(endedSpans.map(span => spanToStaticSpanJSON(span).description)).toEqual(['evaluate jev-latest']); }); }); From a63f8efc0f7a65a5996ad99c93ab667a66c4bb25 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 15:23:55 +0200 Subject: [PATCH 3/7] Record the classifier's wire state as the span input --- .../v1/scenario-typesafe-classifier.mjs | 2 ++ .../suites/tracing/langchain/v1/test.ts | 14 +++++++-- .../src/ai/langchain/typesafe-classifier.ts | 31 ++++++++++++++++--- .../src/integrations/langchain.ts | 8 ++++- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs index 5d5e35bb30a5..50f0d2e68c14 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs @@ -1,3 +1,4 @@ +import { HumanMessage } from '@langchain/core/messages'; import { TypeSafeClassifier } from '@langchain/typesafe'; import * as Sentry from '@sentry/node'; import express from 'express'; @@ -33,6 +34,7 @@ async function run() { }); await classifier.invoke('My payouts have been failing.'); + await classifier.invoke(new HumanMessage('My card was charged twice.')); }); await Sentry.flush(2000); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 49fafc100461..55da9078247c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -256,9 +256,11 @@ describe('LangChain integration (v1)', () => { .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest']); + expect(container.items.map(span => span.name)).toEqual(['evaluate jev-latest', 'evaluate jev-latest']); - const evaluateSpan = container.items[0]!; + const [evaluateSpan, messageEvaluateSpan] = container.items.sort( + (a, b) => a.start_timestamp - b.start_timestamp, + ); expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE); expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain'); expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate'); @@ -275,6 +277,14 @@ describe('LangChain integration (v1)', () => { questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, }, ]); + // A message is recorded as the transcript line the classifier sends, not as LangChain's serialized form. + expect(JSON.parse(messageEvaluateSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([ + { + type: 'evaluation', + state: 'user: My card was charged twice.', + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }, + ]); expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([ { type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } }, ]); diff --git a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts index 5d477a10cb8a..685a8e826a96 100644 --- a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts +++ b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts @@ -12,6 +12,28 @@ const TYPESAFE_CLASSIFIER_ID = 'langchain/classifiers/typesafe/TypeSafeClassifie /** The package's default, used when the classifier is constructed without a `model`. */ const DEFAULT_TYPESAFE_CLASSIFIER_MODEL = 'jev-latest'; +// The `state` the classifier sends to Jev (messages rendered as transcript lines), keyed by the input +// passed to `invoke()`. LangChain hands that same object to `handleChainStart`, so the span records what +// Jev received rather than LangChain's serialized messages. +const wireStates = new WeakMap(); + +/** Record the `state` a `TypeSafeClassifier` sends for `input`, using the classifier's own serializer. */ +export function recordTypeSafeClassifierState(classifier: unknown, input: unknown): void { + // A string state is sent as is, so there is nothing to record. + if (!isObjectLike(input) || !isObjectLike(classifier) || typeof classifier.payload !== 'function') { + return; + } + + try { + const body: unknown = JSON.parse(classifier.payload(input)); + if (isObjectLike(body)) { + wireStates.set(input, body.state); + } + } catch { + // The classifier rejects the same input itself; the span keeps LangChain's form of it. + } +} + // Typed loosely: LangChain's `Serialized` union does not match our handler's chain type. export function isTypeSafeClassifier(chain: unknown): boolean { return isObjectLike(chain) && Array.isArray(chain.id) && chain.id.join('/') === TYPESAFE_CLASSIFIER_ID; @@ -52,11 +74,12 @@ export function addTypeSafeClassifierResponseAttributes(span: Span, outputs: unk ); } -/** LangChain hands a string or array input to callbacks wrapped as `{ input }`. */ function getState(inputs: Record): unknown { + // LangChain hands a string or array input to callbacks wrapped as `{ input }`. const keys = Object.keys(inputs); const input = inputs.input; - return keys.length === 1 && keys[0] === 'input' && (typeof input === 'string' || Array.isArray(input)) - ? input - : inputs; + const state = + keys.length === 1 && keys[0] === 'input' && (typeof input === 'string' || Array.isArray(input)) ? input : inputs; + + return isObjectLike(state) && wireStates.has(state) ? wireStates.get(state) : state; } diff --git a/packages/server-utils/src/integrations/langchain.ts b/packages/server-utils/src/integrations/langchain.ts index dac3145a8b66..1ab64477d87c 100644 --- a/packages/server-utils/src/integrations/langchain.ts +++ b/packages/server-utils/src/integrations/langchain.ts @@ -6,6 +6,7 @@ import { GOOGLE_GENAI_INTEGRATION_NAME } from '../ai/google-genai/constants'; import { createLangChainCallbackHandler } from '../ai/langchain'; import { LANGCHAIN_INTEGRATION_NAME } from '../ai/langchain/constants'; import { _INTERNAL_getLangChainEmbeddingsSpanOptions } from '../ai/langchain/embeddings'; +import { recordTypeSafeClassifierState } from '../ai/langchain/typesafe-classifier'; import type { LangChainOptions } from '../ai/langchain/types'; import { getConversationIdMetadataFromConfig, _INTERNAL_mergeLangChainCallbackHandler } from '../ai/langchain/utils'; import { MISTRAL_INTEGRATION_NAME } from '../ai/mistral/constants'; @@ -35,6 +36,7 @@ const SKIPPED_PROVIDERS = [ // The chat-model channels carry the live args array of `invoke(input, options)` / `_streamIterator(input, options)`. interface RunnableChannelContext { arguments: unknown[]; + self?: unknown; } // The embeddings channels carry the instance (`self`) and the `embedQuery(text)` / `embedDocuments(texts)` args. @@ -106,7 +108,11 @@ function instrumentChatModels(options: LangChainOptions): void { // `TypeSafeClassifier` calls Jev with `fetch`, not through a provider SDK, so nothing needs skipping. diagnosticsChannel .tracingChannel(CHANNELS.LANGCHAIN_TYPESAFE_CLASSIFIER_INVOKE) - .start.subscribe(injectHandler); + .start.subscribe(message => { + injectHandler(message); + const { self, arguments: args } = message as RunnableChannelContext; + recordTypeSafeClassifierState(self, args?.[0]); + }); } // Embeddings don't use the callback system. Wrap the method in its own span. From de8bbcafa2e74886e9611db7380d52d84078c0b8 Mon Sep 17 00:00:00 2001 From: Andrei <168741329+andreiborza@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:57:30 +0200 Subject: [PATCH 4/7] Update dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts Co-authored-by: Charly Gomez --- .../node-integration-tests/suites/tracing/langchain/v1/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 55da9078247c..a83790f6c517 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -298,7 +298,7 @@ describe('LangChain integration (v1)', () => { additionalDependencies: { langchain: '^1.0.0', '@langchain/core': '^1.0.0', - '@langchain/typesafe': '0.0.2', + '@langchain/typesafe': '^0.0.2', }, }, ); From 9bcd985ce5937460b5e818c67c099417d184278b Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Wed, 7 Oct 2026 17:10:30 +0200 Subject: [PATCH 5/7] Remove the flush from the TypeSafe classifier scenario --- .../suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs index 50f0d2e68c14..eaaf7a9c7e52 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier.mjs @@ -37,7 +37,6 @@ async function run() { await classifier.invoke(new HumanMessage('My card was charged twice.')); }); - await Sentry.flush(2000); server.close(); } From 3c88c34cea55b699dd5165160c361144e03f94a1 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 8 Oct 2026 10:51:44 +0200 Subject: [PATCH 6/7] Keep the parent run callbacks when a LangChain call has none --- ...ypesafe-classifier-inherited-callbacks.mjs | 61 +++++++++++++++++++ .../suites/tracing/langchain/v1/test.ts | 27 ++++++++ .../server-utils/src/ai/langchain/utils.ts | 16 +++++ .../src/integrations/langchain.ts | 12 +++- 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs new file mode 100644 index 000000000000..642945063c88 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-typesafe-classifier-inherited-callbacks.mjs @@ -0,0 +1,61 @@ +import { BaseCallbackHandler } from '@langchain/core/callbacks/base'; +import { RunnableLambda } from '@langchain/core/runnables'; +import { TypeSafeClassifier } from '@langchain/typesafe'; +// Sets up the AsyncLocalStorage that passes a parent run's config to its children, as it is inside `createAgent`. +import '@langchain/langgraph'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockTypeSafeServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + res.json({ + model: 'jev-1.13', + answers: { urgent: { type: 'noul', noul: 0.9 } }, + usage: { input_tokens: 30, output_tokens: 2 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +// Stands in for a tracer such as LangSmith, which the user passes to the parent run. +class RecordingHandler extends BaseCallbackHandler { + name = 'RecordingHandler'; + runs = []; + + handleChainStart(chain, _inputs, _runId, parentRunId) { + this.runs.push(`${chain.id.at(-1)}:${parentRunId ? 'child' : 'root'}`); + } +} + +async function run() { + const server = await startMockTypeSafeServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async span => { + const classifier = new TypeSafeClassifier({ + apiKey: 'mock-api-key', + baseUrl, + questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } }, + }); + + // Like the `@langchain/typesafe` middlewares, call the classifier without a config. + const parent = RunnableLambda.from(input => classifier.invoke(input)).withConfig({ runName: 'parent' }); + const recorder = new RecordingHandler(); + await parent.invoke('My payouts have been failing.', { callbacks: [recorder] }); + + span.setAttribute('test.recorded_runs', recorder.runs.join(',')); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts index 2727b5ff3525..ae4f7a1554ef 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -290,6 +290,33 @@ describe('LangChain integration (v1)', () => { }, ); + createEsmAndCjsTests( + __dirname, + 'scenario-typesafe-classifier-inherited-callbacks.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('keeps the parent run callbacks for a TypeSafeClassifier called without config', async () => { + const runner = createRunner().ignore('event'); + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); + + await runner.start().completed(); + + const spans = await spansPromise; + const segment = spans.find(span => span.is_segment && span.name === 'main'); + expect(segment!.attributes['test.recorded_runs'].value).toBe('RunnableLambda:root,TypeSafeClassifier:child'); + expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)).toHaveLength(1); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/typesafe': '^0.0.2', + '@langchain/langgraph': '^1.0.0', + }, + }, + ); + createEsmTests( __dirname, 'scenario-openai-before-langchain.mjs', diff --git a/packages/server-utils/src/ai/langchain/utils.ts b/packages/server-utils/src/ai/langchain/utils.ts index 067a82fc2d75..8ae219e42437 100644 --- a/packages/server-utils/src/ai/langchain/utils.ts +++ b/packages/server-utils/src/ai/langchain/utils.ts @@ -564,6 +564,22 @@ export function extractToolDefinitions(extraParams?: Record): s return JSON.stringify(toolDefs); } +// A run started without config inherits the parent run's config, which LangChain keeps in a global +// AsyncLocalStorage. `ensureConfig` lets an explicit `callbacks` key replace the inherited one. +const LANGCHAIN_ALS_KEY = Symbol.for('ls:tracing_async_local_storage'); +const LANGCHAIN_CHILD_CONFIG_KEY = Symbol.for('lc:child_config'); + +/** The callbacks LangChain gives a run started without config: the ones of the parent run, if any. */ +export function getInheritedLangChainCallbacks(): unknown { + const storage = (globalThis as Record)[LANGCHAIN_ALS_KEY] as + | { getStore?: () => unknown } + | undefined; + const store = typeof storage?.getStore === 'function' ? storage.getStore() : undefined; + const extra = isObjectLike(store) ? (store.extra as Record | undefined) : undefined; + const config = isObjectLike(extra) ? extra[LANGCHAIN_CHILD_CONFIG_KEY] : undefined; + return isObjectLike(config) ? config.callbacks : undefined; +} + /** Duck-types a LangChain `CallbackManager` (avoids coupling to a specific `@langchain/core` resolution). */ function isCallbackManager(value: unknown): value is { addHandler: (handler: unknown, inherit?: boolean) => void; diff --git a/packages/server-utils/src/integrations/langchain.ts b/packages/server-utils/src/integrations/langchain.ts index 1ab64477d87c..c10b9c79b6ae 100644 --- a/packages/server-utils/src/integrations/langchain.ts +++ b/packages/server-utils/src/integrations/langchain.ts @@ -8,7 +8,11 @@ import { LANGCHAIN_INTEGRATION_NAME } from '../ai/langchain/constants'; import { _INTERNAL_getLangChainEmbeddingsSpanOptions } from '../ai/langchain/embeddings'; import { recordTypeSafeClassifierState } from '../ai/langchain/typesafe-classifier'; import type { LangChainOptions } from '../ai/langchain/types'; -import { getConversationIdMetadataFromConfig, _INTERNAL_mergeLangChainCallbackHandler } from '../ai/langchain/utils'; +import { + getConversationIdMetadataFromConfig, + getInheritedLangChainCallbacks, + _INTERNAL_mergeLangChainCallbackHandler, +} from '../ai/langchain/utils'; import { MISTRAL_INTEGRATION_NAME } from '../ai/mistral/constants'; import { OPENAI_INTEGRATION_NAME } from '../ai/openai/constants'; import { GROQ_INTEGRATION_NAME } from './groq'; @@ -90,7 +94,11 @@ function instrumentChatModels(options: LangChainOptions): void { args[1] = callOptions; } - callOptions.callbacks = _INTERNAL_mergeLangChainCallbackHandler(callOptions.callbacks, sentryHandler); + // Without callbacks of its own, the call would otherwise lose the parent run and its tracers (e.g. LangSmith). + callOptions.callbacks = _INTERNAL_mergeLangChainCallbackHandler( + callOptions.callbacks ?? getInheritedLangChainCallbacks(), + sentryHandler, + ); const conversationIdMetadata = getConversationIdMetadataFromConfig(callOptions); if (Object.keys(conversationIdMetadata).length) { From cce3f615e0338b4cf2375928d5e9e95b6f7a9766 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 8 Oct 2026 10:56:22 +0200 Subject: [PATCH 7/7] Clean up sampled-out chain runs and match the classifier by class name --- packages/server-utils/src/ai/langchain/index.ts | 7 ++++--- .../server-utils/src/ai/langchain/typesafe-classifier.ts | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/server-utils/src/ai/langchain/index.ts b/packages/server-utils/src/ai/langchain/index.ts index 265e6c27136d..f7b5408d1d59 100644 --- a/packages/server-utils/src/ai/langchain/index.ts +++ b/packages/server-utils/src/ai/langchain/index.ts @@ -57,8 +57,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): const span = spanMap.get(runId); if (span?.isRecording()) { span.end(); - spanMap.delete(runId); } + spanMap.delete(runId); evaluateRunIds.delete(runId); }; @@ -285,8 +285,9 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): 'langchain.chain.outputs': JSON.stringify(outputs), }); } - exitSpan(runId); } + // Also for a sampled-out span, which would otherwise stay tracked. + exitSpan(runId); }, // Chain Error Handler @@ -296,8 +297,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): const span = spanMap.get(runId); if (span?.isRecording()) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); - exitSpan(runId); } + exitSpan(runId); }, // Tool Start Handler diff --git a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts index 685a8e826a96..e254685a4561 100644 --- a/packages/server-utils/src/ai/langchain/typesafe-classifier.ts +++ b/packages/server-utils/src/ai/langchain/typesafe-classifier.ts @@ -6,8 +6,9 @@ import { LANGCHAIN_ORIGIN } from './constants'; import { getAgentNameFromMetadata, getConversationIdFromMetadata } from './utils'; // `TypeSafeClassifier` from `@langchain/typesafe` calls Jev with `fetch`, not through `@typesafe-ai/sdk`, -// so the TypeSafe integration does not see it. Its serialized id is `[...lc_namespace, lc_name()]`. -const TYPESAFE_CLASSIFIER_ID = 'langchain/classifiers/typesafe/TypeSafeClassifier'; +// so the TypeSafe integration does not see it. Its serialized id is `[...lc_namespace, lc_name()]`; only +// the class name is matched, as the namespace is the part that changes between versions. +const TYPESAFE_CLASSIFIER_NAME = 'TypeSafeClassifier'; /** The package's default, used when the classifier is constructed without a `model`. */ const DEFAULT_TYPESAFE_CLASSIFIER_MODEL = 'jev-latest'; @@ -36,7 +37,7 @@ export function recordTypeSafeClassifierState(classifier: unknown, input: unknow // Typed loosely: LangChain's `Serialized` union does not match our handler's chain type. export function isTypeSafeClassifier(chain: unknown): boolean { - return isObjectLike(chain) && Array.isArray(chain.id) && chain.id.join('/') === TYPESAFE_CLASSIFIER_ID; + return isObjectLike(chain) && Array.isArray(chain.id) && chain.id[chain.id.length - 1] === TYPESAFE_CLASSIFIER_NAME; } /** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */