From afc7d6722a0e168da73b310b3b84d3ba3d45e99a Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Jan 2026 11:20:52 +0100 Subject: [PATCH 1/4] record tool calls in langchain --- .../suites/tracing/langchain/index.ts | 17 +++- .../suites/tracing/langchain/mocks.ts | 15 +++- .../suites/tracing/langchain/test.ts | 15 +++- .../langchain/scenario-chain-tool-calls.mjs | 87 +++++++++++++++++++ .../suites/tracing/langchain/test.ts | 54 ++++++++++++ packages/core/src/tracing/langchain/index.ts | 13 ++- packages/core/src/tracing/langchain/utils.ts | 22 +++++ 7 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts index 0d59fd91c2b7..77150a1f35be 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts @@ -18,6 +18,12 @@ export default Sentry.withSentry( recordOutputs: false, }); + // Create a second handler with recordOutputs enabled for tool_calls test + const callbackHandlerWithOutputs = Sentry.createLangChainCallbackHandler({ + recordInputs: false, + recordOutputs: true, + }); + // Test 1: Chat model invocation const chatModel = new MockChatModel({ model: 'claude-3-5-sonnet-20241022', @@ -29,7 +35,7 @@ export default Sentry.withSentry( callbacks: [callbackHandler], }); - // Test 2: Chain invocation + // Test 2: Chain invocation (without tool calls) const chain = new MockChain('my_test_chain'); await chain.invoke( { input: 'test input' }, @@ -44,6 +50,15 @@ export default Sentry.withSentry( callbacks: [callbackHandler], }); + // Test 4: Chain invocation with tool calls (recordOutputs enabled) + const chainWithToolCalls = new MockChain('chain_with_tool_calls', { includeToolCalls: true }); + await chainWithToolCalls.invoke( + { input: 'test input for tool calls' }, + { + callbacks: [callbackHandlerWithOutputs], + }, + ); + return new Response(JSON.stringify({ success: true })); }, }, diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/mocks.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/mocks.ts index 946ae8252dbe..c2e0733eaf23 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/mocks.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/mocks.ts @@ -132,9 +132,11 @@ export class MockChatModel { // Mock LangChain Chain export class MockChain { private _name: string; + private _includeToolCalls: boolean; - public constructor(name: string) { + public constructor(name: string, options?: { includeToolCalls?: boolean }) { this._name = name; + this._includeToolCalls = options?.includeToolCalls ?? false; } public async invoke( @@ -151,7 +153,16 @@ export class MockChain { } } - const outputs = { result: 'Chain execution completed!' }; + const outputs = this._includeToolCalls + ? { + result: 'Chain execution completed!', + messages: [ + { + tool_calls: [{ name: 'search_tool', args: { query: 'test query' } }], + }, + ], + } + : { result: 'Chain execution completed!' }; // Call handleChainEnd for (const callback of callbacks) { diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts index 875b4191b84b..d06146899e9a 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts @@ -33,7 +33,7 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal } op: 'gen_ai.chat', origin: 'auto.ai.langchain', }), - // Chain span + // Chain span (without tool calls) expect.objectContaining({ data: expect.objectContaining({ 'sentry.origin': 'auto.ai.langchain', @@ -55,6 +55,19 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal } op: 'gen_ai.execute_tool', origin: 'auto.ai.langchain', }), + // Chain span with tool calls (recordOutputs enabled) + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.ai.langchain', + 'sentry.op': 'gen_ai.invoke_agent', + 'langchain.chain.name': 'chain_with_tool_calls', + 'langchain.chain.outputs': expect.stringContaining('Chain execution completed'), + 'gen_ai.response.tool_calls': expect.stringContaining('search_tool'), + }), + description: 'chain chain_with_tool_calls', + op: 'gen_ai.invoke_agent', + origin: 'auto.ai.langchain', + }), ]), ); }) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs new file mode 100644 index 000000000000..6747b8acdd77 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs @@ -0,0 +1,87 @@ +import { RunnableLambda } from '@langchain/core/runnables'; +import * as Sentry from '@sentry/node'; + +async function run() { + // Create callback handler with recordOutputs enabled + const callbackHandler = Sentry.createLangChainCallbackHandler({ + recordInputs: true, + recordOutputs: true, + }); + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // Test 1: Chain without tool calls + const simpleChain = new RunnableLambda({ + func: input => { + return { result: `Processed: ${input.query}` }; + }, + }).withConfig({ runName: 'simple_chain' }); + + await simpleChain.invoke( + { query: 'Hello world' }, + { + callbacks: [callbackHandler], + }, + ); + + // Test 2: Chain with tool calls in output + const chainWithToolCalls = new RunnableLambda({ + func: input => { + return { + result: `Processed with tools: ${input.query}`, + messages: [ + { + role: 'assistant', + content: 'I will use the search tool', + tool_calls: [ + { + name: 'search', + args: { query: input.query }, + id: 'tool_call_123', + }, + { + name: 'calculator', + args: { expression: '2+2' }, + id: 'tool_call_456', + }, + ], + }, + ], + }; + }, + }).withConfig({ runName: 'chain_with_tool_calls' }); + + await chainWithToolCalls.invoke( + { query: 'Search for something' }, + { + callbacks: [callbackHandler], + }, + ); + + // Test 3: Chain with direct tool_calls on output (alternative format) + const chainWithDirectToolCalls = new RunnableLambda({ + func: input => { + return { + result: `Direct tool calls: ${input.query}`, + tool_calls: [ + { + name: 'weather', + args: { location: 'San Francisco' }, + id: 'tool_call_789', + }, + ], + }; + }, + }).withConfig({ runName: 'chain_with_direct_tool_calls' }); + + await chainWithDirectToolCalls.invoke( + { query: 'Get weather' }, + { + callbacks: [callbackHandler], + }, + ); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index e75e0ec7f5da..240748371ea6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -246,6 +246,60 @@ describe('LangChain integration', () => { }, ); + const EXPECTED_TRANSACTION_CHAIN_TOOL_CALLS = { + transaction: 'main', + spans: expect.arrayContaining([ + // Simple chain without tool calls (RunnableLambda reports name as "unknown_chain") + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.ai.langchain', + 'sentry.op': 'gen_ai.invoke_agent', + 'langchain.chain.inputs': expect.stringContaining('Hello world'), + 'langchain.chain.outputs': expect.stringContaining('Processed'), + }), + op: 'gen_ai.invoke_agent', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Chain with tool calls in messages format + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.ai.langchain', + 'sentry.op': 'gen_ai.invoke_agent', + 'langchain.chain.outputs': expect.stringContaining('Processed with tools'), + // This is the key attribute we're testing + 'gen_ai.response.tool_calls': expect.stringContaining('search'), + }), + op: 'gen_ai.invoke_agent', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Chain with direct tool_calls on output + expect.objectContaining({ + data: expect.objectContaining({ + 'sentry.origin': 'auto.ai.langchain', + 'sentry.op': 'gen_ai.invoke_agent', + 'langchain.chain.outputs': expect.stringContaining('Direct tool calls'), + // This tests the alternative format (tool_calls directly on output) + 'gen_ai.response.tool_calls': expect.stringContaining('weather'), + }), + op: 'gen_ai.invoke_agent', + origin: 'auto.ai.langchain', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario-chain-tool-calls.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates langchain chain spans with tool calls', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_CHAIN_TOOL_CALLS }) + .start() + .completed(); + }); + }); + createEsmAndCjsTests( __dirname, 'scenario-openai-before-langchain.mjs', diff --git a/packages/core/src/tracing/langchain/index.ts b/packages/core/src/tracing/langchain/index.ts index 1930be794be5..1414faa566ce 100644 --- a/packages/core/src/tracing/langchain/index.ts +++ b/packages/core/src/tracing/langchain/index.ts @@ -3,7 +3,11 @@ import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from ' import { SPAN_STATUS_ERROR } from '../../tracing'; import { startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; -import { GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE } from '../ai/gen-ai-attributes'; +import { + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, +} from '../ai/gen-ai-attributes'; import { LANGCHAIN_ORIGIN } from './constants'; import type { LangChainCallbackHandler, @@ -16,6 +20,7 @@ import { extractChatModelRequestAttributes, extractLLMRequestAttributes, extractLlmResponseAttributes, + extractToolCallsFromChainOutput, getInvocationParams, } from './utils'; @@ -215,6 +220,12 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): span.setAttributes({ 'langchain.chain.outputs': JSON.stringify(outputs), }); + + // Extract tool calls from chain outputs + const toolCalls = extractToolCallsFromChainOutput(outputs); + if (toolCalls && toolCalls.length > 0) { + span.setAttribute(GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, JSON.stringify(toolCalls)); + } } exitSpan(runId); } diff --git a/packages/core/src/tracing/langchain/utils.ts b/packages/core/src/tracing/langchain/utils.ts index 0a07ae8df370..2d28c64727e5 100644 --- a/packages/core/src/tracing/langchain/utils.ts +++ b/packages/core/src/tracing/langchain/utils.ts @@ -318,6 +318,28 @@ function addToolCallsAttributes(generations: LangChainMessage[][], attrs: Record } } +/** + * Extracts tool calls from chain outputs. + * Handles: { messages: [{ tool_calls }] }, { output: { messages } }, { tool_calls } + */ +export function extractToolCallsFromChainOutput(outputs: unknown): unknown[] | null { + if (!outputs || typeof outputs !== 'object') return null; + + const toolCalls: unknown[] = []; + const out = outputs as Record; + const messages = out.messages ?? (out.output as Record | undefined)?.messages; + + if (Array.isArray(messages)) { + for (const msg of messages) { + const calls = (msg as Record | null)?.tool_calls; + if (Array.isArray(calls)) toolCalls.push(...calls); + } + } + if (Array.isArray(out.tool_calls)) toolCalls.push(...out.tool_calls); + + return toolCalls.length > 0 ? toolCalls : null; +} + /** * Adds token usage attributes, supporting both OpenAI (`tokenUsage`) and Anthropic (`usage`) formats. * - Preserve zero values (0 tokens) by avoiding truthy checks. From 31120f78b09172e970be072ac681d08b8d92f007 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Jan 2026 14:41:22 +0100 Subject: [PATCH 2/4] fix size limit --- .size-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.size-limit.js b/.size-limit.js index 24772d8380f5..cfa11e00611c 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -261,7 +261,7 @@ module.exports = [ import: createImport('init'), ignore: [...builtinModules, ...nodePrefixedBuiltinModules], gzip: true, - limit: '162 KB', + limit: '163 KB', }, { name: '@sentry/node - without tracing', From 1e7068ae2e7c2e1c62bde03d9acfaaf57e6318eb Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Jan 2026 14:45:08 +0100 Subject: [PATCH 3/4] clean up comments --- .../suites/tracing/langchain/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts index 77150a1f35be..ba1e0f63c33b 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts @@ -18,7 +18,7 @@ export default Sentry.withSentry( recordOutputs: false, }); - // Create a second handler with recordOutputs enabled for tool_calls test + // Create LangChain callback handler with recordOutputs enabled for tool_calls test const callbackHandlerWithOutputs = Sentry.createLangChainCallbackHandler({ recordInputs: false, recordOutputs: true, @@ -35,7 +35,7 @@ export default Sentry.withSentry( callbacks: [callbackHandler], }); - // Test 2: Chain invocation (without tool calls) + // Test 2: Chain invocation const chain = new MockChain('my_test_chain'); await chain.invoke( { input: 'test input' }, @@ -50,7 +50,7 @@ export default Sentry.withSentry( callbacks: [callbackHandler], }); - // Test 4: Chain invocation with tool calls (recordOutputs enabled) + // Test 4: Chain invocation with tool calls const chainWithToolCalls = new MockChain('chain_with_tool_calls', { includeToolCalls: true }); await chainWithToolCalls.invoke( { input: 'test input for tool calls' }, From f6c20b88fec6abe658f2c87538b4dc9fbdcdf897 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 8 Jan 2026 15:10:51 +0100 Subject: [PATCH 4/4] simplify --- .../suites/tracing/langchain/index.ts | 12 +++--------- .../suites/tracing/langchain/test.ts | 5 ++--- .../langchain/scenario-chain-tool-calls.mjs | 6 +++--- .../suites/tracing/langchain/test.ts | 14 ++++---------- packages/core/src/tracing/langchain/index.ts | 11 ++++++----- 5 files changed, 18 insertions(+), 30 deletions(-) diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts index ba1e0f63c33b..82e4a9fd6ea8 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/index.ts @@ -18,12 +18,6 @@ export default Sentry.withSentry( recordOutputs: false, }); - // Create LangChain callback handler with recordOutputs enabled for tool_calls test - const callbackHandlerWithOutputs = Sentry.createLangChainCallbackHandler({ - recordInputs: false, - recordOutputs: true, - }); - // Test 1: Chat model invocation const chatModel = new MockChatModel({ model: 'claude-3-5-sonnet-20241022', @@ -35,7 +29,7 @@ export default Sentry.withSentry( callbacks: [callbackHandler], }); - // Test 2: Chain invocation + // Test 2: Chain invocation (without tool calls) const chain = new MockChain('my_test_chain'); await chain.invoke( { input: 'test input' }, @@ -50,12 +44,12 @@ export default Sentry.withSentry( callbacks: [callbackHandler], }); - // Test 4: Chain invocation with tool calls + // Test 4: Chain invocation with tool calls (tool_calls captured regardless of recordOutputs) const chainWithToolCalls = new MockChain('chain_with_tool_calls', { includeToolCalls: true }); await chainWithToolCalls.invoke( { input: 'test input for tool calls' }, { - callbacks: [callbackHandlerWithOutputs], + callbacks: [callbackHandler], }, ); diff --git a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts index d06146899e9a..e63d7036856b 100644 --- a/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/cloudflare-integration-tests/suites/tracing/langchain/test.ts @@ -33,7 +33,7 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal } op: 'gen_ai.chat', origin: 'auto.ai.langchain', }), - // Chain span (without tool calls) + // Chain span expect.objectContaining({ data: expect.objectContaining({ 'sentry.origin': 'auto.ai.langchain', @@ -55,13 +55,12 @@ it('traces langchain chat model, chain, and tool invocations', async ({ signal } op: 'gen_ai.execute_tool', origin: 'auto.ai.langchain', }), - // Chain span with tool calls (recordOutputs enabled) + // Chain span with tool calls (captured regardless of recordOutputs) expect.objectContaining({ data: expect.objectContaining({ 'sentry.origin': 'auto.ai.langchain', 'sentry.op': 'gen_ai.invoke_agent', 'langchain.chain.name': 'chain_with_tool_calls', - 'langchain.chain.outputs': expect.stringContaining('Chain execution completed'), 'gen_ai.response.tool_calls': expect.stringContaining('search_tool'), }), description: 'chain chain_with_tool_calls', diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs index 6747b8acdd77..9c47363cac51 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-chain-tool-calls.mjs @@ -2,10 +2,10 @@ import { RunnableLambda } from '@langchain/core/runnables'; import * as Sentry from '@sentry/node'; async function run() { - // Create callback handler with recordOutputs enabled + // Create callback handler - tool_calls are captured regardless of recordOutputs const callbackHandler = Sentry.createLangChainCallbackHandler({ - recordInputs: true, - recordOutputs: true, + recordInputs: false, + recordOutputs: false, }); await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 240748371ea6..3e4f5d426e96 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -249,38 +249,32 @@ describe('LangChain integration', () => { const EXPECTED_TRANSACTION_CHAIN_TOOL_CALLS = { transaction: 'main', spans: expect.arrayContaining([ - // Simple chain without tool calls (RunnableLambda reports name as "unknown_chain") + // Simple chain without tool calls expect.objectContaining({ data: expect.objectContaining({ 'sentry.origin': 'auto.ai.langchain', 'sentry.op': 'gen_ai.invoke_agent', - 'langchain.chain.inputs': expect.stringContaining('Hello world'), - 'langchain.chain.outputs': expect.stringContaining('Processed'), }), op: 'gen_ai.invoke_agent', origin: 'auto.ai.langchain', status: 'ok', }), - // Chain with tool calls in messages format + // Chain with tool calls in messages format (captured regardless of recordOutputs) expect.objectContaining({ data: expect.objectContaining({ 'sentry.origin': 'auto.ai.langchain', 'sentry.op': 'gen_ai.invoke_agent', - 'langchain.chain.outputs': expect.stringContaining('Processed with tools'), - // This is the key attribute we're testing 'gen_ai.response.tool_calls': expect.stringContaining('search'), }), op: 'gen_ai.invoke_agent', origin: 'auto.ai.langchain', status: 'ok', }), - // Chain with direct tool_calls on output + // Chain with direct tool_calls on output (captured regardless of recordOutputs) expect.objectContaining({ data: expect.objectContaining({ 'sentry.origin': 'auto.ai.langchain', 'sentry.op': 'gen_ai.invoke_agent', - 'langchain.chain.outputs': expect.stringContaining('Direct tool calls'), - // This tests the alternative format (tool_calls directly on output) 'gen_ai.response.tool_calls': expect.stringContaining('weather'), }), op: 'gen_ai.invoke_agent', @@ -290,7 +284,7 @@ describe('LangChain integration', () => { ]), }; - createEsmAndCjsTests(__dirname, 'scenario-chain-tool-calls.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + createEsmAndCjsTests(__dirname, 'scenario-chain-tool-calls.mjs', 'instrument.mjs', (createRunner, test) => { test('creates langchain chain spans with tool calls', async () => { await createRunner() .ignore('event') diff --git a/packages/core/src/tracing/langchain/index.ts b/packages/core/src/tracing/langchain/index.ts index 1414faa566ce..65d94ebf1b92 100644 --- a/packages/core/src/tracing/langchain/index.ts +++ b/packages/core/src/tracing/langchain/index.ts @@ -220,13 +220,14 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): span.setAttributes({ 'langchain.chain.outputs': JSON.stringify(outputs), }); + } - // Extract tool calls from chain outputs - const toolCalls = extractToolCallsFromChainOutput(outputs); - if (toolCalls && toolCalls.length > 0) { - span.setAttribute(GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, JSON.stringify(toolCalls)); - } + // Tool calls metadata (names, IDs) are not PII, so capture them regardless of recordOutputs + const toolCalls = extractToolCallsFromChainOutput(outputs); + if (toolCalls && toolCalls.length > 0) { + span.setAttribute(GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, JSON.stringify(toolCalls)); } + exitSpan(runId); } },