From 996a818f859e2b6a5c8fcef87b1f5ad9fc7133d2 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 1 May 2025 22:54:29 +0200 Subject: [PATCH 01/67] chore: progress till now --- .../src/orchestration/client.test.ts | 49 +++++++- .../langchain/src/orchestration/client.ts | 115 +++++++++++++++++- packages/langchain/src/orchestration/types.ts | 3 +- 3 files changed, 160 insertions(+), 7 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index a527a03ea..7c2ec7edc 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -5,7 +5,8 @@ import { mockClientCredentialsGrantCall, mockDeploymentsList, mockInference, - parseMockResponse + parseMockResponse, + parseFileToString } from '../../../../test-util/mock-http.js'; import { OrchestrationClient } from './client.js'; import type { LangchainOrchestrationModuleConfig } from './types.js'; @@ -129,4 +130,50 @@ describe('orchestration service client', () => { 'Request failed with status code 400' ); }, 1000); + + it('supports streaming responses', async () => { + // Load the mock streaming response + const streamMockResponse = await parseFileToString( + 'orchestration', + 'orchestration-chat-completion-stream-chunks.txt' + ); + + // Mock the streaming API call + mockInference( + { + data: constructCompletionPostRequest( + config, + { messagesHistory: [] }, + true + ) + }, + { + data: streamMockResponse, + status: 200 + }, + { + url: 'inference/deployments/1234/completion' + } + ); + + const client = new OrchestrationClient(config); + + // Test the stream method + const stream = await client.stream([]); + + // Collect all chunks + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + + // Verify we received chunks + expect(chunks.length).toBeGreaterThan(0); + + // Verify the chunks are of the expected type + expect(chunks[0]).toBeDefined(); + expect(chunks[0].content).toBeDefined(); + }); + + // Test for disableStreaming property has been removed as the feature is no longer supported }); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index cc3fb7db7..d2cfa1af2 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -1,17 +1,18 @@ import { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orchestration'; +import { ChatGenerationChunk } from '@langchain/core/outputs'; +import { type BaseMessage } from '@langchain/core/messages'; import { isTemplate, mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; +import { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; import type { Runnable, RunnableLike } from '@langchain/core/runnables'; -import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { ChatResult } from '@langchain/core/outputs'; import type { BaseChatModelParams } from '@langchain/core/language_models/chat_models'; import type { ResourceGroupConfig } from '@sap-ai-sdk/ai-api'; -import type { BaseMessage } from '@langchain/core/messages'; import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager'; import type { OrchestrationCallOptions, @@ -34,7 +35,6 @@ export class OrchestrationClient extends BaseChatModel< OrchestrationMessageChunk > { constructor( - // TODO: Omit streaming until supported public orchestrationConfig: LangchainOrchestrationModuleConfig, public langchainOptions: BaseChatModelParams = {}, public deploymentConfig?: ResourceGroupConfig, @@ -108,7 +108,6 @@ export class OrchestrationClient extends BaseChatModel< const content = res.getContent(); - // TODO: Add streaming as soon as we support it await runManager?.handleLLMNewToken( typeof content === 'string' ? content : '' ); @@ -116,6 +115,114 @@ export class OrchestrationClient extends BaseChatModel< return mapOutputToChatResult(res.data); } + /** + * Stream response chunks from the OrchestrationClient. + * @param messages - The messages to send to the model. + * @param options - The call options. + * @param runManager - The callback manager for the run. + * @returns An async generator of chat generation chunks. + */ + override async *_streamResponseChunks( + messages: BaseMessage[], + options: typeof this.ParsedCallOptions, + runManager?: CallbackManagerForLLMRun + ): AsyncGenerator { + // Setup abort controller + const controller = new AbortController(); + if (options.signal) { + options.signal.addEventListener('abort', () => { + controller.abort(); + }); + } + + try { + // Extract options + const { inputParams, customRequestConfig } = options; + const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); + + // Create orchestration client + const orchestrationClient = new OrchestrationClientBase( + mergedOrchestrationConfig, + this.deploymentConfig, + this.destination + ); + + // Convert messages to orchestration format + const messagesHistory = + mapLangchainMessagesToOrchestrationMessages(messages); + + // Call the stream API + const response = await this.caller.callWithOptions( + { signal: options.signal }, + () => + orchestrationClient.stream( + { + messagesHistory, + inputParams + }, + controller, + { + llm: { include_usage: true } + }, + customRequestConfig + ) + ); + + // Process the stream + for await (const chunk of response.stream) { + const deltaContent = chunk.getDeltaContent(); + if (!deltaContent) { + continue; + } + + await runManager?.handleLLMNewToken(deltaContent); + + // Create a message chunk + const messageChunk = new OrchestrationMessageChunk( + deltaContent, + chunk.data.module_results || {}, + chunk.data.request_id || '' + ); + + // Yield a generation chunk + yield new ChatGenerationChunk({ + message: messageChunk, + text: deltaContent + }); + } + + // After all chunks are processed, yield a final chunk with usage metadata + // const tokenUsage = response.getTokenUsage(); + // const finishReason = response.getFinishReason(); + + // if (tokenUsage) { + // // Create a message chunk with usage metadata + // const finalMessageChunk = new OrchestrationMessageChunk( + // '', // Empty content for the final chunk + // {}, // No module results for the final chunk + // '' // No request ID for the final chunk + // ); + + // // Add usage metadata + // finalMessageChunk.usage_metadata = { + // input_tokens: tokenUsage.completion_tokens, + // output_tokens: tokenUsage.prompt_tokens, + // total_tokens: tokenUsage.total_tokens + // }; + + // // Yield the final chunk with metadata + // yield new ChatGenerationChunk({ + // message: finalMessageChunk, + // text: '', + // generationInfo: { finish_reason: finishReason } + // }); + // } + } catch (e) { + await runManager?.handleLLMError(e); + throw e; + } + } + private mergeOrchestrationConfig( options: typeof this.ParsedCallOptions ): LangchainOrchestrationModuleConfig { diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index 6b8ceefd6..8ea688217 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -29,10 +29,9 @@ export type OrchestrationCallOptions = Pick< /** * Orchestration module configuration for Langchain. */ -// TODO: Omit streaming until supported export type LangchainOrchestrationModuleConfig = Omit< OrchestrationModuleConfigWithStringTemplating, - 'streaming' | 'templating' + 'templating' > & { templating: TemplatingModuleConfig; }; From 6c9039f7334f6ca8c372e9161f5184647ed4387f Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 7 May 2025 23:46:20 +0200 Subject: [PATCH 02/67] chore: progress till now --- .../langchain/src/orchestration/client.ts | 156 +++++++++--------- packages/langchain/src/orchestration/types.ts | 2 + packages/langchain/src/orchestration/util.ts | 87 +++++++++- .../orchestration-stream-chunk-response.ts | 7 + sample-code/src/index.ts | 3 +- sample-code/src/langchain-orchestration.ts | 44 +++++ sample-code/src/server.ts | 49 +++++- .../src/orchestration-langchain.test.ts | 25 ++- 8 files changed, 291 insertions(+), 82 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index d2cfa1af2..5373b883f 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -3,6 +3,7 @@ import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orch import { ChatGenerationChunk } from '@langchain/core/outputs'; import { type BaseMessage } from '@langchain/core/messages'; import { + _convertOrchestrationChunkToMessageChunk, isTemplate, mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult @@ -127,99 +128,98 @@ export class OrchestrationClient extends BaseChatModel< options: typeof this.ParsedCallOptions, runManager?: CallbackManagerForLLMRun ): AsyncGenerator { - // Setup abort controller const controller = new AbortController(); if (options.signal) { - options.signal.addEventListener('abort', () => { - controller.abort(); - }); + options.signal.addEventListener('abort', () => controller.abort()); } - try { - // Extract options - const { inputParams, customRequestConfig } = options; - const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); - - // Create orchestration client - const orchestrationClient = new OrchestrationClientBase( - mergedOrchestrationConfig, - this.deploymentConfig, - this.destination - ); + let defaultRole: string | undefined; + const messagesHistory = + mapLangchainMessagesToOrchestrationMessages(messages); - // Convert messages to orchestration format - const messagesHistory = - mapLangchainMessagesToOrchestrationMessages(messages); + const { inputParams, customRequestConfig } = options; + const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); - // Call the stream API - const response = await this.caller.callWithOptions( - { signal: options.signal }, - () => - orchestrationClient.stream( - { - messagesHistory, - inputParams - }, - controller, - { - llm: { include_usage: true } - }, - customRequestConfig - ) - ); + const orchestrationClient = new OrchestrationClientBase( + mergedOrchestrationConfig, + this.deploymentConfig, + this.destination + ); - // Process the stream - for await (const chunk of response.stream) { - const deltaContent = chunk.getDeltaContent(); - if (!deltaContent) { - continue; - } + const streamOptions = { + llm: { + include_usage: true, + ...options.streamOptions?.llm + }, + outputFiltering: options.streamOptions?.outputFiltering, + global: options.streamOptions?.global + }; - await runManager?.handleLLMNewToken(deltaContent); + const response = await this.caller.callWithOptions( + { signal: options.signal }, + () => + orchestrationClient.stream( + { messagesHistory, inputParams }, + controller, + streamOptions, + customRequestConfig + ) + ); - // Create a message chunk - const messageChunk = new OrchestrationMessageChunk( - deltaContent, - chunk.data.module_results || {}, - chunk.data.request_id || '' - ); + for await (const chunk of response.stream) { + if (!chunk.data) { + continue; + } - // Yield a generation chunk - yield new ChatGenerationChunk({ - message: messageChunk, - text: deltaContent - }); + const delta = chunk.getDelta(); + if (!delta) { + continue; } + const messageChunk = _convertOrchestrationChunkToMessageChunk( + chunk.data, + delta, + defaultRole + ); + + defaultRole = delta.role ?? defaultRole; + const finishReason = chunk.getFinishReason(); + const tokenUsage = chunk.getTokenUsage(); - // After all chunks are processed, yield a final chunk with usage metadata - // const tokenUsage = response.getTokenUsage(); - // const finishReason = response.getFinishReason(); + // Add token usage to the message chunk if this is the final chunk + if (finishReason && tokenUsage) { + if (messageChunk instanceof OrchestrationMessageChunk) { + messageChunk.usage_metadata = { + input_tokens: tokenUsage.prompt_tokens, + output_tokens: tokenUsage.completion_tokens, + total_tokens: tokenUsage.total_tokens + }; + } + } + const content = delta.content ?? ''; + const generationChunk = new ChatGenerationChunk({ + message: messageChunk, + text: content + }); - // if (tokenUsage) { - // // Create a message chunk with usage metadata - // const finalMessageChunk = new OrchestrationMessageChunk( - // '', // Empty content for the final chunk - // {}, // No module results for the final chunk - // '' // No request ID for the final chunk - // ); + // Notify the run manager about the new token + await runManager?.handleLLMNewToken( + content, + { + prompt: 0, + completion: 0 + }, + undefined, + undefined, + undefined, + { chunk: generationChunk } + ); - // // Add usage metadata - // finalMessageChunk.usage_metadata = { - // input_tokens: tokenUsage.completion_tokens, - // output_tokens: tokenUsage.prompt_tokens, - // total_tokens: tokenUsage.total_tokens - // }; + // Yield the chunk + yield generationChunk; + } - // // Yield the final chunk with metadata - // yield new ChatGenerationChunk({ - // message: finalMessageChunk, - // text: '', - // generationInfo: { finish_reason: finishReason } - // }); - // } - } catch (e) { - await runManager?.handleLLMError(e); - throw e; + if (options.signal?.aborted) { + throw new Error('AbortError'); } } diff --git a/packages/langchain/src/orchestration/types.ts b/packages/langchain/src/orchestration/types.ts index 8ea688217..1cd2dac42 100644 --- a/packages/langchain/src/orchestration/types.ts +++ b/packages/langchain/src/orchestration/types.ts @@ -2,6 +2,7 @@ import type { Prompt, Template, TemplatingModuleConfig, + StreamOptions, OrchestrationModuleConfig as OrchestrationModuleConfigWithStringTemplating } from '@sap-ai-sdk/orchestration'; import type { BaseChatModelCallOptions } from '@langchain/core/language_models/chat_models'; @@ -24,6 +25,7 @@ export type OrchestrationCallOptions = Pick< customRequestConfig?: CustomRequestConfig; tools?: Template['tools']; inputParams?: Prompt['inputParams']; + streamOptions?: StreamOptions; }; /** diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index ee327de30..f0b697579 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,14 +1,24 @@ -import { AIMessage } from '@langchain/core/messages'; +import { + AIMessage, + ChatMessageChunk, + HumanMessageChunk, + SystemMessageChunk, + ToolMessageChunk +} from '@langchain/core/messages'; +import { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { ChatResult } from '@langchain/core/outputs'; import type { + ChatDelta, ChatMessage, CompletionPostResponse, + CompletionPostResponseStreaming, Template } from '@sap-ai-sdk/orchestration'; import type { ToolCall } from '@langchain/core/messages/tool'; import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; import type { BaseMessage, + BaseMessageChunk, HumanMessage, SystemMessage } from '@langchain/core/messages'; @@ -174,3 +184,78 @@ export function mapOutputToChatResult( } }; } + +/** + * Converts orchestration stream chunk to an appropriate message chunk based on role. + * @param chunkData - The content of the message. + * @param delta - The delta content from the chunk. + * @param defaultRole - The default role to use if not specified in the delta. + * @returns A message chunk of the appropriate type based on the role. + * @internal + */ +export function _convertOrchestrationChunkToMessageChunk( + chunkData: CompletionPostResponseStreaming, + delta: ChatDelta, + defaultRole?: string +): BaseMessageChunk { + const { module_results, request_id } = chunkData; + const role = delta.role ?? defaultRole ?? 'assistant'; + const content = delta.content ?? ''; + + // Handle additional kwargs for function and tool calls + const additional_kwargs: Record = {}; + + // Handle tool calls + if (delta.tool_calls && delta.tool_calls.length > 0) { + additional_kwargs.tool_calls = delta.tool_calls; + } + + // Create tool call chunks if present + const toolCallChunks: { + name?: string; + args?: string; + id?: string; + index?: number; + type: string; + }[] = []; + if (Array.isArray(delta.tool_calls)) { + for (const toolCall of delta.tool_calls) { + toolCallChunks.push({ + name: toolCall.function?.name, + args: toolCall.function?.arguments, + id: toolCall.id, + index: toolCall.index, + type: 'function' + }); + } + } + + // Return the appropriate message chunk based on role + switch (role) { + case 'user': + return new HumanMessageChunk({ content }); + + case 'assistant': { + return new OrchestrationMessageChunk( + { content, additional_kwargs }, + module_results ?? {}, + request_id + ); + } + + case 'system': + return new SystemMessageChunk({ content }); + + case 'tool': { + const toolCallId = delta.tool_calls?.[0]?.id ?? ''; + return new ToolMessageChunk({ + content, + additional_kwargs, + tool_call_id: toolCallId + }); + } + + default: + return new ChatMessageChunk({ content, role }); + } +} diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.ts b/packages/orchestration/src/orchestration-stream-chunk-response.ts index 9a3b4a2cf..5b22e1b09 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.ts @@ -1,4 +1,5 @@ import type { + ChatDelta, CompletionPostResponseStreaming, LlmChoiceStreaming, TokenUsage @@ -42,6 +43,12 @@ export class OrchestrationStreamChunkResponse { )?.delta.content; } + getDelta(choiceIndex = 0): ChatDelta | undefined { + return this.getChoices()?.find( + (c: LlmChoiceStreaming) => c.index === choiceIndex + )?.delta; + } + private getChoices(): LlmChoiceStreaming[] | undefined { return this.data.orchestration_result?.choices; } diff --git a/sample-code/src/index.ts b/sample-code/src/index.ts index 4e0b0ea68..4b30a2372 100644 --- a/sample-code/src/index.ts +++ b/sample-code/src/index.ts @@ -31,7 +31,8 @@ export { } from './langchain-azure-openai.js'; export { invokeChain as orchestrationInvokeChain, - invokeLangGraphChain + invokeLangGraphChain, + streamOrchestrationLangChain } from './langchain-orchestration.js'; export { getDeployments, diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 31a7b0eb3..5a0d3acab 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -14,6 +14,7 @@ import { } from '@langchain/langgraph'; import { v4 as uuidv4 } from 'uuid'; import type { LangchainOrchestrationModuleConfig } from '@sap-ai-sdk/langchain'; +import type { OrchestrationMessageChunk } from '@sap-ai-sdk/langchain/orchestration/index.js'; /** * Ask GPT about an introduction to SAP Cloud SDK. @@ -187,6 +188,49 @@ export async function invokeLangGraphChain(): Promise { return `${firstResponse}\n\n${secondResponse}`; } +/** + * Stream responses from the OrchestrationClient using LangChain. + * @param controller - The abort controller to cancel the request if needed. + * @returns An async iterable of AIMessageChunk objects. + */ +export async function streamOrchestrationLangChain( + controller = new AbortController() +): Promise> { + const orchestrationConfig: LangchainOrchestrationModuleConfig = { + // define the language model to be used + llm: { + model_name: 'gpt-4o' + }, + // define the template + templating: { + template: [ + { + role: 'user', + content: 'Write a 100 word explanation about {{?topic}}' + } + ] + } + }; + + const client = new OrchestrationClient(orchestrationConfig); + + // Return the stream + return client.stream( + [ + { + role: 'user', + content: 'I need information about a topic.' + } + ], + { + inputParams: { + topic: 'SAP Cloud SDK and its capabilities' + }, + signal: controller.signal + } + ); +} + /** * Trigger masking the input provided to the large language model. * @returns The answer from ChatGPT. diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index f2ff3420b..cbd7d81ba 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -48,7 +48,8 @@ import { invokeChainWithInputFilter as invokeChainWithInputFilterOrchestration, invokeChainWithOutputFilter as invokeChainWithOutputFilterOrchestration, invokeLangGraphChain, - invokeChainWithMasking + invokeChainWithMasking, + streamOrchestrationLangChain } from './langchain-orchestration.js'; import { createCollection, @@ -465,6 +466,52 @@ app.get('/langchain/invoke-stateful-chain', async (req, res) => { } }); +app.get('/langchain/stream-orchestration', async (req, res) => { + const controller = new AbortController(); + try { + const stream = await streamOrchestrationLangChain(controller); + + // Set headers for event stream. + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Connection', 'keep-alive'); + res.flushHeaders(); + + let connectionAlive = true; + + // Abort the stream if the client connection is closed. + res.on('close', () => { + controller.abort(); + connectionAlive = false; + res.end(); + }); + + // Stream the delta content. + for await (const chunk of stream) { + if (!connectionAlive) { + break; + } + // Use the content property similar to getDeltaContent() + res.write(chunk.content + '\n'); + if (connectionAlive && chunk.usage_metadata) { + res.write('\n\n---------------------------\n'); + res.write('Finish reason: stop\n'); + res.write('Token usage:\n'); + res.write( + ` - Completion tokens: ${chunk.usage_metadata?.output_tokens}\n` + ); + res.write(` - Prompt tokens: ${chunk.usage_metadata?.input_tokens}\n`); + res.write(` - Total tokens: ${chunk.usage_metadata?.total_tokens}\n`); + } + } + + // Write the finish reason and token usage after the stream ends. + } catch (error: any) { + sendError(res, error, false); + } finally { + res.end(); + } +}); + /* Document Grounding */ app.get( '/document-grounding/orchestration-grounding-vector', diff --git a/tests/e2e-tests/src/orchestration-langchain.test.ts b/tests/e2e-tests/src/orchestration-langchain.test.ts index b08b1cbce..62c500363 100644 --- a/tests/e2e-tests/src/orchestration-langchain.test.ts +++ b/tests/e2e-tests/src/orchestration-langchain.test.ts @@ -1,6 +1,7 @@ import { orchestrationInvokeChain, - invokeLangGraphChain + invokeLangGraphChain, + streamOrchestrationLangChain } from '@sap-ai-sdk/sample-code'; import { loadEnv } from './utils/load-env.js'; @@ -11,8 +12,30 @@ describe('Orchestration LangChain client', () => { const result = await orchestrationInvokeChain(); expect(result).toContain('SAP Cloud SDK'); }); + it('executes an invoke with LangGraph', async () => { const result = await invokeLangGraphChain(); expect(result).toContain('SAP Cloud SDK'); }); + + it('supports streaming responses', async () => { + // Create an abort controller for the test + const controller = new AbortController(); + + // Get the stream + const stream = await streamOrchestrationLangChain(controller); + + // Collect all chunks + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + + // Verify we received chunks + expect(chunks.length).toBeGreaterThan(0); + + // Verify the chunks contain expected content + const combinedContent = chunks.map(chunk => chunk.content).join(''); + expect(combinedContent).toContain('SAP Cloud SDK'); + }); }); From a54250fcec06200bdf4eb1e9fbe0340e8a8a9648 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 9 May 2025 10:37:17 +0200 Subject: [PATCH 03/67] chore: update implementation --- packages/langchain/src/index.ts | 5 +- .../src/orchestration/client.test.ts | 5 +- .../langchain/src/orchestration/client.ts | 1 + .../orchestration-message-chunk.ts | 7 +- packages/langchain/src/orchestration/util.ts | 91 ++++++++----------- sample-code/src/langchain-orchestration.ts | 6 +- sample-code/src/server.ts | 2 +- 7 files changed, 54 insertions(+), 63 deletions(-) diff --git a/packages/langchain/src/index.ts b/packages/langchain/src/index.ts index 8823abeb8..df507a939 100644 --- a/packages/langchain/src/index.ts +++ b/packages/langchain/src/index.ts @@ -7,7 +7,10 @@ export type { AzureOpenAiEmbeddingModelParams, AzureOpenAiChatCallOptions } from './openai/index.js'; -export { OrchestrationClient } from './orchestration/index.js'; +export { + OrchestrationClient, + OrchestrationMessageChunk +} from './orchestration/index.js'; export type { OrchestrationCallOptions, LangchainOrchestrationModuleConfig diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 7c2ec7edc..99e72323c 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -9,6 +9,7 @@ import { parseFileToString } from '../../../../test-util/mock-http.js'; import { OrchestrationClient } from './client.js'; +import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { LangchainOrchestrationModuleConfig } from './types.js'; import type { CompletionPostResponse, @@ -162,7 +163,7 @@ describe('orchestration service client', () => { const stream = await client.stream([]); // Collect all chunks - const chunks = []; + const chunks: OrchestrationMessageChunk[] = []; for await (const chunk of stream) { chunks.push(chunk); } @@ -174,6 +175,4 @@ describe('orchestration service client', () => { expect(chunks[0]).toBeDefined(); expect(chunks[0].content).toBeDefined(); }); - - // Test for disableStreaming property has been removed as the feature is no longer supported }); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 5373b883f..122f1a307 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -188,6 +188,7 @@ export class OrchestrationClient extends BaseChatModel< // Add token usage to the message chunk if this is the final chunk if (finishReason && tokenUsage) { if (messageChunk instanceof OrchestrationMessageChunk) { + messageChunk.additional_kwargs.finish_reason = finishReason; messageChunk.usage_metadata = { input_tokens: tokenUsage.prompt_tokens, output_tokens: tokenUsage.completion_tokens, diff --git a/packages/langchain/src/orchestration/orchestration-message-chunk.ts b/packages/langchain/src/orchestration/orchestration-message-chunk.ts index 67236b7e7..45c4cb6c0 100644 --- a/packages/langchain/src/orchestration/orchestration-message-chunk.ts +++ b/packages/langchain/src/orchestration/orchestration-message-chunk.ts @@ -4,18 +4,21 @@ import type { ModuleResults } from '@sap-ai-sdk/orchestration'; /** * An AI Message Chunk containing module results and request ID. - * @internal */ export class OrchestrationMessageChunk extends AIMessageChunk { module_results: ModuleResults; request_id: string; + // Adding additonal properties to also store properties from other types of Message Chunks. + additional_properties?: Record; constructor( fields: string | AIMessageChunkFields, module_results: ModuleResults, - request_id: string + request_id: string, + additional_properties?: Record ) { super(fields); this.module_results = module_results; this.request_id = request_id; + this.additional_properties = additional_properties; } } diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index f0b697579..621dbccce 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,10 +1,4 @@ -import { - AIMessage, - ChatMessageChunk, - HumanMessageChunk, - SystemMessageChunk, - ToolMessageChunk -} from '@langchain/core/messages'; +import { AIMessage } from '@langchain/core/messages'; import { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { ChatResult } from '@langchain/core/outputs'; import type { @@ -12,9 +6,10 @@ import type { ChatMessage, CompletionPostResponse, CompletionPostResponseStreaming, - Template + Template, + ToolCallChunk as OrchestrationToolCallChunk } from '@sap-ai-sdk/orchestration'; -import type { ToolCall } from '@langchain/core/messages/tool'; +import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool'; import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; import type { BaseMessage, @@ -133,6 +128,27 @@ function mapAzureOpenAiToLangchainToolCall( } } +/** + * Maps {@link OrchestrationToolCallChunk} to LangChain's {@link ToolCallChunk}. + * @param toolCallChunks - The {@link OrchestrationToolCallChunk} in a stream response chunk. + * @returns An array of LangChain {@link ToolCallChunk}. + */ +function mapOrchestrationToLangchainToolCallChunk( + toolCallChunks: OrchestrationToolCallChunk[] +): ToolCallChunk[] { + const tool_call_chunks: ToolCallChunk[] = []; + for (const chunk of toolCallChunks) { + tool_call_chunks.push({ + name: chunk.function?.name, + args: chunk.function?.arguments, + id: chunk.id, + index: chunk.index, + type: 'tool_call_chunk' + }); + } + return tool_call_chunks; +} + /** * Maps the completion response to a {@link ChatResult}. * @param completionResponse - The completion response to map. @@ -211,51 +227,18 @@ export function _convertOrchestrationChunkToMessageChunk( } // Create tool call chunks if present - const toolCallChunks: { - name?: string; - args?: string; - id?: string; - index?: number; - type: string; - }[] = []; + let tool_call_chunks: ToolCallChunk[] = []; if (Array.isArray(delta.tool_calls)) { - for (const toolCall of delta.tool_calls) { - toolCallChunks.push({ - name: toolCall.function?.name, - args: toolCall.function?.arguments, - id: toolCall.id, - index: toolCall.index, - type: 'function' - }); - } - } - - // Return the appropriate message chunk based on role - switch (role) { - case 'user': - return new HumanMessageChunk({ content }); - - case 'assistant': { - return new OrchestrationMessageChunk( - { content, additional_kwargs }, - module_results ?? {}, - request_id - ); - } - - case 'system': - return new SystemMessageChunk({ content }); - - case 'tool': { - const toolCallId = delta.tool_calls?.[0]?.id ?? ''; - return new ToolMessageChunk({ - content, - additional_kwargs, - tool_call_id: toolCallId - }); - } - - default: - return new ChatMessageChunk({ content, role }); + tool_call_chunks = mapOrchestrationToLangchainToolCallChunk( + delta.tool_calls + ); } + const toolCallId = delta.tool_calls?.[0]?.id ?? undefined; + // Use OrchestrationMessageChunk to represent message chunks for roles like 'tool' and 'user' too + return new OrchestrationMessageChunk( + { content, additional_kwargs, tool_call_chunks }, + module_results ?? {}, + request_id, + { role, ...(toolCallId && { tool_call_id: toolCallId }) } + ); } diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 5a0d3acab..c6de2ae4f 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -13,8 +13,10 @@ import { MemorySaver } from '@langchain/langgraph'; import { v4 as uuidv4 } from 'uuid'; -import type { LangchainOrchestrationModuleConfig } from '@sap-ai-sdk/langchain'; -import type { OrchestrationMessageChunk } from '@sap-ai-sdk/langchain/orchestration/index.js'; +import type { + LangchainOrchestrationModuleConfig, + OrchestrationMessageChunk +} from '@sap-ai-sdk/langchain'; /** * Ask GPT about an introduction to SAP Cloud SDK. diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index 9a2865364..d398c7b49 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -495,7 +495,7 @@ app.get('/langchain/stream-orchestration', async (req, res) => { res.write(chunk.content + '\n'); if (connectionAlive && chunk.usage_metadata) { res.write('\n\n---------------------------\n'); - res.write('Finish reason: stop\n'); + res.write(`Finish reason: ${chunk.additional_kwargs.finish_reason}\n`); res.write('Token usage:\n'); res.write( ` - Completion tokens: ${chunk.usage_metadata?.output_tokens}\n` From e0f0e68fa7377957930b408b16a1bf19c5b33a87 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 9 May 2025 10:42:32 +0200 Subject: [PATCH 04/67] chore: minor corrections --- packages/langchain/src/orchestration/util.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 621dbccce..0d9ca41dc 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -202,11 +202,11 @@ export function mapOutputToChatResult( } /** - * Converts orchestration stream chunk to an appropriate message chunk based on role. + * Converts orchestration stream chunk to amessage chunk. * @param chunkData - The content of the message. * @param delta - The delta content from the chunk. * @param defaultRole - The default role to use if not specified in the delta. - * @returns A message chunk of the appropriate type based on the role. + * @returns A message chunk compatible with Langchain's {@link AIMessageChunk} * @internal */ export function _convertOrchestrationChunkToMessageChunk( @@ -218,7 +218,7 @@ export function _convertOrchestrationChunkToMessageChunk( const role = delta.role ?? defaultRole ?? 'assistant'; const content = delta.content ?? ''; - // Handle additional kwargs for function and tool calls + // Handle additional kwargs for tool calls const additional_kwargs: Record = {}; // Handle tool calls From 9a743b611347743269171d04db83ac4d29ac5e26 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 9 May 2025 10:46:06 +0200 Subject: [PATCH 05/67] chore: add changeset --- .changeset/chilly-steaks-divide.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/chilly-steaks-divide.md diff --git a/.changeset/chilly-steaks-divide.md b/.changeset/chilly-steaks-divide.md new file mode 100644 index 000000000..b59d9c67c --- /dev/null +++ b/.changeset/chilly-steaks-divide.md @@ -0,0 +1,6 @@ +--- +'@sap-ai-sdk/orchestration': minor +'@sap-ai-sdk/langchain': minor +--- + +[New Functionality] Support streaming in Langchain orchestration client. From 8c4a7ecf0267e475c8357e2819d508323060f2ec Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 9 May 2025 11:26:50 +0200 Subject: [PATCH 06/67] chore: change finish_reason storage --- packages/langchain/src/orchestration/client.ts | 2 +- packages/langchain/src/orchestration/util.ts | 3 +-- sample-code/src/server.ts | 4 +++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 122f1a307..ca90ac786 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -188,7 +188,7 @@ export class OrchestrationClient extends BaseChatModel< // Add token usage to the message chunk if this is the final chunk if (finishReason && tokenUsage) { if (messageChunk instanceof OrchestrationMessageChunk) { - messageChunk.additional_kwargs.finish_reason = finishReason; + messageChunk.response_metadata.finish_reason = finishReason; messageChunk.usage_metadata = { input_tokens: tokenUsage.prompt_tokens, output_tokens: tokenUsage.completion_tokens, diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 0d9ca41dc..bacf38849 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -13,7 +13,6 @@ import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool'; import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; import type { BaseMessage, - BaseMessageChunk, HumanMessage, SystemMessage } from '@langchain/core/messages'; @@ -213,7 +212,7 @@ export function _convertOrchestrationChunkToMessageChunk( chunkData: CompletionPostResponseStreaming, delta: ChatDelta, defaultRole?: string -): BaseMessageChunk { +): OrchestrationMessageChunk { const { module_results, request_id } = chunkData; const role = delta.role ?? defaultRole ?? 'assistant'; const content = delta.content ?? ''; diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index d398c7b49..93c93e8bc 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -495,7 +495,9 @@ app.get('/langchain/stream-orchestration', async (req, res) => { res.write(chunk.content + '\n'); if (connectionAlive && chunk.usage_metadata) { res.write('\n\n---------------------------\n'); - res.write(`Finish reason: ${chunk.additional_kwargs.finish_reason}\n`); + res.write( + `Finish reason: ${chunk.response_metadata?.finish_reason}\n` + ); res.write('Token usage:\n'); res.write( ` - Completion tokens: ${chunk.usage_metadata?.output_tokens}\n` From db8491f0537ae2949cfbb761e5cc67a7d8b461a9 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 9 May 2025 11:32:17 +0200 Subject: [PATCH 07/67] chore: update additonal properties --- .../src/orchestration/orchestration-message-chunk.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/orchestration-message-chunk.ts b/packages/langchain/src/orchestration/orchestration-message-chunk.ts index 45c4cb6c0..8ad626dc2 100644 --- a/packages/langchain/src/orchestration/orchestration-message-chunk.ts +++ b/packages/langchain/src/orchestration/orchestration-message-chunk.ts @@ -14,7 +14,11 @@ export class OrchestrationMessageChunk extends AIMessageChunk { fields: string | AIMessageChunkFields, module_results: ModuleResults, request_id: string, - additional_properties?: Record + additional_properties?: { + role?: string; + tool_call_id?: string; + [key: string]: any; + } ) { super(fields); this.module_results = module_results; From fd37e19c4e67b7f5947a7b6a62e2c7aafdd027fd Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 9 May 2025 11:37:22 +0200 Subject: [PATCH 08/67] chore: revert type change for additional_properties --- .../src/orchestration/orchestration-message-chunk.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/langchain/src/orchestration/orchestration-message-chunk.ts b/packages/langchain/src/orchestration/orchestration-message-chunk.ts index 8ad626dc2..45c4cb6c0 100644 --- a/packages/langchain/src/orchestration/orchestration-message-chunk.ts +++ b/packages/langchain/src/orchestration/orchestration-message-chunk.ts @@ -14,11 +14,7 @@ export class OrchestrationMessageChunk extends AIMessageChunk { fields: string | AIMessageChunkFields, module_results: ModuleResults, request_id: string, - additional_properties?: { - role?: string; - tool_call_id?: string; - [key: string]: any; - } + additional_properties?: Record ) { super(fields); this.module_results = module_results; From 175e9b98581c341a099c4dca81828473f1f6424e Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Wed, 14 May 2025 09:59:52 +0200 Subject: [PATCH 09/67] Update .changeset/chilly-steaks-divide.md Co-authored-by: Zhongpin Wang --- .changeset/chilly-steaks-divide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/chilly-steaks-divide.md b/.changeset/chilly-steaks-divide.md index b59d9c67c..517818944 100644 --- a/.changeset/chilly-steaks-divide.md +++ b/.changeset/chilly-steaks-divide.md @@ -3,4 +3,4 @@ '@sap-ai-sdk/langchain': minor --- -[New Functionality] Support streaming in Langchain orchestration client. +[New Functionality] Support streaming in LangChain orchestration client. From dd0a47c3971fc1fb4a9a148073570b103d1a7f00 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 14 May 2025 17:01:42 +0200 Subject: [PATCH 10/67] chore: address review comments --- .../src/orchestration/client.test.ts | 4 +- .../langchain/src/orchestration/client.ts | 41 ++++--------- .../orchestration-message-chunk.ts | 6 +- packages/langchain/src/orchestration/util.ts | 58 +++++++------------ .../orchestration-stream-chunk-response.ts | 13 +++-- sample-code/src/langchain-orchestration.ts | 8 +-- 6 files changed, 46 insertions(+), 84 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 99e72323c..cffe86e7c 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -9,12 +9,12 @@ import { parseFileToString } from '../../../../test-util/mock-http.js'; import { OrchestrationClient } from './client.js'; -import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { LangchainOrchestrationModuleConfig } from './types.js'; import type { CompletionPostResponse, ErrorResponse } from '@sap-ai-sdk/orchestration'; +import type { AIMessageChunk } from '@langchain/core/messages'; jest.setTimeout(30000); @@ -163,7 +163,7 @@ describe('orchestration service client', () => { const stream = await client.stream([]); // Collect all chunks - const chunks: OrchestrationMessageChunk[] = []; + const chunks: AIMessageChunk[] = []; for await (const chunk of stream) { chunks.push(chunk); } diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index ca90ac786..bc64175e4 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -3,12 +3,12 @@ import { OrchestrationClient as OrchestrationClientBase } from '@sap-ai-sdk/orch import { ChatGenerationChunk } from '@langchain/core/outputs'; import { type BaseMessage } from '@langchain/core/messages'; import { - _convertOrchestrationChunkToMessageChunk, + mapOrchestrationChunkToLangChainMessageChunk, isTemplate, mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; -import { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; +import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; import type { Runnable, RunnableLike } from '@langchain/core/runnables'; import type { ChatResult } from '@langchain/core/outputs'; @@ -133,7 +133,6 @@ export class OrchestrationClient extends BaseChatModel< options.signal.addEventListener('abort', () => controller.abort()); } - let defaultRole: string | undefined; const messagesHistory = mapLangchainMessagesToOrchestrationMessages(messages); @@ -146,22 +145,13 @@ export class OrchestrationClient extends BaseChatModel< this.destination ); - const streamOptions = { - llm: { - include_usage: true, - ...options.streamOptions?.llm - }, - outputFiltering: options.streamOptions?.outputFiltering, - global: options.streamOptions?.global - }; - const response = await this.caller.callWithOptions( { signal: options.signal }, () => orchestrationClient.stream( { messagesHistory, inputParams }, controller, - streamOptions, + options.streamOptions, customRequestConfig ) ); @@ -171,30 +161,23 @@ export class OrchestrationClient extends BaseChatModel< continue; } - const delta = chunk.getDelta(); + const delta = chunk.data.orchestration_result?.choices[0]?.delta; if (!delta) { continue; } - const messageChunk = _convertOrchestrationChunkToMessageChunk( - chunk.data, - delta, - defaultRole - ); + const messageChunk = mapOrchestrationChunkToLangChainMessageChunk(chunk); - defaultRole = delta.role ?? defaultRole; const finishReason = chunk.getFinishReason(); const tokenUsage = chunk.getTokenUsage(); // Add token usage to the message chunk if this is the final chunk if (finishReason && tokenUsage) { - if (messageChunk instanceof OrchestrationMessageChunk) { - messageChunk.response_metadata.finish_reason = finishReason; - messageChunk.usage_metadata = { - input_tokens: tokenUsage.prompt_tokens, - output_tokens: tokenUsage.completion_tokens, - total_tokens: tokenUsage.total_tokens - }; - } + messageChunk.response_metadata.finish_reason = finishReason; + messageChunk.usage_metadata = { + input_tokens: tokenUsage.prompt_tokens, + output_tokens: tokenUsage.completion_tokens, + total_tokens: tokenUsage.total_tokens + }; } const content = delta.content ?? ''; const generationChunk = new ChatGenerationChunk({ @@ -202,7 +185,6 @@ export class OrchestrationClient extends BaseChatModel< text: content }); - // Notify the run manager about the new token await runManager?.handleLLMNewToken( content, { @@ -215,7 +197,6 @@ export class OrchestrationClient extends BaseChatModel< { chunk: generationChunk } ); - // Yield the chunk yield generationChunk; } diff --git a/packages/langchain/src/orchestration/orchestration-message-chunk.ts b/packages/langchain/src/orchestration/orchestration-message-chunk.ts index 45c4cb6c0..7056829f6 100644 --- a/packages/langchain/src/orchestration/orchestration-message-chunk.ts +++ b/packages/langchain/src/orchestration/orchestration-message-chunk.ts @@ -8,17 +8,13 @@ import type { ModuleResults } from '@sap-ai-sdk/orchestration'; export class OrchestrationMessageChunk extends AIMessageChunk { module_results: ModuleResults; request_id: string; - // Adding additonal properties to also store properties from other types of Message Chunks. - additional_properties?: Record; constructor( fields: string | AIMessageChunkFields, module_results: ModuleResults, - request_id: string, - additional_properties?: Record + request_id: string ) { super(fields); this.module_results = module_results; this.request_id = request_id; - this.additional_properties = additional_properties; } } diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index bacf38849..12d13ad74 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -1,13 +1,11 @@ -import { AIMessage } from '@langchain/core/messages'; -import { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; +import { AIMessage, AIMessageChunk } from '@langchain/core/messages'; import type { ChatResult } from '@langchain/core/outputs'; import type { - ChatDelta, ChatMessage, CompletionPostResponse, - CompletionPostResponseStreaming, Template, - ToolCallChunk as OrchestrationToolCallChunk + ToolCallChunk as OrchestrationToolCallChunk, + OrchestrationStreamChunkResponse } from '@sap-ai-sdk/orchestration'; import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool'; import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; @@ -201,43 +199,27 @@ export function mapOutputToChatResult( } /** - * Converts orchestration stream chunk to amessage chunk. - * @param chunkData - The content of the message. - * @param delta - The delta content from the chunk. - * @param defaultRole - The default role to use if not specified in the delta. - * @returns A message chunk compatible with Langchain's {@link AIMessageChunk} + * Converts orchestration stream chunk to a LangChain message chunk. + * @param chunk- The orchestration stream chunk. + * @returns An {@link AIMessageChunk} * @internal */ -export function _convertOrchestrationChunkToMessageChunk( - chunkData: CompletionPostResponseStreaming, - delta: ChatDelta, - defaultRole?: string -): OrchestrationMessageChunk { - const { module_results, request_id } = chunkData; - const role = delta.role ?? defaultRole ?? 'assistant'; - const content = delta.content ?? ''; +export function mapOrchestrationChunkToLangChainMessageChunk( + chunk: OrchestrationStreamChunkResponse +): AIMessageChunk { + const { module_results, request_id } = chunk.data; + const content = chunk.getDeltaContent() ?? ''; + const toolCallChunks = chunk.getDeltaToolCallChunks(); - // Handle additional kwargs for tool calls - const additional_kwargs: Record = {}; - - // Handle tool calls - if (delta.tool_calls && delta.tool_calls.length > 0) { - additional_kwargs.tool_calls = delta.tool_calls; - } + const additional_kwargs: Record = { + module_results, + request_id + }; - // Create tool call chunks if present let tool_call_chunks: ToolCallChunk[] = []; - if (Array.isArray(delta.tool_calls)) { - tool_call_chunks = mapOrchestrationToLangchainToolCallChunk( - delta.tool_calls - ); + if (Array.isArray(toolCallChunks)) { + tool_call_chunks = mapOrchestrationToLangchainToolCallChunk(toolCallChunks); } - const toolCallId = delta.tool_calls?.[0]?.id ?? undefined; - // Use OrchestrationMessageChunk to represent message chunks for roles like 'tool' and 'user' too - return new OrchestrationMessageChunk( - { content, additional_kwargs, tool_call_chunks }, - module_results ?? {}, - request_id, - { role, ...(toolCallId && { tool_call_id: toolCallId }) } - ); + // Use AIMessageChunk to represent message chunks for roles like 'tool' and 'user' too + return new AIMessageChunk({ content, additional_kwargs, tool_call_chunks }); } diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.ts b/packages/orchestration/src/orchestration-stream-chunk-response.ts index 5b22e1b09..33915f999 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.ts @@ -1,8 +1,8 @@ import type { - ChatDelta, CompletionPostResponseStreaming, LlmChoiceStreaming, - TokenUsage + TokenUsage, + ToolCallChunk } from './client/api/schema/index.js'; /** @@ -43,10 +43,15 @@ export class OrchestrationStreamChunkResponse { )?.delta.content; } - getDelta(choiceIndex = 0): ChatDelta | undefined { + /** + * Parses the chunk response and returns the tool call chunks generated by the model. + * @param choiceIndex - The index of the choice to parse. + * @returns The message tool call chunks. + */ + getDeltaToolCallChunks(choiceIndex = 0): ToolCallChunk[] | undefined { return this.getChoices()?.find( (c: LlmChoiceStreaming) => c.index === choiceIndex - )?.delta; + )?.delta.tool_calls; } private getChoices(): LlmChoiceStreaming[] | undefined { diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index c6de2ae4f..fcf31e42a 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -13,10 +13,8 @@ import { MemorySaver } from '@langchain/langgraph'; import { v4 as uuidv4 } from 'uuid'; -import type { - LangchainOrchestrationModuleConfig, - OrchestrationMessageChunk -} from '@sap-ai-sdk/langchain'; +import type { AIMessageChunk } from '@langchain/core/messages'; +import type { LangchainOrchestrationModuleConfig } from '@sap-ai-sdk/langchain'; /** * Ask GPT about an introduction to SAP Cloud SDK. @@ -197,7 +195,7 @@ export async function invokeLangGraphChain(): Promise { */ export async function streamOrchestrationLangChain( controller = new AbortController() -): Promise> { +): Promise> { const orchestrationConfig: LangchainOrchestrationModuleConfig = { // define the language model to be used llm: { From 96df5645063df164481ad152d960de58e8309a84 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 14 May 2025 17:07:08 +0200 Subject: [PATCH 11/67] chore: use convenience functions always --- packages/langchain/src/orchestration/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index bc64175e4..20440ba5a 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -179,7 +179,7 @@ export class OrchestrationClient extends BaseChatModel< total_tokens: tokenUsage.total_tokens }; } - const content = delta.content ?? ''; + const content = chunk.getDeltaContent() ?? ''; const generationChunk = new ChatGenerationChunk({ message: messageChunk, text: content From 5769148a6abfe82914d5fdf08d5cb6cf132c0b7e Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 15 May 2025 10:30:28 +0200 Subject: [PATCH 12/67] chore: switch to map --- packages/langchain/src/orchestration/util.ts | 23 +++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 12d13ad74..32ba66c37 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -133,17 +133,13 @@ function mapAzureOpenAiToLangchainToolCall( function mapOrchestrationToLangchainToolCallChunk( toolCallChunks: OrchestrationToolCallChunk[] ): ToolCallChunk[] { - const tool_call_chunks: ToolCallChunk[] = []; - for (const chunk of toolCallChunks) { - tool_call_chunks.push({ - name: chunk.function?.name, - args: chunk.function?.arguments, - id: chunk.id, - index: chunk.index, - type: 'tool_call_chunk' - }); - } - return tool_call_chunks; + return toolCallChunks.map(chunk => ({ + name: chunk.function?.name, + args: chunk.function?.arguments, + id: chunk.id, + index: chunk.index, + type: 'tool_call_chunk' + })); } /** @@ -207,13 +203,14 @@ export function mapOutputToChatResult( export function mapOrchestrationChunkToLangChainMessageChunk( chunk: OrchestrationStreamChunkResponse ): AIMessageChunk { - const { module_results, request_id } = chunk.data; + const { module_results, request_id, orchestration_result } = chunk.data; const content = chunk.getDeltaContent() ?? ''; const toolCallChunks = chunk.getDeltaToolCallChunks(); const additional_kwargs: Record = { module_results, - request_id + request_id, + orchestration_result }; let tool_call_chunks: ToolCallChunk[] = []; From 643a6bbf901a57c9fbeb50c4350cc0e311ee567b Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 15 May 2025 11:23:03 +0200 Subject: [PATCH 13/67] chore: add more convenience functions --- .../langchain/src/orchestration/client.ts | 13 ++----- packages/langchain/src/orchestration/util.ts | 37 ++++++++++++++++++- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 20440ba5a..1e62d90df 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -5,6 +5,8 @@ import { type BaseMessage } from '@langchain/core/messages'; import { mapOrchestrationChunkToLangChainMessageChunk, isTemplate, + setFinishReason, + setUsageMetadata, mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; @@ -170,15 +172,8 @@ export class OrchestrationClient extends BaseChatModel< const finishReason = chunk.getFinishReason(); const tokenUsage = chunk.getTokenUsage(); - // Add token usage to the message chunk if this is the final chunk - if (finishReason && tokenUsage) { - messageChunk.response_metadata.finish_reason = finishReason; - messageChunk.usage_metadata = { - input_tokens: tokenUsage.prompt_tokens, - output_tokens: tokenUsage.completion_tokens, - total_tokens: tokenUsage.total_tokens - }; - } + setFinishReason(messageChunk, finishReason); + setUsageMetadata(messageChunk, tokenUsage); const content = chunk.getDeltaContent() ?? ''; const generationChunk = new ChatGenerationChunk({ message: messageChunk, diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 32ba66c37..4941318f5 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -5,7 +5,8 @@ import type { CompletionPostResponse, Template, ToolCallChunk as OrchestrationToolCallChunk, - OrchestrationStreamChunkResponse + OrchestrationStreamChunkResponse, + TokenUsage } from '@sap-ai-sdk/orchestration'; import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool'; import type { AzureOpenAiChatCompletionMessageToolCalls } from '@sap-ai-sdk/foundation-models'; @@ -220,3 +221,37 @@ export function mapOrchestrationChunkToLangChainMessageChunk( // Use AIMessageChunk to represent message chunks for roles like 'tool' and 'user' too return new AIMessageChunk({ content, additional_kwargs, tool_call_chunks }); } + +/** + * Sets finish reason on a message chunk if available. + * @param messageChunk - The message chunk to update. + * @param finishReason - The finish reason from the response. + * @internal + */ +export function setFinishReason( + messageChunk: AIMessageChunk, + finishReason: string | undefined +): void { + if (finishReason) { + messageChunk.response_metadata.finish_reason = finishReason; + } +} + +/** + * Sets usage metadata on a message chunk if available. + * @param messageChunk - The message chunk to update. + * @param tokenUsage - The token usage information. + * @internal + */ +export function setUsageMetadata( + messageChunk: AIMessageChunk, + tokenUsage: TokenUsage | undefined +): void { + if (tokenUsage) { + messageChunk.usage_metadata = { + input_tokens: tokenUsage.prompt_tokens, + output_tokens: tokenUsage.completion_tokens, + total_tokens: tokenUsage.total_tokens + }; + } +} From 7503191cea535e91ac6465d2ad5ee32d876e064a Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Sun, 18 May 2025 17:17:47 +0200 Subject: [PATCH 14/67] chore: address review comments --- .../langchain/src/orchestration/client.ts | 30 +++++++++++-------- packages/langchain/src/orchestration/util.ts | 8 ++--- sample-code/src/langchain-orchestration.ts | 25 +++++----------- 3 files changed, 28 insertions(+), 35 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 1e62d90df..aa525c991 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -6,7 +6,7 @@ import { mapOrchestrationChunkToLangChainMessageChunk, isTemplate, setFinishReason, - setUsageMetadata, + setTokenUsage, mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult } from './util.js'; @@ -135,7 +135,7 @@ export class OrchestrationClient extends BaseChatModel< options.signal.addEventListener('abort', () => controller.abort()); } - const messagesHistory = + const orchestrationMessages = mapLangchainMessagesToOrchestrationMessages(messages); const { inputParams, customRequestConfig } = options; @@ -151,7 +151,8 @@ export class OrchestrationClient extends BaseChatModel< { signal: options.signal }, () => orchestrationClient.stream( - { messagesHistory, inputParams }, + // Todo Adapt messagesHistory after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 + { messagesHistory: orchestrationMessages, inputParams }, controller, options.streamOptions, customRequestConfig @@ -162,30 +163,33 @@ export class OrchestrationClient extends BaseChatModel< if (!chunk.data) { continue; } - const delta = chunk.data.orchestration_result?.choices[0]?.delta; if (!delta) { continue; } + const newTokenIndices = { + // Indicates the token is part of the first prompt and first completion + prompt: 0, + completion: chunk.data.orchestration_result?.choices[0]?.index ?? 0 + }; const messageChunk = mapOrchestrationChunkToLangChainMessageChunk(chunk); - - const finishReason = chunk.getFinishReason(); - const tokenUsage = chunk.getTokenUsage(); + const finishReason = response.getFinishReason(); + const tokenUsage = response.getTokenUsage(); setFinishReason(messageChunk, finishReason); - setUsageMetadata(messageChunk, tokenUsage); + setTokenUsage(messageChunk, tokenUsage); + const generationInfo: Record = { ...newTokenIndices }; const content = chunk.getDeltaContent() ?? ''; const generationChunk = new ChatGenerationChunk({ message: messageChunk, - text: content + text: content, + generationInfo }); + // Notify the run manager about the new token, some parameters are undefined as they are implicitly read from the context. await runManager?.handleLLMNewToken( content, - { - prompt: 0, - completion: 0 - }, + newTokenIndices, undefined, undefined, undefined, diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 4941318f5..e456dc911 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -204,14 +204,13 @@ export function mapOutputToChatResult( export function mapOrchestrationChunkToLangChainMessageChunk( chunk: OrchestrationStreamChunkResponse ): AIMessageChunk { - const { module_results, request_id, orchestration_result } = chunk.data; + const { module_results, request_id } = chunk.data; const content = chunk.getDeltaContent() ?? ''; const toolCallChunks = chunk.getDeltaToolCallChunks(); const additional_kwargs: Record = { module_results, - request_id, - orchestration_result + request_id }; let tool_call_chunks: ToolCallChunk[] = []; @@ -243,7 +242,7 @@ export function setFinishReason( * @param tokenUsage - The token usage information. * @internal */ -export function setUsageMetadata( +export function setTokenUsage( messageChunk: AIMessageChunk, tokenUsage: TokenUsage | undefined ): void { @@ -253,5 +252,6 @@ export function setUsageMetadata( output_tokens: tokenUsage.completion_tokens, total_tokens: tokenUsage.total_tokens }; + messageChunk.response_metadata.token_usage = tokenUsage; } } diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index fcf31e42a..9fd904db4 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -196,12 +196,11 @@ export async function invokeLangGraphChain(): Promise { export async function streamOrchestrationLangChain( controller = new AbortController() ): Promise> { + // Todo Remove template and use messages after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 const orchestrationConfig: LangchainOrchestrationModuleConfig = { - // define the language model to be used llm: { model_name: 'gpt-4o' }, - // define the template templating: { template: [ { @@ -213,22 +212,12 @@ export async function streamOrchestrationLangChain( }; const client = new OrchestrationClient(orchestrationConfig); - - // Return the stream - return client.stream( - [ - { - role: 'user', - content: 'I need information about a topic.' - } - ], - { - inputParams: { - topic: 'SAP Cloud SDK and its capabilities' - }, - signal: controller.signal - } - ); + return client.stream([], { + inputParams: { + topic: 'SAP Cloud SDK and its capabilities' + }, + signal: controller.signal + }); } /** From a6cb6d722c5997e7e2f12810bf40c6883cd90383 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Sun, 18 May 2025 17:30:15 +0200 Subject: [PATCH 15/67] chore: simplify function --- .../langchain/src/orchestration/client.ts | 15 +++++------- packages/langchain/src/orchestration/util.ts | 24 ++++++++++++++++--- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index aa525c991..0ef376e88 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -8,7 +8,8 @@ import { setFinishReason, setTokenUsage, mapLangchainMessagesToOrchestrationMessages, - mapOutputToChatResult + mapOutputToChatResult, + computeTokenIndices } from './util.js'; import type { OrchestrationMessageChunk } from './orchestration-message-chunk.js'; import type { BaseLanguageModelInput } from '@langchain/core/language_models/base'; @@ -167,29 +168,25 @@ export class OrchestrationClient extends BaseChatModel< if (!delta) { continue; } - const newTokenIndices = { - // Indicates the token is part of the first prompt and first completion - prompt: 0, - completion: chunk.data.orchestration_result?.choices[0]?.index ?? 0 - }; + const tokenIndices = computeTokenIndices(chunk); const messageChunk = mapOrchestrationChunkToLangChainMessageChunk(chunk); const finishReason = response.getFinishReason(); const tokenUsage = response.getTokenUsage(); setFinishReason(messageChunk, finishReason); setTokenUsage(messageChunk, tokenUsage); - const generationInfo: Record = { ...newTokenIndices }; const content = chunk.getDeltaContent() ?? ''; + const generationChunk = new ChatGenerationChunk({ message: messageChunk, text: content, - generationInfo + generationInfo: { ...tokenIndices } }); // Notify the run manager about the new token, some parameters are undefined as they are implicitly read from the context. await runManager?.handleLLMNewToken( content, - newTokenIndices, + tokenIndices, undefined, undefined, undefined, diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index e456dc911..806cb8b06 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -222,8 +222,8 @@ export function mapOrchestrationChunkToLangChainMessageChunk( } /** - * Sets finish reason on a message chunk if available. - * @param messageChunk - The message chunk to update. + * Sets finish reason on a LangChain message chunk if available. + * @param messageChunk - The LangChain message chunk to update. * @param finishReason - The finish reason from the response. * @internal */ @@ -238,7 +238,7 @@ export function setFinishReason( /** * Sets usage metadata on a message chunk if available. - * @param messageChunk - The message chunk to update. + * @param messageChunk - The LangChain message chunk to update. * @param tokenUsage - The token usage information. * @internal */ @@ -255,3 +255,21 @@ export function setTokenUsage( messageChunk.response_metadata.token_usage = tokenUsage; } } + +/** + * Computes token indices for a chunk of the stream response. + * @param chunk - A chunk of the stream response. + * @returns An object with prompt and completion indices. + * @internal + */ +export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { + prompt: number; + completion: number; +} { + return { + // Indicates the token is part of the first prompt + prompt: 0, + // Use the choice index from the response or default to 0 + completion: chunk.data.orchestration_result?.choices[0]?.index ?? 0 + }; +} From aad9a05602f43a223ccdab5c64ca3787ebdf5499 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Sun, 18 May 2025 17:41:28 +0200 Subject: [PATCH 16/67] chore: review comment --- sample-code/src/index.ts | 2 +- sample-code/src/langchain-orchestration.ts | 2 +- sample-code/src/server.ts | 12 ++---------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/sample-code/src/index.ts b/sample-code/src/index.ts index 4d55e5ac5..8c414f286 100644 --- a/sample-code/src/index.ts +++ b/sample-code/src/index.ts @@ -32,7 +32,7 @@ export { export { invokeChain as orchestrationInvokeChain, invokeLangGraphChain, - streamOrchestrationLangChain + orchestrationStreamChain } from './langchain-orchestration.js'; export { getDeployments, diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 9fd904db4..0fd30e75c 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -193,7 +193,7 @@ export async function invokeLangGraphChain(): Promise { * @param controller - The abort controller to cancel the request if needed. * @returns An async iterable of AIMessageChunk objects. */ -export async function streamOrchestrationLangChain( +export async function orchestrationStreamChain( controller = new AbortController() ): Promise> { // Todo Remove template and use messages after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index 93c93e8bc..85b7bc53b 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -49,7 +49,7 @@ import { invokeChainWithOutputFilter as invokeChainWithOutputFilterOrchestration, invokeLangGraphChain, invokeChainWithMasking, - streamOrchestrationLangChain + orchestrationStreamChain } from './langchain-orchestration.js'; import { createCollection, @@ -470,28 +470,22 @@ app.get('/langchain/invoke-stateful-chain', async (req, res) => { app.get('/langchain/stream-orchestration', async (req, res) => { const controller = new AbortController(); try { - const stream = await streamOrchestrationLangChain(controller); - - // Set headers for event stream. + const stream = await orchestrationStreamChain(controller); res.setHeader('Content-Type', 'text/event-stream'); res.setHeader('Connection', 'keep-alive'); res.flushHeaders(); let connectionAlive = true; - - // Abort the stream if the client connection is closed. res.on('close', () => { controller.abort(); connectionAlive = false; res.end(); }); - // Stream the delta content. for await (const chunk of stream) { if (!connectionAlive) { break; } - // Use the content property similar to getDeltaContent() res.write(chunk.content + '\n'); if (connectionAlive && chunk.usage_metadata) { res.write('\n\n---------------------------\n'); @@ -506,8 +500,6 @@ app.get('/langchain/stream-orchestration', async (req, res) => { res.write(` - Total tokens: ${chunk.usage_metadata?.total_tokens}\n`); } } - - // Write the finish reason and token usage after the stream ends. } catch (error: any) { sendError(res, error, false); } finally { From 25a4a79d3712908cfcb30286a39b75d6995f726a Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Sun, 18 May 2025 17:55:31 +0200 Subject: [PATCH 17/67] chore: simplify --- packages/langchain/src/orchestration/client.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 0ef376e88..28551288a 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -161,15 +161,8 @@ export class OrchestrationClient extends BaseChatModel< ); for await (const chunk of response.stream) { - if (!chunk.data) { - continue; - } - const delta = chunk.data.orchestration_result?.choices[0]?.delta; - if (!delta) { - continue; - } - const tokenIndices = computeTokenIndices(chunk); const messageChunk = mapOrchestrationChunkToLangChainMessageChunk(chunk); + const tokenIndices = computeTokenIndices(chunk); const finishReason = response.getFinishReason(); const tokenUsage = response.getTokenUsage(); From c72f7423ab1c47858c9401049b44a20d2724d7bf Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Sun, 18 May 2025 19:20:40 +0200 Subject: [PATCH 18/67] chore: remove e2e test --- .../src/orchestration-langchain.test.ts | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/tests/e2e-tests/src/orchestration-langchain.test.ts b/tests/e2e-tests/src/orchestration-langchain.test.ts index 62c500363..96ac1ea6c 100644 --- a/tests/e2e-tests/src/orchestration-langchain.test.ts +++ b/tests/e2e-tests/src/orchestration-langchain.test.ts @@ -1,7 +1,6 @@ import { orchestrationInvokeChain, - invokeLangGraphChain, - streamOrchestrationLangChain + invokeLangGraphChain } from '@sap-ai-sdk/sample-code'; import { loadEnv } from './utils/load-env.js'; @@ -17,25 +16,4 @@ describe('Orchestration LangChain client', () => { const result = await invokeLangGraphChain(); expect(result).toContain('SAP Cloud SDK'); }); - - it('supports streaming responses', async () => { - // Create an abort controller for the test - const controller = new AbortController(); - - // Get the stream - const stream = await streamOrchestrationLangChain(controller); - - // Collect all chunks - const chunks = []; - for await (const chunk of stream) { - chunks.push(chunk); - } - - // Verify we received chunks - expect(chunks.length).toBeGreaterThan(0); - - // Verify the chunks contain expected content - const combinedContent = chunks.map(chunk => chunk.content).join(''); - expect(combinedContent).toContain('SAP Cloud SDK'); - }); }); From 2a3d8108eb24a62ce7e2042fc33d51cad5368e29 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Sun, 18 May 2025 20:49:04 +0200 Subject: [PATCH 19/67] chore: add tests, minor changes --- .../src/orchestration/client.test.ts | 108 +++++++++++++----- .../langchain/src/orchestration/client.ts | 18 ++- 2 files changed, 85 insertions(+), 41 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index cffe86e7c..b49126c43 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -21,6 +21,7 @@ jest.setTimeout(30000); describe('orchestration service client', () => { let mockResponse: CompletionPostResponse; let mockResponseInputFilterError: ErrorResponse; + let mockResponseStream: string; beforeEach(async () => { mockClientCredentialsGrantCall(); mockDeploymentsList({ scenarioId: 'orchestration' }, { id: '1234' }); @@ -32,6 +33,10 @@ describe('orchestration service client', () => { 'orchestration', 'orchestration-chat-completion-input-filter-error.json' ); + mockResponseStream = await parseFileToString( + 'orchestration', + 'orchestration-chat-completion-stream-chunks.txt' + ); }); afterEach(() => { @@ -44,11 +49,16 @@ describe('orchestration service client', () => { retry?: number; delay?: number; }, - status: number = 200 + status: number = 200, + isStream: boolean = false ) { mockInference( { - data: constructCompletionPostRequest(config, { messagesHistory: [] }) + data: constructCompletionPostRequest( + config, + { messagesHistory: [] }, + isStream + ) }, { data: response, @@ -61,6 +71,17 @@ describe('orchestration service client', () => { ); } + function mockStreamInferenceWithResilience( + response: any = mockResponseStream, + resilience: { + retry?: number; + delay?: number; + } = { retry: 0 }, + status: number = 200 + ) { + mockInferenceWithResilience(response, resilience, status, true); + } + const config: LangchainOrchestrationModuleConfig = { llm: { model_name: 'gpt-4o', @@ -133,46 +154,71 @@ describe('orchestration service client', () => { }, 1000); it('supports streaming responses', async () => { - // Load the mock streaming response - const streamMockResponse = await parseFileToString( - 'orchestration', - 'orchestration-chat-completion-stream-chunks.txt' - ); + mockStreamInferenceWithResilience(); - // Mock the streaming API call - mockInference( - { - data: constructCompletionPostRequest( - config, - { messagesHistory: [] }, - true - ) - }, - { - data: streamMockResponse, - status: 200 - }, - { - url: 'inference/deployments/1234/completion' + const client = new OrchestrationClient(config); + const stream = await client.stream([]); + let iterations = 0; + const maxIterations = 2; + let intermediateChunk: AIMessageChunk | undefined; + + for await (const chunk of stream) { + iterations++; + intermediateChunk = !intermediateChunk + ? chunk + : intermediateChunk.concat(chunk); + if (iterations >= maxIterations) { + break; } + } + expect(intermediateChunk).toBeDefined(); + expect(intermediateChunk!.content).toBeDefined(); + expect(intermediateChunk!.content).toEqual( + 'The SAP Cloud SDK is a comprehensive development toolkit designed to simplify and accelerate the cre' ); + }); + it('test streaming with abort signal', async () => { + mockStreamInferenceWithResilience(); const client = new OrchestrationClient(config); + const controller = new AbortController(); + const { signal } = controller; + const stream = await client.stream([], { signal }); + const streamPromise = async () => { + for await (const _chunk of stream) { + controller.abort(); + } + }; + + await expect(streamPromise()).rejects.toThrow(); + }, 1000); - // Test the stream method + it('test streaming with callbacks', async () => { + mockStreamInferenceWithResilience(); + let tokenCount = 0; + const callbackHandler = { + handleLLMNewToken: jest.fn().mockImplementation(() => { + tokenCount += 1; + }) + }; + const client = new OrchestrationClient(config, { + callbacks: [callbackHandler] + }); const stream = await client.stream([]); + const chunks = []; - // Collect all chunks - const chunks: AIMessageChunk[] = []; for await (const chunk of stream) { chunks.push(chunk); + break; } - - // Verify we received chunks + expect(callbackHandler.handleLLMNewToken).toHaveBeenCalled(); + const firstCallArgs = callbackHandler.handleLLMNewToken.mock.calls[0]; + expect(firstCallArgs).toBeDefined(); + // First chunk content is empty + expect(firstCallArgs[0]).toEqual(''); + // Second argument should be the token indices + expect(firstCallArgs[1]).toEqual({ prompt: 0, completion: 0 }); + expect(tokenCount).toBeGreaterThan(0); expect(chunks.length).toBeGreaterThan(0); - - // Verify the chunks are of the expected type - expect(chunks[0]).toBeDefined(); - expect(chunks[0].content).toBeDefined(); }); }); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 28551288a..9a77ebf7d 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -148,16 +148,14 @@ export class OrchestrationClient extends BaseChatModel< this.destination ); - const response = await this.caller.callWithOptions( - { signal: options.signal }, - () => - orchestrationClient.stream( - // Todo Adapt messagesHistory after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 - { messagesHistory: orchestrationMessages, inputParams }, - controller, - options.streamOptions, - customRequestConfig - ) + const response = await this.caller.call(() => + orchestrationClient.stream( + // Todo Adapt messagesHistory after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 + { messagesHistory: orchestrationMessages, inputParams }, + controller, + options.streamOptions, + customRequestConfig + ) ); for await (const chunk of response.stream) { From f5c8473a3ac374ba4e21acb2d728f499c88dc47c Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 09:10:18 +0200 Subject: [PATCH 20/67] chore: additional test for utility functions --- .../langchain/src/orchestration/util.test.ts | 92 ++++++++++++++++++- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/packages/langchain/src/orchestration/util.test.ts b/packages/langchain/src/orchestration/util.test.ts index f871db8d1..51f19c14f 100644 --- a/packages/langchain/src/orchestration/util.test.ts +++ b/packages/langchain/src/orchestration/util.test.ts @@ -1,17 +1,23 @@ import { AIMessage, + AIMessageChunk, HumanMessage, SystemMessage, ToolMessage } from '@langchain/core/messages'; +import { OrchestrationStreamChunkResponse } from '@sap-ai-sdk/orchestration'; import { mapLangchainMessagesToOrchestrationMessages, - mapOutputToChatResult + mapOutputToChatResult, + setFinishReason, + setTokenUsage, + computeTokenIndices } from './util.js'; import type { OrchestrationMessage } from './orchestration-message.js'; import type { CompletionPostResponse, - ResponseMessageToolCall + MessageToolCall, + TokenUsage } from '@sap-ai-sdk/orchestration'; describe('mapLangchainMessagesToOrchestrationMessages', () => { @@ -150,7 +156,7 @@ describe('mapOutputToChatResult', () => { }); it('should map tool_calls correctly', () => { - const toolCallData: ResponseMessageToolCall = { + const toolCallData: MessageToolCall = { id: 'call-123', type: 'function', function: { @@ -200,3 +206,83 @@ describe('mapOutputToChatResult', () => { ]); }); }); + +describe('setFinishReason', () => { + it('should set finish reason on message chunk when provided', () => { + const messageChunk = new AIMessageChunk({ content: 'Test content' }); + + setFinishReason(messageChunk, 'stop'); + + expect(messageChunk.response_metadata.finish_reason).toBe('stop'); + }); + + it('should not modify response_metadata when finish reason is falsy', () => { + const messageChunk = new AIMessageChunk({ content: 'Test content' }); + const originalMetadata = { ...messageChunk.response_metadata }; + + setFinishReason(messageChunk, ''); + + expect(messageChunk.response_metadata).toEqual(originalMetadata); + }); +}); + +describe('setTokenUsage', () => { + it('should set token usage metadata when provided', () => { + const messageChunk = new AIMessageChunk({ content: 'Test content' }); + const tokenUsage: TokenUsage = { + completion_tokens: 10, + prompt_tokens: 20, + total_tokens: 30 + }; + + setTokenUsage(messageChunk, tokenUsage); + + expect(messageChunk.usage_metadata).toEqual({ + input_tokens: 20, + output_tokens: 10, + total_tokens: 30 + }); + expect(messageChunk.response_metadata.token_usage).toEqual(tokenUsage); + }); + + it('should not modify message chunk when token usage is undefined', () => { + const messageChunk = new AIMessageChunk({ + content: 'Test content', + response_metadata: { finish_reason: 'stop' } + }); + + setTokenUsage(messageChunk, undefined); + + expect(messageChunk.usage_metadata).toBeUndefined(); + expect(messageChunk.response_metadata.token_usage).toBeUndefined(); + expect(messageChunk.response_metadata.finish_reason).toBe('stop'); + }); +}); + +describe('computeTokenIndices', () => { + it('should compute token indices with choice index from chunk', () => { + const mockChunk = new OrchestrationStreamChunkResponse({ + request_id: 'req-123', + orchestration_result: { + id: 'test-id', + object: 'chat.completion.chunk', + created: 1634840000, + model: 'test-model', + system_fingerprint: 'fp_123', + choices: [ + { + index: 0, + delta: { content: 'Test' } + } + ] + } + }); + + const result = computeTokenIndices(mockChunk); + + expect(result).toEqual({ + prompt: 0, + completion: 0 + }); + }); +}); From fd331effa51a98cc7e0ae580ffab31c7a8cfec33 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 09:42:04 +0200 Subject: [PATCH 21/67] chore: add test for tool calls --- .../src/orchestration/client.test.ts | 26 +++++++++++++++++++ ...at-completion-stream-chunks-tool-calls.txt | 13 ++++++++++ 2 files changed, 39 insertions(+) create mode 100644 test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index b49126c43..028301d14 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -9,6 +9,7 @@ import { parseFileToString } from '../../../../test-util/mock-http.js'; import { OrchestrationClient } from './client.js'; +import type { ToolCall } from '@langchain/core/messages/tool'; import type { LangchainOrchestrationModuleConfig } from './types.js'; import type { CompletionPostResponse, @@ -22,6 +23,7 @@ describe('orchestration service client', () => { let mockResponse: CompletionPostResponse; let mockResponseInputFilterError: ErrorResponse; let mockResponseStream: string; + let mockResponseStreamToolCalls: string; beforeEach(async () => { mockClientCredentialsGrantCall(); mockDeploymentsList({ scenarioId: 'orchestration' }, { id: '1234' }); @@ -37,6 +39,10 @@ describe('orchestration service client', () => { 'orchestration', 'orchestration-chat-completion-stream-chunks.txt' ); + mockResponseStreamToolCalls = await parseFileToString( + 'orchestration', + 'orchestration-chat-completion-stream-chunks-tool-calls.txt' + ); }); afterEach(() => { @@ -221,4 +227,24 @@ describe('orchestration service client', () => { expect(tokenCount).toBeGreaterThan(0); expect(chunks.length).toBeGreaterThan(0); }); + + it('supports streaming responses with tool calls', async () => { + mockStreamInferenceWithResilience(mockResponseStreamToolCalls); + + const client = new OrchestrationClient(config); + const stream = await client.stream([]); + + let finalOutput: AIMessageChunk | undefined; + for await (const chunk of stream) { + finalOutput = !finalOutput ? chunk : finalOutput.concat(chunk); + } + + expect(finalOutput).toBeDefined(); + expect(finalOutput!.tool_call_chunks).toBeDefined(); + expect(finalOutput!.tool_calls).toBeDefined(); + + const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; + expect(completeToolCall!.name).toEqual('convert_temperature_to_fahrenheit'); + expect(completeToolCall!.args).toEqual({ temperature: 20 }); + }); }); diff --git a/test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt b/test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt new file mode 100644 index 000000000..5d643460e --- /dev/null +++ b/test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt @@ -0,0 +1,13 @@ +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"templating": [{"content": "Convert 20 degrees Celsius to Fahrenheit.", "role": "user"}]}, "orchestration_result": {"id": "", "object": "", "created": 0, "model": "", "system_fingerprint": "", "choices": [{"index": 0, "delta": {"content": ""}}]}} + +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", "type": "function", "function": {"name": "convert_temperature_to_fahrenheit", "arguments": ""}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", "type": "function", "function": {"name": "convert_temperature_to_fahrenheit", "arguments": ""}}]}}]}} + +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "{\""}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "{\""}}]}}]}} + +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "temperature"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "temperature"}}]}}]}} + +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "\":"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "\":"}}]}}]}} + +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "20"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "20"}}]}}]}} + +data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "}"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "}"}}]}}]}} \ No newline at end of file From 0be794448f555fdd9a8065c9cc52ca54b7a5a329 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 10:05:53 +0200 Subject: [PATCH 22/67] chore: add tests for mapping function --- .../__snapshots__/util.test.ts.snap | 45 +++++++ .../langchain/src/orchestration/util.test.ts | 115 +++++++++++++++++- packages/langchain/src/orchestration/util.ts | 1 - 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap diff --git a/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap new file mode 100644 index 000000000..bb9c74b8e --- /dev/null +++ b/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap @@ -0,0 +1,45 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`mapOrchestrationChunkToLangChainMessageChunk should map a chunk with content to AIMessageChunk: AIMessageChunk with content 1`] = ` +{ + "id": [ + "langchain_core", + "messages", + "AIMessageChunk", + ], + "kwargs": { + "additional_kwargs": { + "module_results": { + "llm": { + "choices": [ + { + "delta": { + "content": "Test content", + "role": "assistant", + "tool_calls": undefined, + }, + "finish_reason": "", + "index": 0, + }, + ], + "created": 1634840000, + "id": "test-id", + "model": "test-model", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_123", + "usage": undefined, + }, + }, + "request_id": "req-123", + }, + "content": "Test content", + "invalid_tool_calls": [], + "response_metadata": {}, + "tool_call_chunks": [], + "tool_calls": [], + "usage_metadata": undefined, + }, + "lc": 1, + "type": "constructor", +} +`; diff --git a/packages/langchain/src/orchestration/util.test.ts b/packages/langchain/src/orchestration/util.test.ts index 51f19c14f..a1b3ccbdc 100644 --- a/packages/langchain/src/orchestration/util.test.ts +++ b/packages/langchain/src/orchestration/util.test.ts @@ -6,18 +6,23 @@ import { ToolMessage } from '@langchain/core/messages'; import { OrchestrationStreamChunkResponse } from '@sap-ai-sdk/orchestration'; +import { jest } from '@jest/globals'; import { mapLangchainMessagesToOrchestrationMessages, mapOutputToChatResult, setFinishReason, setTokenUsage, - computeTokenIndices + computeTokenIndices, + mapOrchestrationChunkToLangChainMessageChunk } from './util.js'; import type { OrchestrationMessage } from './orchestration-message.js'; +import type { ToolCallChunk } from '@langchain/core/messages/tool'; import type { CompletionPostResponse, MessageToolCall, - TokenUsage + TokenUsage, + ToolCallChunk as OrchestrationToolCallChunk, + CompletionPostResponseStreaming } from '@sap-ai-sdk/orchestration'; describe('mapLangchainMessagesToOrchestrationMessages', () => { @@ -286,3 +291,109 @@ describe('computeTokenIndices', () => { }); }); }); + +describe('mapOrchestrationChunkToLangChainMessageChunk', () => { + function createMockChunk( + content?: string, + toolCallChunks?: OrchestrationToolCallChunk[], + finishReason?: string, + tokenUsage?: { + completion_tokens: number; + prompt_tokens: number; + total_tokens: number; + } + ): OrchestrationStreamChunkResponse { + const mockData: CompletionPostResponseStreaming = { + request_id: 'req-123', + module_results: { + llm: { + id: 'test-id', + object: 'chat.completion.chunk', + created: 1634840000, + model: 'test-model', + system_fingerprint: 'fp_123', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + content: content || '', + tool_calls: toolCallChunks + }, + finish_reason: finishReason || '' + } + ], + usage: tokenUsage + } + }, + orchestration_result: { + id: 'test-id', + object: 'chat.completion.chunk', + created: 1634840000, + model: 'test-model', + system_fingerprint: 'fp_123', + choices: [ + { + index: 0, + delta: { + role: 'assistant', + content: content || '', + tool_calls: toolCallChunks + }, + finish_reason: finishReason || '' + } + ], + usage: tokenUsage + } + }; + + const mockChunk = new OrchestrationStreamChunkResponse(mockData); + jest.spyOn(mockChunk, 'getDeltaContent').mockReturnValue(content); + jest + .spyOn(mockChunk, 'getDeltaToolCallChunks') + .mockReturnValue(toolCallChunks); + + return mockChunk; + } + + it('should map a chunk with content to AIMessageChunk', () => { + const mockChunk = createMockChunk('Test content'); + const result = mapOrchestrationChunkToLangChainMessageChunk(mockChunk); + + expect(result).toBeInstanceOf(AIMessageChunk); + expect(result.content).toBe('Test content'); + expect(result.additional_kwargs).toEqual({ + module_results: mockChunk.data.module_results, + request_id: 'req-123' + }); + expect(result.tool_call_chunks).toEqual([]); + expect(result).toMatchSnapshot('AIMessageChunk with content'); + }); + + it('should map a chunk with tool call chunks to AIMessageChunk', () => { + const toolCallChunk: OrchestrationToolCallChunk = { + id: 'call-123', + index: 0, + type: 'function', + function: { + name: 'test_function', + arguments: '' + } + }; + const mockChunk = createMockChunk('', [toolCallChunk]); + const result = mapOrchestrationChunkToLangChainMessageChunk(mockChunk); + + expect(result).toBeInstanceOf(AIMessageChunk); + expect(result.content).toBe(''); + expect(result.tool_call_chunks).toHaveLength(1); + const expectedToolCallChunk: ToolCallChunk = { + id: 'call-123', + index: 0, + name: 'test_function', + args: '{"arg1":"value1"}', + type: 'tool_call_chunk' + }; + expect(result.tool_call_chunks?.[0]).toEqual(expectedToolCallChunk); + expect(result).toMatchSnapshot('AIMessageChunk with tool call chunks'); + }); +}); diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 806cb8b06..2d5a84997 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -269,7 +269,6 @@ export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { return { // Indicates the token is part of the first prompt prompt: 0, - // Use the choice index from the response or default to 0 completion: chunk.data.orchestration_result?.choices[0]?.index ?? 0 }; } From ffd24568de60f1bf73f79d3870a06bc709a22329 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 10:13:08 +0200 Subject: [PATCH 23/67] chore: fix failing test --- .../__snapshots__/util.test.ts.snap | 69 +++++++++++++++++++ .../langchain/src/orchestration/util.test.ts | 2 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap index bb9c74b8e..001eb7370 100644 --- a/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap +++ b/packages/langchain/src/orchestration/__snapshots__/util.test.ts.snap @@ -43,3 +43,72 @@ exports[`mapOrchestrationChunkToLangChainMessageChunk should map a chunk with co "type": "constructor", } `; + +exports[`mapOrchestrationChunkToLangChainMessageChunk should map a chunk with tool call chunks to AIMessageChunk: AIMessageChunk with tool call chunks 1`] = ` +{ + "id": [ + "langchain_core", + "messages", + "AIMessageChunk", + ], + "kwargs": { + "additional_kwargs": { + "module_results": { + "llm": { + "choices": [ + { + "delta": { + "content": "", + "role": "assistant", + "tool_calls": [ + { + "function": { + "arguments": "", + "name": "test_function", + }, + "id": "call-123", + "index": 0, + "type": "function", + }, + ], + }, + "finish_reason": "", + "index": 0, + }, + ], + "created": 1634840000, + "id": "test-id", + "model": "test-model", + "object": "chat.completion.chunk", + "system_fingerprint": "fp_123", + "usage": undefined, + }, + }, + "request_id": "req-123", + }, + "content": "", + "invalid_tool_calls": [], + "response_metadata": {}, + "tool_call_chunks": [ + { + "args": "", + "id": "call-123", + "index": 0, + "name": "test_function", + "type": "tool_call_chunk", + }, + ], + "tool_calls": [ + { + "args": {}, + "id": "call-123", + "name": "test_function", + "type": "tool_call", + }, + ], + "usage_metadata": undefined, + }, + "lc": 1, + "type": "constructor", +} +`; diff --git a/packages/langchain/src/orchestration/util.test.ts b/packages/langchain/src/orchestration/util.test.ts index a1b3ccbdc..bf70cb694 100644 --- a/packages/langchain/src/orchestration/util.test.ts +++ b/packages/langchain/src/orchestration/util.test.ts @@ -390,7 +390,7 @@ describe('mapOrchestrationChunkToLangChainMessageChunk', () => { id: 'call-123', index: 0, name: 'test_function', - args: '{"arg1":"value1"}', + args: '', type: 'tool_call_chunk' }; expect(result.tool_call_chunks?.[0]).toEqual(expectedToolCallChunk); From 364ae6b19c9d8c5700db26d67a6a6748ee49b4b1 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 10:29:18 +0200 Subject: [PATCH 24/67] chore: refactor test names --- packages/langchain/src/orchestration/client.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 028301d14..56a6bb605 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -184,7 +184,7 @@ describe('orchestration service client', () => { ); }); - it('test streaming with abort signal', async () => { + it('streaming with abort signal', async () => { mockStreamInferenceWithResilience(); const client = new OrchestrationClient(config); const controller = new AbortController(); @@ -199,7 +199,7 @@ describe('orchestration service client', () => { await expect(streamPromise()).rejects.toThrow(); }, 1000); - it('test streaming with callbacks', async () => { + it('streaming with callbacks', async () => { mockStreamInferenceWithResilience(); let tokenCount = 0; const callbackHandler = { From 53666d96800da7026182dc33b2bffe7cfe2b83b5 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Mon, 19 May 2025 14:27:11 +0200 Subject: [PATCH 25/67] Update packages/langchain/src/orchestration/client.ts Co-authored-by: Zhongpin Wang --- packages/langchain/src/orchestration/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 9a77ebf7d..915f86d7a 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -150,7 +150,7 @@ export class OrchestrationClient extends BaseChatModel< const response = await this.caller.call(() => orchestrationClient.stream( - // Todo Adapt messagesHistory after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 + // TODO: Adapt messagesHistory after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 { messagesHistory: orchestrationMessages, inputParams }, controller, options.streamOptions, From a6900991f216984eb7dd508a05a0ff2b8025ac69 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 16:08:49 +0200 Subject: [PATCH 26/67] chore: adapt name --- packages/langchain/src/orchestration/client.ts | 6 +++--- packages/langchain/src/orchestration/util.test.ts | 14 +++++++------- packages/langchain/src/orchestration/util.ts | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 915f86d7a..67cee3592 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -7,7 +7,7 @@ import { isTemplate, setFinishReason, setTokenUsage, - mapLangchainMessagesToOrchestrationMessages, + mapLangChainMessagesToOrchestrationMessages, mapOutputToChatResult, computeTokenIndices } from './util.js'; @@ -99,7 +99,7 @@ export class OrchestrationClient extends BaseChatModel< this.destination ); const messagesHistory = - mapLangchainMessagesToOrchestrationMessages(messages); + mapLangChainMessagesToOrchestrationMessages(messages); return orchestrationClient.chatCompletion( { messagesHistory, @@ -137,7 +137,7 @@ export class OrchestrationClient extends BaseChatModel< } const orchestrationMessages = - mapLangchainMessagesToOrchestrationMessages(messages); + mapLangChainMessagesToOrchestrationMessages(messages); const { inputParams, customRequestConfig } = options; const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); diff --git a/packages/langchain/src/orchestration/util.test.ts b/packages/langchain/src/orchestration/util.test.ts index bf70cb694..cd6112ad8 100644 --- a/packages/langchain/src/orchestration/util.test.ts +++ b/packages/langchain/src/orchestration/util.test.ts @@ -8,7 +8,7 @@ import { import { OrchestrationStreamChunkResponse } from '@sap-ai-sdk/orchestration'; import { jest } from '@jest/globals'; import { - mapLangchainMessagesToOrchestrationMessages, + mapLangChainMessagesToOrchestrationMessages, mapOutputToChatResult, setFinishReason, setTokenUsage, @@ -34,7 +34,7 @@ describe('mapLangchainMessagesToOrchestrationMessages', () => { ]; const result = - mapLangchainMessagesToOrchestrationMessages(langchainMessages); + mapLangChainMessagesToOrchestrationMessages(langchainMessages); expect(result).toEqual([ { role: 'system', content: 'System message content' }, @@ -49,7 +49,7 @@ describe('mapLangchainMessagesToOrchestrationMessages', () => { ]; expect(() => - mapLangchainMessagesToOrchestrationMessages(langchainMessages) + mapLangChainMessagesToOrchestrationMessages(langchainMessages) ).toThrow('Unsupported message type: tool'); }); }); @@ -59,7 +59,7 @@ describe('mapBaseMessageToChatMessage', () => { const humanMessage = new HumanMessage('Human message content'); // Since mapBaseMessageToChatMessage is internal, we'll test it through mapLangchainMessagesToOrchestrationMessages - const result = mapLangchainMessagesToOrchestrationMessages([humanMessage]); + const result = mapLangChainMessagesToOrchestrationMessages([humanMessage]); expect(result[0]).toEqual({ role: 'user', @@ -70,7 +70,7 @@ describe('mapBaseMessageToChatMessage', () => { it('should map SystemMessage to ChatMessage with system role', () => { const systemMessage = new SystemMessage('System message content'); - const result = mapLangchainMessagesToOrchestrationMessages([systemMessage]); + const result = mapLangChainMessagesToOrchestrationMessages([systemMessage]); expect(result[0]).toEqual({ role: 'system', @@ -81,7 +81,7 @@ describe('mapBaseMessageToChatMessage', () => { it('should map AIMessage to ChatMessage with assistant role', () => { const aiMessage = new AIMessage('AI message content'); - const result = mapLangchainMessagesToOrchestrationMessages([aiMessage]); + const result = mapLangChainMessagesToOrchestrationMessages([aiMessage]); expect(result[0]).toEqual({ role: 'assistant', @@ -101,7 +101,7 @@ describe('mapBaseMessageToChatMessage', () => { }); expect(() => - mapLangchainMessagesToOrchestrationMessages([systemMessage]) + mapLangChainMessagesToOrchestrationMessages([systemMessage]) ).toThrow( 'System messages with image URLs are not supported by the Orchestration Client.' ); diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 2d5a84997..d7acb6270 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -102,7 +102,7 @@ function mapSystemMessageToAzureOpenAiSystemMessage( * @returns The orchestration messages mapped from LangChain messages. * @internal */ -export function mapLangchainMessagesToOrchestrationMessages( +export function mapLangChainMessagesToOrchestrationMessages( messages: BaseMessage[] ): ChatMessage[] { return messages.map(mapBaseMessageToChatMessage); From c24f9f52db233c540fcafa45d20515884e403c30 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Mon, 19 May 2025 16:50:45 +0200 Subject: [PATCH 27/67] chore: review changes --- packages/langchain/src/orchestration/util.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index d7acb6270..d9ec8bee7 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -131,7 +131,7 @@ function mapAzureOpenAiToLangchainToolCall( * @param toolCallChunks - The {@link OrchestrationToolCallChunk} in a stream response chunk. * @returns An array of LangChain {@link ToolCallChunk}. */ -function mapOrchestrationToLangchainToolCallChunk( +function mapOrchestrationToLangChainToolCallChunk( toolCallChunks: OrchestrationToolCallChunk[] ): ToolCallChunk[] { return toolCallChunks.map(chunk => ({ @@ -214,10 +214,11 @@ export function mapOrchestrationChunkToLangChainMessageChunk( }; let tool_call_chunks: ToolCallChunk[] = []; - if (Array.isArray(toolCallChunks)) { - tool_call_chunks = mapOrchestrationToLangchainToolCallChunk(toolCallChunks); + if (toolCallChunks) { + tool_call_chunks = mapOrchestrationToLangChainToolCallChunk(toolCallChunks); } - // Use AIMessageChunk to represent message chunks for roles like 'tool' and 'user' too + // Use `AIMessageChunk` to represent message chunks for roles such as 'tool' and 'user' as well. + // While the `ChatDelta` type can accommodate other roles in the orchestration service's stream chunk response, in realtime, we only expect messages with the 'assistant' role to be returned. return new AIMessageChunk({ content, additional_kwargs, tool_call_chunks }); } From acecf11712bc8a972e6b96915dc4155730737f27 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Mon, 19 May 2025 16:51:48 +0200 Subject: [PATCH 28/67] Update sample-code/src/langchain-orchestration.ts Co-authored-by: Zhongpin Wang --- sample-code/src/langchain-orchestration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 0fd30e75c..feb6b0b61 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -189,7 +189,7 @@ export async function invokeLangGraphChain(): Promise { } /** - * Stream responses from the OrchestrationClient using LangChain. + * Stream responses using LangChain Orchestration client. * @param controller - The abort controller to cancel the request if needed. * @returns An async iterable of AIMessageChunk objects. */ From 3459207ea83c077887bd560d744b0ccf1d70ec3a Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Tue, 20 May 2025 09:17:59 +0200 Subject: [PATCH 29/67] Update sample-code/src/langchain-orchestration.ts Co-authored-by: Zhongpin Wang --- sample-code/src/langchain-orchestration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index feb6b0b61..42ccc6d8d 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -193,7 +193,7 @@ export async function invokeLangGraphChain(): Promise { * @param controller - The abort controller to cancel the request if needed. * @returns An async iterable of AIMessageChunk objects. */ -export async function orchestrationStreamChain( +export async function streamChain( controller = new AbortController() ): Promise> { // Todo Remove template and use messages after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 From 5dab701b59f724413f61aa59bd1fe02ce9d0ca0c Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 09:19:50 +0200 Subject: [PATCH 30/67] chore: fix rename --- sample-code/src/index.ts | 2 +- sample-code/src/server.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sample-code/src/index.ts b/sample-code/src/index.ts index 8c414f286..2b49d4f46 100644 --- a/sample-code/src/index.ts +++ b/sample-code/src/index.ts @@ -32,7 +32,7 @@ export { export { invokeChain as orchestrationInvokeChain, invokeLangGraphChain, - orchestrationStreamChain + streamChain } from './langchain-orchestration.js'; export { getDeployments, diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index 85b7bc53b..eb6338e5e 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -49,7 +49,7 @@ import { invokeChainWithOutputFilter as invokeChainWithOutputFilterOrchestration, invokeLangGraphChain, invokeChainWithMasking, - orchestrationStreamChain + streamChain } from './langchain-orchestration.js'; import { createCollection, @@ -470,7 +470,7 @@ app.get('/langchain/invoke-stateful-chain', async (req, res) => { app.get('/langchain/stream-orchestration', async (req, res) => { const controller = new AbortController(); try { - const stream = await orchestrationStreamChain(controller); + const stream = await streamChain(controller); res.setHeader('Content-Type', 'text/event-stream'); res.setHeader('Connection', 'keep-alive'); res.flushHeaders(); From 448c403a48600414954fbb1e19ba1b5358dd46a5 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 09:25:27 +0200 Subject: [PATCH 31/67] chore: address JSDoc --- packages/langchain/src/orchestration/client.ts | 2 +- packages/langchain/src/orchestration/util.ts | 4 ++-- sample-code/src/langchain-orchestration.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 67cee3592..d711b003a 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -120,7 +120,7 @@ export class OrchestrationClient extends BaseChatModel< } /** - * Stream response chunks from the OrchestrationClient. + * Stream response chunks from the Orchestration client. * @param messages - The messages to send to the model. * @param options - The call options. * @param runManager - The callback manager for the run. diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index d9ec8bee7..3e9afdd7b 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -258,8 +258,8 @@ export function setTokenUsage( } /** - * Computes token indices for a chunk of the stream response. - * @param chunk - A chunk of the stream response. + * Computes token indices for a chunk of the orchestration stream response. + * @param chunk - A chunk of the orchestration stream response. * @returns An object with prompt and completion indices. * @internal */ diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 42ccc6d8d..7c6f06ce9 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -191,7 +191,7 @@ export async function invokeLangGraphChain(): Promise { /** * Stream responses using LangChain Orchestration client. * @param controller - The abort controller to cancel the request if needed. - * @returns An async iterable of AIMessageChunk objects. + * @returns An async iterable of {@link AIMessageChunk} objects. */ export async function streamChain( controller = new AbortController() From b6191a6fb77d56baf947d754ba2f069e354a3ff8 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 09:42:23 +0200 Subject: [PATCH 32/67] chore: link BLI instead --- packages/langchain/src/orchestration/client.ts | 2 +- sample-code/src/langchain-orchestration.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index d711b003a..1b60ab390 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -150,7 +150,7 @@ export class OrchestrationClient extends BaseChatModel< const response = await this.caller.call(() => orchestrationClient.stream( - // TODO: Adapt messagesHistory after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 + // TODO: Adapt messagesHistory during https://github.com/SAP/ai-sdk-js-backlog/issues/317 { messagesHistory: orchestrationMessages, inputParams }, controller, options.streamOptions, diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 7c6f06ce9..b8c003b7b 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -196,7 +196,7 @@ export async function invokeLangGraphChain(): Promise { export async function streamChain( controller = new AbortController() ): Promise> { - // Todo Remove template and use messages after: https://github.com/SAP/ai-sdk-js-backlog/issues/293 + // TODO: Remove template and use messages during https://github.com/SAP/ai-sdk-js-backlog/issues/317 const orchestrationConfig: LangchainOrchestrationModuleConfig = { llm: { model_name: 'gpt-4o' From 56ea273fcdcc84999514a28749fa75a1ca5a5fb5 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 09:51:10 +0200 Subject: [PATCH 33/67] chore: switch to hardcoded choice index --- packages/langchain/src/orchestration/util.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 3e9afdd7b..3d9ce144a 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -263,6 +263,7 @@ export function setTokenUsage( * @returns An object with prompt and completion indices. * @internal */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { prompt: number; completion: number; @@ -270,6 +271,8 @@ export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { return { // Indicates the token is part of the first prompt prompt: 0, - completion: chunk.data.orchestration_result?.choices[0]?.index ?? 0 + // Hardcoding to 0 as mutiple choices are not currently supported in the orchestration service. + // Switch to `chunk.data.orchestration_result.choices[0].index` when support is added. + completion: 0 }; } From dbf717460649eb52b73ee42319306f0a80ef126a Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 12:06:38 +0200 Subject: [PATCH 34/67] chore: add convenience functions --- .../src/orchestration-stream-chunk-response.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.ts b/packages/orchestration/src/orchestration-stream-chunk-response.ts index 33915f999..b47a33ae3 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.ts @@ -27,9 +27,7 @@ export class OrchestrationStreamChunkResponse { * @returns The finish reason. */ getFinishReason(choiceIndex = 0): string | undefined { - return this.getChoices()?.find( - (c: LlmChoiceStreaming) => c.index === choiceIndex - )?.finish_reason; + return this.findChoiceByIndex(choiceIndex)?.finish_reason; } /** @@ -38,9 +36,7 @@ export class OrchestrationStreamChunkResponse { * @returns The message delta content. */ getDeltaContent(choiceIndex = 0): string | undefined { - return this.getChoices()?.find( - (c: LlmChoiceStreaming) => c.index === choiceIndex - )?.delta.content; + return this.findChoiceByIndex(choiceIndex)?.delta.content; } /** @@ -49,12 +45,16 @@ export class OrchestrationStreamChunkResponse { * @returns The message tool call chunks. */ getDeltaToolCallChunks(choiceIndex = 0): ToolCallChunk[] | undefined { - return this.getChoices()?.find( - (c: LlmChoiceStreaming) => c.index === choiceIndex - )?.delta.tool_calls; + return this.findChoiceByIndex(choiceIndex)?.delta.tool_calls; } private getChoices(): LlmChoiceStreaming[] | undefined { return this.data.orchestration_result?.choices; } + + private findChoiceByIndex(index: number): LlmChoiceStreaming | undefined { + return this.getChoices()?.find( + (c: LlmChoiceStreaming) => c.index === index + ); + } } From f82301e9443426ca021f77594c95c1b3eabbfd9f Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 13:10:52 +0200 Subject: [PATCH 35/67] chore: add convenience + tests --- ...rchestration-stream-chunk-response.test.ts | 38 ++++++++++++++++++ .../orchestration-stream-chunk-response.ts | 15 ++++--- ...etion-stream-chunk-response-tool-call.json | 39 +++++++++++++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-tool-call.json diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts index 703a27358..bfa75e93c 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts @@ -5,11 +5,13 @@ describe('Orchestration chat completion stream chunk response', () => { let mockResponses: { tokenUsageAndFinishReasonResponse: any; deltaContentResponse: any; + toolCallResponse: any; }; let orchestrationStreamChunkResponses: { tokenUsageResponse: OrchestrationStreamChunkResponse; finishReasonResponse: OrchestrationStreamChunkResponse; deltaContentResponse: OrchestrationStreamChunkResponse; + toolCallResponse: OrchestrationStreamChunkResponse; }; beforeAll(async () => { @@ -21,6 +23,10 @@ describe('Orchestration chat completion stream chunk response', () => { deltaContentResponse: await parseMockResponse( 'orchestration', 'orchestration-chat-completion-stream-chunk-response-delta-content.json' + ), + toolCallResponse: await parseMockResponse( + 'orchestration', + 'orchestration-chat-completion-stream-chunk-response-tool-call.json' ) }; orchestrationStreamChunkResponses = { @@ -32,6 +38,9 @@ describe('Orchestration chat completion stream chunk response', () => { ), deltaContentResponse: new OrchestrationStreamChunkResponse( mockResponses.deltaContentResponse + ), + toolCallResponse: new OrchestrationStreamChunkResponse( + mockResponses.toolCallResponse ) }; }); @@ -71,4 +80,33 @@ describe('Orchestration chat completion stream chunk response', () => { '"rimarily focusing on Java and JavaScript/Node.js environments, allowing developers to work in their "' ); }); + + it('should return delta tool call chunks with default index 0', () => { + const toolCallChunks = + orchestrationStreamChunkResponses.toolCallResponse.getDeltaToolCallChunks(); + + expect(toolCallChunks).toBeDefined(); + expect(toolCallChunks).toHaveLength(1); + expect(toolCallChunks?.[0]).toEqual({ + index: 0, + function: { + arguments: '20' + } + }); + }); + + it('should find choice by valid index', () => { + const choice = + orchestrationStreamChunkResponses.toolCallResponse.findChoiceByIndex(0); + + expect(choice).toBeDefined(); + expect(choice?.index).toBe(0); + expect(choice?.delta?.role).toEqual('assistant'); + expect(choice?.delta.content).toEqual( + 'rimarily focusing on Java and JavaScript/Node.js environments, allowing developers to work in their ' + ); + expect(choice?.delta.tool_calls).toEqual([ + { index: 0, function: { arguments: '20' } } + ]); + }); }); diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.ts b/packages/orchestration/src/orchestration-stream-chunk-response.ts index b47a33ae3..888cf82c8 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.ts @@ -48,13 +48,18 @@ export class OrchestrationStreamChunkResponse { return this.findChoiceByIndex(choiceIndex)?.delta.tool_calls; } - private getChoices(): LlmChoiceStreaming[] | undefined { - return this.data.orchestration_result?.choices; - } - - private findChoiceByIndex(index: number): LlmChoiceStreaming | undefined { + /** + * Parses the chunk response and returns the choice by index. + * @param index - The index of the choice to find. + * @returns An {@link LLMChoiceStreaming} object associated withe index. + */ + findChoiceByIndex(index: number): LlmChoiceStreaming | undefined { return this.getChoices()?.find( (c: LlmChoiceStreaming) => c.index === index ); } + + private getChoices(): LlmChoiceStreaming[] | undefined { + return this.data.orchestration_result?.choices; + } } diff --git a/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-tool-call.json b/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-tool-call.json new file mode 100644 index 000000000..f5ec3ba8d --- /dev/null +++ b/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-tool-call.json @@ -0,0 +1,39 @@ +{ + "request_id": "ceaa358c-48b8-4ce1-8a62-b0c47675fc9c", + "module_results": { + "llm": { + "id": "chatcmpl-AfmsPYkaH9uHogKZusAaVPC3zSNys", + "object": "chat.completion.chunk", + "created": 1734522693, + "model": "gpt-4o-2024-08-06", + "system_fingerprint": "fp_4e924a4b48", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "rimarily focusing on Java and JavaScript/Node.js environments, allowing developers to work in their ", + "tool_calls": [{ "index": 0, "function": { "arguments": "20" } }] + } + } + ] + } + }, + "orchestration_result": { + "id": "chatcmpl-AfmsPYkaH9uHogKZusAaVPC3zSNys", + "object": "chat.completion.chunk", + "created": 1734522693, + "model": "gpt-4o-2024-08-06", + "system_fingerprint": "fp_4e924a4b48", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "rimarily focusing on Java and JavaScript/Node.js environments, allowing developers to work in their ", + "tool_calls": [{ "index": 0, "function": { "arguments": "20" } }] + } + } + ] + } +} From ac9f6e83356a46c6233b96de8a6a73e668d2342f Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 14:04:30 +0200 Subject: [PATCH 36/67] chore: clean up generate --- .../langchain/src/orchestration/client.ts | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 1b60ab390..1c9aae195 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -85,30 +85,24 @@ export class OrchestrationClient extends BaseChatModel< options: typeof this.ParsedCallOptions, runManager?: CallbackManagerForLLMRun ): Promise { - const res = await this.caller.callWithOptions( - { - signal: options.signal - }, - () => { - const { inputParams, customRequestConfig } = options; - const mergedOrchestrationConfig = - this.mergeOrchestrationConfig(options); - const orchestrationClient = new OrchestrationClientBase( - mergedOrchestrationConfig, - this.deploymentConfig, - this.destination - ); - const messagesHistory = - mapLangChainMessagesToOrchestrationMessages(messages); - return orchestrationClient.chatCompletion( - { - messagesHistory, - inputParams - }, - customRequestConfig - ); - } - ); + const res = await this.caller.call(() => { + const { inputParams, customRequestConfig } = options; + const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); + const orchestrationClient = new OrchestrationClientBase( + mergedOrchestrationConfig, + this.deploymentConfig, + this.destination + ); + const messagesHistory = + mapLangChainMessagesToOrchestrationMessages(messages); + return orchestrationClient.chatCompletion( + { + messagesHistory, + inputParams + }, + customRequestConfig + ); + }); const content = res.getContent(); From cdb4749b40438b591f3836aace79ee983018b3a9 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 15:31:11 +0200 Subject: [PATCH 37/67] chore: add callWithOptions back --- packages/langchain/src/orchestration/client.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 1c9aae195..60e4a6aca 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -85,7 +85,11 @@ export class OrchestrationClient extends BaseChatModel< options: typeof this.ParsedCallOptions, runManager?: CallbackManagerForLLMRun ): Promise { - const res = await this.caller.call(() => { + const res = await this.caller.callWithOptions( + { + signal: options.signal + }, + (() => { const { inputParams, customRequestConfig } = options; const mergedOrchestrationConfig = this.mergeOrchestrationConfig(options); const orchestrationClient = new OrchestrationClientBase( @@ -102,7 +106,7 @@ export class OrchestrationClient extends BaseChatModel< }, customRequestConfig ); - }); + })); const content = res.getContent(); From c073c88f9985ac323ff04646b560e2e42596f2e8 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Tue, 20 May 2025 16:08:01 +0200 Subject: [PATCH 38/67] Update packages/langchain/src/orchestration/util.ts Co-authored-by: Deeksha Sinha <88374536+deekshas8@users.noreply.github.com> --- packages/langchain/src/orchestration/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 3d9ce144a..99e32df78 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -245,7 +245,7 @@ export function setFinishReason( */ export function setTokenUsage( messageChunk: AIMessageChunk, - tokenUsage: TokenUsage | undefined + tokenUsage?: TokenUsage ): void { if (tokenUsage) { messageChunk.usage_metadata = { From 58434d5a651a6a24e045ca8d2c38fa6f00f95a02 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Tue, 20 May 2025 16:08:14 +0200 Subject: [PATCH 39/67] Update packages/langchain/src/orchestration/util.ts Co-authored-by: Deeksha Sinha <88374536+deekshas8@users.noreply.github.com> --- packages/langchain/src/orchestration/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 99e32df78..270e3ece8 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -230,7 +230,7 @@ export function mapOrchestrationChunkToLangChainMessageChunk( */ export function setFinishReason( messageChunk: AIMessageChunk, - finishReason: string | undefined + finishReason?: string ): void { if (finishReason) { messageChunk.response_metadata.finish_reason = finishReason; From 64f16c7639e34f75235dd976b7db3a3bfd15e5c3 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 16:20:34 +0200 Subject: [PATCH 40/67] chore: fix build --- packages/langchain/src/orchestration/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index f781c66b2..16285af2a 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -98,11 +98,11 @@ export class OrchestrationClient extends BaseChatModel< this.deploymentConfig, this.destination ); - const allMesages = - mapLangchainMessagesToOrchestrationMessages(messages); + const allMessages = + mapLangChainMessagesToOrchestrationMessages(messages); return orchestrationClient.chatCompletion( { - messages: allMesages, + messages: allMessages, inputParams }, customRequestConfig From 1cacf75229208761acc8c8ae8196114ec61fb1e6 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Tue, 20 May 2025 16:38:58 +0200 Subject: [PATCH 41/67] chore: fix build --- packages/langchain/src/orchestration/client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 599d509b2..4b9b92923 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -60,7 +60,7 @@ describe('orchestration service client', () => { ) { mockInference( { - data: constructCompletionPostRequest(config) + data: constructCompletionPostRequest(config, { messages: [] }, isStream) }, { data: response, From 0b347608ae472e556d2fb81b8fa1f10b1b8bc907 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Tue, 20 May 2025 16:40:23 +0200 Subject: [PATCH 42/67] Update packages/langchain/src/orchestration/client.test.ts Co-authored-by: Zhongpin Wang --- packages/langchain/src/orchestration/client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 4b9b92923..291987526 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -180,7 +180,7 @@ describe('orchestration service client', () => { ); }); - it('streaming with abort signal', async () => { + it('streams and aborts with a signal', async () => { mockStreamInferenceWithResilience(); const client = new OrchestrationClient(config); const controller = new AbortController(); From 206067a86d291c6b9d08a59aac6ed216c4316c01 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 21 May 2025 12:15:31 +0200 Subject: [PATCH 43/67] chore: rename --- packages/langchain/src/orchestration/util.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 270e3ece8..0110e8f91 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -113,7 +113,7 @@ export function mapLangChainMessagesToOrchestrationMessages( * @param toolCalls - The {@link AzureOpenAiChatCompletionMessageToolCalls} response. * @returns The LangChain {@link ToolCall}. */ -function mapAzureOpenAiToLangchainToolCall( +function mapAzureOpenAiToLangChainToolCall( toolCalls?: AzureOpenAiChatCompletionMessageToolCalls ): ToolCall[] | undefined { if (toolCalls) { @@ -161,7 +161,7 @@ export function mapOutputToChatResult( text: choice.message.content ?? '', message: new AIMessage({ content: choice.message.content ?? '', - tool_calls: mapAzureOpenAiToLangchainToolCall( + tool_calls: mapAzureOpenAiToLangChainToolCall( choice.message.tool_calls ), additional_kwargs: { From 4560687ee3399fe3c752785dd6b744bd2028cfcb Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 21 May 2025 16:25:33 +0200 Subject: [PATCH 44/67] chore: fix build --- .../src/orchestration/client.test.ts | 68 ++++++++++++++++++- .../langchain/src/orchestration/client.ts | 3 +- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 291987526..4fd9d8ecd 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -10,12 +10,12 @@ import { } from '../../../../test-util/mock-http.js'; import { OrchestrationClient } from './client.js'; import type { ToolCall } from '@langchain/core/messages/tool'; +import type { AIMessageChunk } from '@langchain/core/messages'; import type { LangchainOrchestrationModuleConfig } from './types.js'; import type { CompletionPostResponse, ErrorResponse } from '@sap-ai-sdk/orchestration'; -import type { AIMessageChunk } from '@langchain/core/messages'; jest.setTimeout(30000); @@ -56,7 +56,7 @@ describe('orchestration service client', () => { delay?: number; }, status: number = 200, - isStream: boolean = false + isStream?: boolean ) { mockInference( { @@ -243,4 +243,68 @@ describe('orchestration service client', () => { expect(completeToolCall!.name).toEqual('convert_temperature_to_fahrenheit'); expect(completeToolCall!.args).toEqual({ temperature: 20 }); }); + + it('streams and aborts with a signal', async () => { + mockStreamInferenceWithResilience(); + const client = new OrchestrationClient(config); + const controller = new AbortController(); + const { signal } = controller; + const stream = await client.stream([], { signal }); + const streamPromise = async () => { + for await (const _chunk of stream) { + controller.abort(); + } + }; + + await expect(streamPromise()).rejects.toThrow(); + }, 1000); + + it('streaming with callbacks', async () => { + mockStreamInferenceWithResilience(); + let tokenCount = 0; + const callbackHandler = { + handleLLMNewToken: jest.fn().mockImplementation(() => { + tokenCount += 1; + }) + }; + const client = new OrchestrationClient(config, { + callbacks: [callbackHandler] + }); + const stream = await client.stream([]); + const chunks = []; + + for await (const chunk of stream) { + chunks.push(chunk); + break; + } + expect(callbackHandler.handleLLMNewToken).toHaveBeenCalled(); + const firstCallArgs = callbackHandler.handleLLMNewToken.mock.calls[0]; + expect(firstCallArgs).toBeDefined(); + // First chunk content is empty + expect(firstCallArgs[0]).toEqual(''); + // Second argument should be the token indices + expect(firstCallArgs[1]).toEqual({ prompt: 0, completion: 0 }); + expect(tokenCount).toBeGreaterThan(0); + expect(chunks.length).toBeGreaterThan(0); + }); + + it('supports streaming responses with tool calls', async () => { + mockStreamInferenceWithResilience(mockResponseStreamToolCalls); + + const client = new OrchestrationClient(config); + const stream = await client.stream([]); + + let finalOutput: AIMessageChunk | undefined; + for await (const chunk of stream) { + finalOutput = !finalOutput ? chunk : finalOutput.concat(chunk); + } + + expect(finalOutput).toBeDefined(); + expect(finalOutput!.tool_call_chunks).toBeDefined(); + expect(finalOutput!.tool_calls).toBeDefined(); + + const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; + expect(completeToolCall!.name).toEqual('convert_temperature_to_fahrenheit'); + expect(completeToolCall!.args).toEqual({ temperature: 20 }); + }); }); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 16285af2a..a41b4f54d 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -150,8 +150,7 @@ export class OrchestrationClient extends BaseChatModel< const response = await this.caller.call(() => orchestrationClient.stream( - // TODO: Adapt messagesHistory during https://github.com/SAP/ai-sdk-js-backlog/issues/317 - { messagesHistory: orchestrationMessages, inputParams }, + { messages: orchestrationMessages, inputParams }, controller, options.streamOptions, customRequestConfig From 32971bd25566841b75286965a190f3bb24bff3d4 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 21 May 2025 17:02:54 +0200 Subject: [PATCH 45/67] chore: adjust test --- packages/langchain/src/orchestration/client.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 4fd9d8ecd..85bd439bf 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -215,13 +215,11 @@ describe('orchestration service client', () => { } expect(callbackHandler.handleLLMNewToken).toHaveBeenCalled(); const firstCallArgs = callbackHandler.handleLLMNewToken.mock.calls[0]; - expect(firstCallArgs).toBeDefined(); // First chunk content is empty expect(firstCallArgs[0]).toEqual(''); // Second argument should be the token indices expect(firstCallArgs[1]).toEqual({ prompt: 0, completion: 0 }); expect(tokenCount).toBeGreaterThan(0); - expect(chunks.length).toBeGreaterThan(0); }); it('supports streaming responses with tool calls', async () => { From a9fc5d9c2eaab06e671787c0db007a707f407d85 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Wed, 21 May 2025 17:13:57 +0200 Subject: [PATCH 46/67] Update packages/langchain/src/orchestration/client.test.ts Co-authored-by: Deeksha Sinha <88374536+deekshas8@users.noreply.github.com> --- packages/langchain/src/orchestration/client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 85bd439bf..090d7be4b 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -257,7 +257,7 @@ describe('orchestration service client', () => { await expect(streamPromise()).rejects.toThrow(); }, 1000); - it('streaming with callbacks', async () => { + it('streams with a callback', async () => { mockStreamInferenceWithResilience(); let tokenCount = 0; const callbackHandler = { From 46ddc012a63c43ad7791ce007bc7ac015484ecfc Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 21 May 2025 17:20:04 +0200 Subject: [PATCH 47/67] chore: address review comment --- packages/langchain/src/orchestration/client.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 85bd439bf..b225c3401 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -24,7 +24,7 @@ describe('orchestration service client', () => { let mockResponseInputFilterError: ErrorResponse; let mockResponseStream: string; let mockResponseStreamToolCalls: string; - beforeEach(async () => { + beforeAll(async () => { mockClientCredentialsGrantCall(); mockDeploymentsList({ scenarioId: 'orchestration' }, { id: '1234' }); mockResponse = await parseMockResponse( @@ -166,9 +166,9 @@ describe('orchestration service client', () => { for await (const chunk of stream) { iterations++; - intermediateChunk = !intermediateChunk - ? chunk - : intermediateChunk.concat(chunk); + intermediateChunk = intermediateChunk + ? intermediateChunk.concat(chunk) + : chunk; if (iterations >= maxIterations) { break; } From 27fc55b822e6d8a391998da143f88dbec3e8bbad Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 21 May 2025 17:35:19 +0200 Subject: [PATCH 48/67] chore: link BLI --- packages/langchain/src/orchestration/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 0110e8f91..c35f854f2 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -272,7 +272,7 @@ export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { // Indicates the token is part of the first prompt prompt: 0, // Hardcoding to 0 as mutiple choices are not currently supported in the orchestration service. - // Switch to `chunk.data.orchestration_result.choices[0].index` when support is added. + // TODO: Switch to `chunk.data.orchestration_result.choices[0].index` when support is added via https://github.com/SAP/ai-sdk-js-backlog/issues/321 completion: 0 }; } From 6b50d39861e7823ca20156d719fbec5d8c7b9ebe Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Wed, 21 May 2025 18:01:06 +0200 Subject: [PATCH 49/67] chore: cleanup tests --- .../src/orchestration/client.test.ts | 98 +++---------------- 1 file changed, 14 insertions(+), 84 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index ef9ce3adc..07af934d5 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -160,23 +160,22 @@ describe('orchestration service client', () => { const client = new OrchestrationClient(config); const stream = await client.stream([]); - let iterations = 0; - const maxIterations = 2; let intermediateChunk: AIMessageChunk | undefined; for await (const chunk of stream) { - iterations++; intermediateChunk = intermediateChunk ? intermediateChunk.concat(chunk) : chunk; - if (iterations >= maxIterations) { - break; - } } - expect(intermediateChunk).toBeDefined(); - expect(intermediateChunk!.content).toBeDefined(); - expect(intermediateChunk!.content).toEqual( - 'The SAP Cloud SDK is a comprehensive development toolkit designed to simplify and accelerate the cre' + expect(intermediateChunk?.content).toEqual( + 'The SAP Cloud SDK is a comprehensive development toolkit designed to simplify and accelerate the creation of applications that integrate with SAP solutions, particularly those built on the SAP Business Technology Platform (BTP). It provides developers with libraries, tools, and best practices that streamline the process of connecting to SAP systems, such as S/4HANA and other services available on the SAP Cloud Platform.\n\n' + + 'Key features of the SAP Cloud SDK include:\n\n' + + '1. **Simplified Connectivity**: The SDK offers pre-built libraries to easily interact with SAP services, providing capabilities for authentication, service consumption, and OData/REST client generation.\n\n' + + '2. **Multi-cloud Support**: It supports multiple cloud environments, ensuring that applications remain flexible and can be deployed across various cloud providers.\n\n' + + '3. **Best Practices and Guidelines**: The SDK includes best practices for development, ensuring high-quality, scalable, and maintainable code.\n\n' + + '4. **Project Scaffolding and Code Samples**: Developers can quickly start their projects using provided templates and samples, accelerating the development process and reducing the learning curve.\n\n' + + '5. **Extensive Documentation and Community Support**: Ample documentation, tutorials, and an active community help developers overcome challenges and adopt the SDK efficiently.\n\n' + + "Overall, the SAP Cloud SDK is an essential tool for developers looking to build cloud-native applications and extensions that seamlessly integrate with SAP's enterprise solutions." ); }); @@ -186,75 +185,13 @@ describe('orchestration service client', () => { const controller = new AbortController(); const { signal } = controller; const stream = await client.stream([], { signal }); - const streamPromise = async () => { - for await (const _chunk of stream) { - controller.abort(); - } - }; - - await expect(streamPromise()).rejects.toThrow(); - }, 1000); - - it('streaming with callbacks', async () => { - mockStreamInferenceWithResilience(); - let tokenCount = 0; - const callbackHandler = { - handleLLMNewToken: jest.fn().mockImplementation(() => { - tokenCount += 1; - }) - }; - const client = new OrchestrationClient(config, { - callbacks: [callbackHandler] - }); - const stream = await client.stream([]); - const chunks = []; - - for await (const chunk of stream) { - chunks.push(chunk); - break; - } - expect(callbackHandler.handleLLMNewToken).toHaveBeenCalled(); - const firstCallArgs = callbackHandler.handleLLMNewToken.mock.calls[0]; - // First chunk content is empty - expect(firstCallArgs[0]).toEqual(''); - // Second argument should be the token indices - expect(firstCallArgs[1]).toEqual({ prompt: 0, completion: 0 }); - expect(tokenCount).toBeGreaterThan(0); - }); - - it('supports streaming responses with tool calls', async () => { - mockStreamInferenceWithResilience(mockResponseStreamToolCalls); - - const client = new OrchestrationClient(config); - const stream = await client.stream([]); - - let finalOutput: AIMessageChunk | undefined; - for await (const chunk of stream) { - finalOutput = !finalOutput ? chunk : finalOutput.concat(chunk); - } - - expect(finalOutput).toBeDefined(); - expect(finalOutput!.tool_call_chunks).toBeDefined(); - expect(finalOutput!.tool_calls).toBeDefined(); - - const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; - expect(completeToolCall!.name).toEqual('convert_temperature_to_fahrenheit'); - expect(completeToolCall!.args).toEqual({ temperature: 20 }); - }); - - it('streams and aborts with a signal', async () => { - mockStreamInferenceWithResilience(); - const client = new OrchestrationClient(config); - const controller = new AbortController(); - const { signal } = controller; - const stream = await client.stream([], { signal }); - const streamPromise = async () => { + const streamFunction = async () => { for await (const _chunk of stream) { controller.abort(); } }; - await expect(streamPromise()).rejects.toThrow(); + await expect(streamFunction()).rejects.toThrow(); }, 1000); it('streams with a callback', async () => { @@ -277,13 +214,11 @@ describe('orchestration service client', () => { } expect(callbackHandler.handleLLMNewToken).toHaveBeenCalled(); const firstCallArgs = callbackHandler.handleLLMNewToken.mock.calls[0]; - expect(firstCallArgs).toBeDefined(); // First chunk content is empty expect(firstCallArgs[0]).toEqual(''); // Second argument should be the token indices expect(firstCallArgs[1]).toEqual({ prompt: 0, completion: 0 }); expect(tokenCount).toBeGreaterThan(0); - expect(chunks.length).toBeGreaterThan(0); }); it('supports streaming responses with tool calls', async () => { @@ -294,15 +229,10 @@ describe('orchestration service client', () => { let finalOutput: AIMessageChunk | undefined; for await (const chunk of stream) { - finalOutput = !finalOutput ? chunk : finalOutput.concat(chunk); + finalOutput = finalOutput ? finalOutput.concat(chunk) : chunk; } - - expect(finalOutput).toBeDefined(); - expect(finalOutput!.tool_call_chunks).toBeDefined(); - expect(finalOutput!.tool_calls).toBeDefined(); - const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; - expect(completeToolCall!.name).toEqual('convert_temperature_to_fahrenheit'); - expect(completeToolCall!.args).toEqual({ temperature: 20 }); + expect(completeToolCall?.name).toEqual('convert_temperature_to_fahrenheit'); + expect(completeToolCall?.args).toEqual({ temperature: 20 }); }); }); From be56c2184c9ed1c1950b32320c1b5cf97a976b20 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 10:49:58 +0200 Subject: [PATCH 50/67] chore: address review comments --- packages/langchain/src/orchestration/client.test.ts | 4 ++-- packages/langchain/src/orchestration/client.ts | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 07af934d5..a9fd0718c 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -191,7 +191,7 @@ describe('orchestration service client', () => { } }; - await expect(streamFunction()).rejects.toThrow(); + await expect(streamFunction()).rejects.toThrow('Aborted'); }, 1000); it('streams with a callback', async () => { @@ -206,7 +206,7 @@ describe('orchestration service client', () => { callbacks: [callbackHandler] }); const stream = await client.stream([]); - const chunks = []; + const chunks: AIMessageChunk[] = []; for await (const chunk of stream) { chunks.push(chunk); diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index a41b4f54d..e028b7329 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -185,10 +185,6 @@ export class OrchestrationClient extends BaseChatModel< yield generationChunk; } - - if (options.signal?.aborted) { - throw new Error('AbortError'); - } } private mergeOrchestrationConfig( From 93b6cf2bf38fb832d7fde81c5b3fb34a3c77b102 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 10:59:02 +0200 Subject: [PATCH 51/67] chore: review comments --- .../orchestration/__snapshots__/client.test.ts.snap | 12 ++++++++++++ packages/langchain/src/orchestration/client.test.ts | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap index cbd68d0c3..28a3d948d 100644 --- a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap +++ b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap @@ -137,3 +137,15 @@ exports[`orchestration service client returns successful response when timeout i "type": "constructor", } `; + +exports[`orchestration service client supports streaming responses with tool calls 1`] = ` +[ + { + "args": "{"temperature":20", + "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", + "index": 0, + "name": "convert_temperature_to_fahrenheit", + "type": "tool_call_chunk", + }, +] +`; diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index a9fd0718c..5dc222ef7 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -232,7 +232,8 @@ describe('orchestration service client', () => { finalOutput = finalOutput ? finalOutput.concat(chunk) : chunk; } const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; - expect(completeToolCall?.name).toEqual('convert_temperature_to_fahrenheit'); - expect(completeToolCall?.args).toEqual({ temperature: 20 }); + expect(completeToolCall.name).toEqual('convert_temperature_to_fahrenheit'); + expect(completeToolCall.args).toEqual({ temperature: 20 }); + expect(finalOutput?.tool_call_chunks).toMatchSnapshot(); }); }); From 40da1192138ab188d6735296cf6322e5720d9bfc Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 11:23:33 +0200 Subject: [PATCH 52/67] chore: Add snapshot tests instead --- .../__snapshots__/client.test.ts.snap | 178 +++++++++++++++++- .../src/orchestration/client.test.ts | 19 +- 2 files changed, 175 insertions(+), 22 deletions(-) diff --git a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap index 28a3d948d..ac79169a4 100644 --- a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap +++ b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap @@ -138,14 +138,176 @@ exports[`orchestration service client returns successful response when timeout i } `; +exports[`orchestration service client supports streaming responses 1`] = ` +{ + "id": [ + "langchain_core", + "messages", + "AIMessageChunk", + ], + "kwargs": { + "additional_kwargs": { + "module_results": { + "llm": { + "choices": [ + { + "delta": { + "content": "The SAP Cloud SDK is a comprehensive development toolkit designed to simplify and accelerate the creation of applications that integrate with SAP solutions, particularly those built on the SAP Business Technology Platform (BTP). It provides developers with libraries, tools, and best practices that streamline the process of connecting to SAP systems, such as S/4HANA and other services available on the SAP Cloud Platform. + +Key features of the SAP Cloud SDK include: + +1. **Simplified Connectivity**: The SDK offers pre-built libraries to easily interact with SAP services, providing capabilities for authentication, service consumption, and OData/REST client generation. + +2. **Multi-cloud Support**: It supports multiple cloud environments, ensuring that applications remain flexible and can be deployed across various cloud providers. + +3. **Best Practices and Guidelines**: The SDK includes best practices for development, ensuring high-quality, scalable, and maintainable code. + +4. **Project Scaffolding and Code Samples**: Developers can quickly start their projects using provided templates and samples, accelerating the development process and reducing the learning curve. + +5. **Extensive Documentation and Community Support**: Ample documentation, tutorials, and an active community help developers overcome challenges and adopt the SDK efficiently. + +Overall, the SAP Cloud SDK is an essential tool for developers looking to build cloud-native applications and extensions that seamlessly integrate with SAP's enterprise solutions.", + "role": "assistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistant", + }, + "finish_reason": "stop", + "index": 0, + }, + ], + "created": 1734524005, + "id": "chatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wpchatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wp", + "model": "gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06", + "object": "chat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunk", + "system_fingerprint": "fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48fp_4e924a4b48", + "usage": { + "completion_tokens": 271, + "prompt_tokens": 17, + "total_tokens": 288, + }, + }, + "templating": [ + { + "content": "Give me a short introduction of SAP Cloud SDK.", + "role": "user", + }, + ], + }, + "request_id": "66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b66172762-8c47-4438-89e7-2689be8f370b", + }, + "content": "The SAP Cloud SDK is a comprehensive development toolkit designed to simplify and accelerate the creation of applications that integrate with SAP solutions, particularly those built on the SAP Business Technology Platform (BTP). It provides developers with libraries, tools, and best practices that streamline the process of connecting to SAP systems, such as S/4HANA and other services available on the SAP Cloud Platform. + +Key features of the SAP Cloud SDK include: + +1. **Simplified Connectivity**: The SDK offers pre-built libraries to easily interact with SAP services, providing capabilities for authentication, service consumption, and OData/REST client generation. + +2. **Multi-cloud Support**: It supports multiple cloud environments, ensuring that applications remain flexible and can be deployed across various cloud providers. + +3. **Best Practices and Guidelines**: The SDK includes best practices for development, ensuring high-quality, scalable, and maintainable code. + +4. **Project Scaffolding and Code Samples**: Developers can quickly start their projects using provided templates and samples, accelerating the development process and reducing the learning curve. + +5. **Extensive Documentation and Community Support**: Ample documentation, tutorials, and an active community help developers overcome challenges and adopt the SDK efficiently. + +Overall, the SAP Cloud SDK is an essential tool for developers looking to build cloud-native applications and extensions that seamlessly integrate with SAP's enterprise solutions.", + "id": undefined, + "invalid_tool_calls": [], + "response_metadata": { + "completion": 0, + "finish_reason": "stop", + "prompt": 0, + "token_usage": { + "completion_tokens": 271, + "prompt_tokens": 17, + "total_tokens": 288, + }, + }, + "tool_call_chunks": [], + "tool_calls": [], + "usage_metadata": { + "input_tokens": 17, + "output_tokens": 271, + "total_tokens": 288, + }, + }, + "lc": 1, + "type": "constructor", +} +`; + exports[`orchestration service client supports streaming responses with tool calls 1`] = ` -[ - { - "args": "{"temperature":20", - "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", - "index": 0, - "name": "convert_temperature_to_fahrenheit", - "type": "tool_call_chunk", +{ + "id": [ + "langchain_core", + "messages", + "AIMessageChunk", + ], + "kwargs": { + "additional_kwargs": { + "module_results": { + "llm": { + "choices": [ + { + "delta": { + "content": "", + "role": "assistantassistantassistantassistantassistant", + "tool_calls": [ + { + "function": { + "arguments": "{"temperature":20", + "name": "convert_temperature_to_fahrenheit", + }, + "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", + "index": 0, + "type": "function", + }, + ], + }, + "index": 0, + }, + ], + "created": 1747305948, + "id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", + "model": "gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06", + "object": "chat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunk", + "system_fingerprint": "fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0", + }, + "templating": [ + { + "content": "Convert 20 degrees Celsius to Fahrenheit.", + "role": "user", + }, + ], + }, + "request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc", + }, + "content": "", + "id": undefined, + "invalid_tool_calls": [], + "response_metadata": { + "completion": 0, + "prompt": 0, + }, + "tool_call_chunks": [ + { + "args": "{"temperature":20", + "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", + "index": 0, + "name": "convert_temperature_to_fahrenheit", + "type": "tool_call_chunk", + }, + ], + "tool_calls": [ + { + "args": { + "temperature": 20, + }, + "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", + "name": "convert_temperature_to_fahrenheit", + "type": "tool_call", + }, + ], + "usage_metadata": undefined, }, -] + "lc": 1, + "type": "constructor", +} `; diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 5dc222ef7..62e9159e4 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -160,23 +160,14 @@ describe('orchestration service client', () => { const client = new OrchestrationClient(config); const stream = await client.stream([]); - let intermediateChunk: AIMessageChunk | undefined; + let finalOutput: AIMessageChunk | undefined; for await (const chunk of stream) { - intermediateChunk = intermediateChunk - ? intermediateChunk.concat(chunk) + finalOutput = finalOutput + ? finalOutput.concat(chunk) : chunk; } - expect(intermediateChunk?.content).toEqual( - 'The SAP Cloud SDK is a comprehensive development toolkit designed to simplify and accelerate the creation of applications that integrate with SAP solutions, particularly those built on the SAP Business Technology Platform (BTP). It provides developers with libraries, tools, and best practices that streamline the process of connecting to SAP systems, such as S/4HANA and other services available on the SAP Cloud Platform.\n\n' + - 'Key features of the SAP Cloud SDK include:\n\n' + - '1. **Simplified Connectivity**: The SDK offers pre-built libraries to easily interact with SAP services, providing capabilities for authentication, service consumption, and OData/REST client generation.\n\n' + - '2. **Multi-cloud Support**: It supports multiple cloud environments, ensuring that applications remain flexible and can be deployed across various cloud providers.\n\n' + - '3. **Best Practices and Guidelines**: The SDK includes best practices for development, ensuring high-quality, scalable, and maintainable code.\n\n' + - '4. **Project Scaffolding and Code Samples**: Developers can quickly start their projects using provided templates and samples, accelerating the development process and reducing the learning curve.\n\n' + - '5. **Extensive Documentation and Community Support**: Ample documentation, tutorials, and an active community help developers overcome challenges and adopt the SDK efficiently.\n\n' + - "Overall, the SAP Cloud SDK is an essential tool for developers looking to build cloud-native applications and extensions that seamlessly integrate with SAP's enterprise solutions." - ); + expect(finalOutput).toMatchSnapshot(); }); it('streams and aborts with a signal', async () => { @@ -234,6 +225,6 @@ describe('orchestration service client', () => { const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; expect(completeToolCall.name).toEqual('convert_temperature_to_fahrenheit'); expect(completeToolCall.args).toEqual({ temperature: 20 }); - expect(finalOutput?.tool_call_chunks).toMatchSnapshot(); + expect(finalOutput).toMatchSnapshot(); }); }); From 94ceda9d77d83459c656ce2b9cfa1782060a790e Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 13:07:59 +0200 Subject: [PATCH 53/67] chore: fix issues after merge conflict --- packages/langchain/src/orchestration/util.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 5701fa92a..f7fc2eeef 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -6,7 +6,7 @@ import type { Template, ToolCallChunk as OrchestrationToolCallChunk, OrchestrationStreamChunkResponse, - TokenUsage + TokenUsage, TemplatingModuleConfig } from '@sap-ai-sdk/orchestration'; import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool'; @@ -207,7 +207,7 @@ export function mapOrchestrationChunkToLangChainMessageChunk( ): AIMessageChunk { const { module_results, request_id } = chunk.data; const content = chunk.getDeltaContent() ?? ''; - const toolCallChunks = chunk.getDeltaToolCallChunks(); + const toolCallChunks = chunk.getDeltaToolCalls(); const additional_kwargs: Record = { module_results, From ad368c1419653c6bb0457ea5c8836a7375d91a73 Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Thu, 22 May 2025 11:09:02 +0000 Subject: [PATCH 54/67] fix: Changes from lint --- packages/langchain/src/orchestration/client.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 62e9159e4..bb6cdb605 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -163,9 +163,7 @@ describe('orchestration service client', () => { let finalOutput: AIMessageChunk | undefined; for await (const chunk of stream) { - finalOutput = finalOutput - ? finalOutput.concat(chunk) - : chunk; + finalOutput = finalOutput ? finalOutput.concat(chunk) : chunk; } expect(finalOutput).toMatchSnapshot(); }); From 8968eb4989e21f6d804b48c35591c82c8d04c6a1 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 13:15:53 +0200 Subject: [PATCH 55/67] chore: fix tests --- packages/langchain/src/orchestration/util.test.ts | 4 +--- .../src/orchestration-stream-chunk-response.test.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/langchain/src/orchestration/util.test.ts b/packages/langchain/src/orchestration/util.test.ts index cd6112ad8..9a01c47e7 100644 --- a/packages/langchain/src/orchestration/util.test.ts +++ b/packages/langchain/src/orchestration/util.test.ts @@ -349,9 +349,7 @@ describe('mapOrchestrationChunkToLangChainMessageChunk', () => { const mockChunk = new OrchestrationStreamChunkResponse(mockData); jest.spyOn(mockChunk, 'getDeltaContent').mockReturnValue(content); - jest - .spyOn(mockChunk, 'getDeltaToolCallChunks') - .mockReturnValue(toolCallChunks); + jest.spyOn(mockChunk, 'getDeltaToolCalls').mockReturnValue(toolCallChunks); return mockChunk; } diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts index bfa75e93c..79b92a145 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts @@ -83,7 +83,7 @@ describe('Orchestration chat completion stream chunk response', () => { it('should return delta tool call chunks with default index 0', () => { const toolCallChunks = - orchestrationStreamChunkResponses.toolCallResponse.getDeltaToolCallChunks(); + orchestrationStreamChunkResponses.toolCallResponse.getDeltaToolCalls(); expect(toolCallChunks).toBeDefined(); expect(toolCallChunks).toHaveLength(1); From dd9c34fe5b2918c20959fcf8af69621c155cff47 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 13:25:03 +0200 Subject: [PATCH 56/67] chore: use mockInference instead of inference with resilience --- .../src/orchestration/client.test.ts | 63 ++++++++++++++----- 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index bb6cdb605..76c7286d0 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -73,17 +73,6 @@ describe('orchestration service client', () => { ); } - function mockStreamInferenceWithResilience( - response: any = mockResponseStream, - resilience: { - retry?: number; - delay?: number; - } = { retry: 0 }, - status: number = 200 - ) { - mockInferenceWithResilience(response, resilience, status, true); - } - const config: LangchainOrchestrationModuleConfig = { llm: { model_name: 'gpt-4o', @@ -156,7 +145,18 @@ describe('orchestration service client', () => { }, 1000); it('supports streaming responses', async () => { - mockStreamInferenceWithResilience(); + mockInference( + { + data: constructCompletionPostRequest(config, { messages: [] }, true) + }, + { + data: mockResponseStream, + status: 200 + }, + { + url: 'inference/deployments/1234/completion' + } + ); const client = new OrchestrationClient(config); const stream = await client.stream([]); @@ -169,7 +169,18 @@ describe('orchestration service client', () => { }); it('streams and aborts with a signal', async () => { - mockStreamInferenceWithResilience(); + mockInference( + { + data: constructCompletionPostRequest(config, { messages: [] }, true) + }, + { + data: mockResponseStream, + status: 200 + }, + { + url: 'inference/deployments/1234/completion' + } + ); const client = new OrchestrationClient(config); const controller = new AbortController(); const { signal } = controller; @@ -184,7 +195,18 @@ describe('orchestration service client', () => { }, 1000); it('streams with a callback', async () => { - mockStreamInferenceWithResilience(); + mockInference( + { + data: constructCompletionPostRequest(config, { messages: [] }, true) + }, + { + data: mockResponseStream, + status: 200 + }, + { + url: 'inference/deployments/1234/completion' + } + ); let tokenCount = 0; const callbackHandler = { handleLLMNewToken: jest.fn().mockImplementation(() => { @@ -211,7 +233,18 @@ describe('orchestration service client', () => { }); it('supports streaming responses with tool calls', async () => { - mockStreamInferenceWithResilience(mockResponseStreamToolCalls); + mockInference( + { + data: constructCompletionPostRequest(config, { messages: [] }, true) + }, + { + data: mockResponseStreamToolCalls, + status: 200 + }, + { + url: 'inference/deployments/1234/completion' + } + ); const client = new OrchestrationClient(config); const stream = await client.stream([]); From 146825f1e40f69035cbc7dcce6eb899ce3681177 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Thu, 22 May 2025 14:59:57 +0200 Subject: [PATCH 57/67] Update packages/langchain/src/orchestration/util.ts Co-authored-by: Tom Frenken <54979414+tomfrenken@users.noreply.github.com> --- packages/langchain/src/orchestration/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index f7fc2eeef..0ad65a92b 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -198,7 +198,7 @@ export function mapOutputToChatResult( /** * Converts orchestration stream chunk to a LangChain message chunk. - * @param chunk- The orchestration stream chunk. + * @param chunk - The orchestration stream chunk. * @returns An {@link AIMessageChunk} * @internal */ From 80f1e7d355e45048ac179e6a3ae7e41125654755 Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Thu, 22 May 2025 15:01:11 +0200 Subject: [PATCH 58/67] Update packages/langchain/src/orchestration/util.ts Co-authored-by: Tom Frenken <54979414+tomfrenken@users.noreply.github.com> --- packages/langchain/src/orchestration/util.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 0ad65a92b..8b9863321 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -265,6 +265,7 @@ export function setTokenUsage( * @internal */ // eslint-disable-next-line @typescript-eslint/no-unused-vars +// TODO: Remove after https://github.com/SAP/ai-sdk-js-backlog/issues/321 export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { prompt: number; completion: number; From d08f4a8d8d9af6875f055ad785098394478162b9 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 15:04:02 +0200 Subject: [PATCH 59/67] chore: review comment --- packages/langchain/src/orchestration/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/util.ts b/packages/langchain/src/orchestration/util.ts index 8b9863321..c136f6c6e 100644 --- a/packages/langchain/src/orchestration/util.ts +++ b/packages/langchain/src/orchestration/util.ts @@ -264,8 +264,8 @@ export function setTokenUsage( * @returns An object with prompt and completion indices. * @internal */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars // TODO: Remove after https://github.com/SAP/ai-sdk-js-backlog/issues/321 +// eslint-disable-next-line @typescript-eslint/no-unused-vars export function computeTokenIndices(chunk: OrchestrationStreamChunkResponse): { prompt: number; completion: number; From 1eac3c71226d3d264a86a4196867d7be83107263 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 15:23:07 +0200 Subject: [PATCH 60/67] chore: Remove test file and adapt --- .../__snapshots__/client.test.ts.snap | 62 ++++++++++++++----- .../src/orchestration/client.test.ts | 9 ++- ...at-completion-stream-chunks-tool-calls.txt | 13 ---- 3 files changed, 51 insertions(+), 33 deletions(-) delete mode 100644 test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt diff --git a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap index ac79169a4..5b3fac291 100644 --- a/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap +++ b/packages/langchain/src/orchestration/__snapshots__/client.test.ts.snap @@ -248,60 +248,88 @@ exports[`orchestration service client supports streaming responses with tool cal { "delta": { "content": "", - "role": "assistantassistantassistantassistantassistant", + "role": "assistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistant", "tool_calls": [ { "function": { - "arguments": "{"temperature":20", - "name": "convert_temperature_to_fahrenheit", + "arguments": "{"a": 2, "b": 3}", + "name": "add", }, - "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", + "id": "call_HPgxxSmD2ctYfcJ3gp1JBc7i", "index": 0, "type": "function", }, + { + "function": { + "arguments": "{"a": 2, "b": 3}", + "name": "multiply", + }, + "id": "call_PExve0Dd9hxD8hOk4Uhr1yhO", + "index": 1, + "type": "function", + }, ], }, + "finish_reason": "tool_calls", "index": 0, }, ], - "created": 1747305948, - "id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZAchatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", - "model": "gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06", - "object": "chat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunk", - "system_fingerprint": "fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0", + "created": 1747660479, + "id": "chatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjdchatcmpl-BYucBnhXmLZ6QGcjM6Ac76HKLibjd", + "model": "gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06gpt-4o-2024-08-06", + "object": "chat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunkchat.completion.chunk", + "system_fingerprint": "fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0fp_ee1d74bde0", }, "templating": [ { - "content": "Convert 20 degrees Celsius to Fahrenheit.", + "content": "Add 2 and 3, and multiply 2 and 3.", "role": "user", }, ], }, - "request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc0c9186b6-be86-40d8-9745-ec81dd6351bc", + "request_id": "b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05b851a041-7bcd-463f-881a-7cb40526bc05", }, "content": "", "id": undefined, "invalid_tool_calls": [], "response_metadata": { "completion": 0, + "finish_reason": "tool_calls", "prompt": 0, }, "tool_call_chunks": [ { - "args": "{"temperature":20", - "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", + "args": "{"a": 2, "b": 3}", + "id": "call_HPgxxSmD2ctYfcJ3gp1JBc7i", "index": 0, - "name": "convert_temperature_to_fahrenheit", + "name": "add", + "type": "tool_call_chunk", + }, + { + "args": "{"a": 2, "b": 3}", + "id": "call_PExve0Dd9hxD8hOk4Uhr1yhO", + "index": 1, + "name": "multiply", "type": "tool_call_chunk", }, ], "tool_calls": [ { "args": { - "temperature": 20, + "a": 2, + "b": 3, + }, + "id": "call_HPgxxSmD2ctYfcJ3gp1JBc7i", + "name": "add", + "type": "tool_call", + }, + { + "args": { + "a": 2, + "b": 3, }, - "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", - "name": "convert_temperature_to_fahrenheit", + "id": "call_PExve0Dd9hxD8hOk4Uhr1yhO", + "name": "multiply", "type": "tool_call", }, ], diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index 76c7286d0..f0923c18f 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -41,7 +41,7 @@ describe('orchestration service client', () => { ); mockResponseStreamToolCalls = await parseFileToString( 'orchestration', - 'orchestration-chat-completion-stream-chunks-tool-calls.txt' + 'orchestration-chat-completion-stream-multiple-tools-chunks.txt' ); }); @@ -254,8 +254,11 @@ describe('orchestration service client', () => { finalOutput = finalOutput ? finalOutput.concat(chunk) : chunk; } const completeToolCall: ToolCall = finalOutput!.tool_calls![0]; - expect(completeToolCall.name).toEqual('convert_temperature_to_fahrenheit'); - expect(completeToolCall.args).toEqual({ temperature: 20 }); + expect(completeToolCall.name).toEqual('add'); + expect(completeToolCall.args).toEqual({ + a: 2, + b: 3 + }); expect(finalOutput).toMatchSnapshot(); }); }); diff --git a/test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt b/test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt deleted file mode 100644 index 5d643460e..000000000 --- a/test-util/data/orchestration/orchestration-chat-completion-stream-chunks-tool-calls.txt +++ /dev/null @@ -1,13 +0,0 @@ -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"templating": [{"content": "Convert 20 degrees Celsius to Fahrenheit.", "role": "user"}]}, "orchestration_result": {"id": "", "object": "", "created": 0, "model": "", "system_fingerprint": "", "choices": [{"index": 0, "delta": {"content": ""}}]}} - -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", "type": "function", "function": {"name": "convert_temperature_to_fahrenheit", "arguments": ""}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "id": "call_O8w2vPQ7pVJBKD4srms1UeOZ", "type": "function", "function": {"name": "convert_temperature_to_fahrenheit", "arguments": ""}}]}}]}} - -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "{\""}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "{\""}}]}}]}} - -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "temperature"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "temperature"}}]}}]}} - -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "\":"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "\":"}}]}}]}} - -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "20"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "20"}}]}}]}} - -data: {"request_id": "0c9186b6-be86-40d8-9745-ec81dd6351bc", "module_results": {"llm": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "}"}}]}}]}}, "orchestration_result": {"id": "chatcmpl-BXQNwHanrLdjGaVrKksL8XRUtSDZA", "object": "chat.completion.chunk", "created": 1747305948, "model": "gpt-4o-2024-08-06", "system_fingerprint": "fp_ee1d74bde0", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [{"index": 0, "function": {"arguments": "}"}}]}}]}} \ No newline at end of file From 9003b02ba4e1dfac2f6c2f01e76f8f2e8213fdfe Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 15:39:34 +0200 Subject: [PATCH 61/67] chore: adapt sample code --- sample-code/src/server.ts | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/sample-code/src/server.ts b/sample-code/src/server.ts index eb6338e5e..38d9bfdcf 100644 --- a/sample-code/src/server.ts +++ b/sample-code/src/server.ts @@ -61,6 +61,7 @@ import { createPromptTemplate, deletePromptTemplate } from './prompt-registry.js'; +import type { AIMessageChunk } from '@langchain/core/messages'; import type { RetievalPerFilterSearchResult } from '@sap-ai-sdk/document-grounding'; import type { AiDeploymentStatus } from '@sap-ai-sdk/ai-api'; import type { OrchestrationResponse } from '@sap-ai-sdk/orchestration'; @@ -482,23 +483,29 @@ app.get('/langchain/stream-orchestration', async (req, res) => { res.end(); }); + let finalResult: AIMessageChunk | undefined; for await (const chunk of stream) { if (!connectionAlive) { break; } res.write(chunk.content + '\n'); - if (connectionAlive && chunk.usage_metadata) { - res.write('\n\n---------------------------\n'); - res.write( - `Finish reason: ${chunk.response_metadata?.finish_reason}\n` - ); - res.write('Token usage:\n'); - res.write( - ` - Completion tokens: ${chunk.usage_metadata?.output_tokens}\n` - ); - res.write(` - Prompt tokens: ${chunk.usage_metadata?.input_tokens}\n`); - res.write(` - Total tokens: ${chunk.usage_metadata?.total_tokens}\n`); - } + finalResult = finalResult ? finalResult.concat(chunk) : chunk; + } + if (connectionAlive && finalResult?.usage_metadata) { + res.write('\n\n---------------------------\n'); + res.write( + `Finish reason: ${finalResult.response_metadata?.finish_reason}\n` + ); + res.write('Token usage:\n'); + res.write( + ` - Completion tokens: ${finalResult.usage_metadata?.output_tokens}\n` + ); + res.write( + ` - Prompt tokens: ${finalResult.usage_metadata?.input_tokens}\n` + ); + res.write( + ` - Total tokens: ${finalResult.usage_metadata?.total_tokens}\n` + ); } } catch (error: any) { sendError(res, error, false); From 92ea3b3671103c3f518814ab4e57f5f775865c77 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 15:53:20 +0200 Subject: [PATCH 62/67] chore: address review comment --- sample-code/src/langchain-orchestration.ts | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/sample-code/src/langchain-orchestration.ts b/sample-code/src/langchain-orchestration.ts index 8e05ad842..fb63add3d 100644 --- a/sample-code/src/langchain-orchestration.ts +++ b/sample-code/src/langchain-orchestration.ts @@ -183,28 +183,25 @@ export async function invokeLangGraphChain(): Promise { export async function streamChain( controller = new AbortController() ): Promise> { - // TODO: Remove template and use messages during https://github.com/SAP/ai-sdk-js-backlog/issues/317 const orchestrationConfig: LangchainOrchestrationModuleConfig = { llm: { model_name: 'gpt-4o' - }, - templating: { - template: [ - { - role: 'user', - content: 'Write a 100 word explanation about {{?topic}}' - } - ] } }; const client = new OrchestrationClient(orchestrationConfig); - return client.stream([], { - inputParams: { - topic: 'SAP Cloud SDK and its capabilities' - }, - signal: controller.signal - }); + return client.stream( + [ + { + role: 'user', + content: + 'Write a 100 word explanation about SAP Cloud SDK and its capabilities' + } + ], + { + signal: controller.signal + } + ); } /** From 3df806ab14b54e593368c0e8f25ee5fa1f1fc8a8 Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 16:16:09 +0200 Subject: [PATCH 63/67] chore: improve comment --- packages/langchain/src/orchestration/client.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index d3d489c1c..817915202 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -173,7 +173,8 @@ export class OrchestrationClient extends BaseChatModel< generationInfo: { ...tokenIndices } }); - // Notify the run manager about the new token, some parameters are undefined as they are implicitly read from the context. + // Notify the run manager about the new token + // Some parameters(`_runId`, `_parentRunId`, `_tags`) are set as undefined as they are implicitly read from the context. await runManager?.handleLLMNewToken( content, tokenIndices, From b4dae2b5c62498f6a18e5f928db5c654154c8f6e Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Thu, 22 May 2025 17:01:06 +0200 Subject: [PATCH 64/67] chore: add test for testing timeout --- .../langchain/src/orchestration/client.test.ts | 18 ++++++++++++++++++ packages/langchain/src/orchestration/client.ts | 18 +++++++++++------- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/langchain/src/orchestration/client.test.ts b/packages/langchain/src/orchestration/client.test.ts index f0923c18f..55ddbb9e0 100644 --- a/packages/langchain/src/orchestration/client.test.ts +++ b/packages/langchain/src/orchestration/client.test.ts @@ -168,6 +168,24 @@ describe('orchestration service client', () => { expect(finalOutput).toMatchSnapshot(); }); + it('throws when delay exceeds timeout during streaming', async () => { + mockInferenceWithResilience(mockResponseStream, { delay: 2000 }, 200, true); + + let finalOutput: AIMessageChunk | undefined; + const client = new OrchestrationClient(config); + try { + const stream = await client.stream([], { timeout: 1000 }); + for await (const chunk of stream) { + finalOutput = finalOutput ? finalOutput.concat(chunk) : chunk; + } + } catch (e) { + expect(e).toEqual( + expect.objectContaining({ + stack: expect.stringMatching(/Timeout/) + }) + ); + } + }); it('streams and aborts with a signal', async () => { mockInference( { diff --git a/packages/langchain/src/orchestration/client.ts b/packages/langchain/src/orchestration/client.ts index 817915202..aa0a751db 100644 --- a/packages/langchain/src/orchestration/client.ts +++ b/packages/langchain/src/orchestration/client.ts @@ -148,13 +148,17 @@ export class OrchestrationClient extends BaseChatModel< this.destination ); - const response = await this.caller.call(() => - orchestrationClient.stream( - { messages: orchestrationMessages, inputParams }, - controller, - options.streamOptions, - customRequestConfig - ) + const response = await this.caller.callWithOptions( + { + signal: controller.signal + }, + () => + orchestrationClient.stream( + { messages: orchestrationMessages, inputParams }, + controller, + options.streamOptions, + customRequestConfig + ) ); for await (const chunk of response.stream) { From 79a3f1a008c038fc513c06c0809db9767f99678a Mon Sep 17 00:00:00 2001 From: KavithaSiva <32287936+KavithaSiva@users.noreply.github.com> Date: Fri, 23 May 2025 11:25:32 +0200 Subject: [PATCH 65/67] Update packages/orchestration/src/orchestration-stream-chunk-response.test.ts Co-authored-by: Tom Frenken <54979414+tomfrenken@users.noreply.github.com> --- .../src/orchestration-stream-chunk-response.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts index 79b92a145..61ceebe5c 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts @@ -5,7 +5,7 @@ describe('Orchestration chat completion stream chunk response', () => { let mockResponses: { tokenUsageAndFinishReasonResponse: any; deltaContentResponse: any; - toolCallResponse: any; + deltaToolCallResponse: any; }; let orchestrationStreamChunkResponses: { tokenUsageResponse: OrchestrationStreamChunkResponse; From cf0f01e8757c873932a3e5d9eaba79e2ef42b47d Mon Sep 17 00:00:00 2001 From: KavithaSiva Date: Fri, 23 May 2025 11:28:09 +0200 Subject: [PATCH 66/67] chore: fix test --- .../src/orchestration-stream-chunk-response.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts index 61ceebe5c..c82a1a9eb 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts @@ -11,7 +11,7 @@ describe('Orchestration chat completion stream chunk response', () => { tokenUsageResponse: OrchestrationStreamChunkResponse; finishReasonResponse: OrchestrationStreamChunkResponse; deltaContentResponse: OrchestrationStreamChunkResponse; - toolCallResponse: OrchestrationStreamChunkResponse; + deltaToolCallResponse: OrchestrationStreamChunkResponse; }; beforeAll(async () => { @@ -24,7 +24,7 @@ describe('Orchestration chat completion stream chunk response', () => { 'orchestration', 'orchestration-chat-completion-stream-chunk-response-delta-content.json' ), - toolCallResponse: await parseMockResponse( + deltaToolCallResponse: await parseMockResponse( 'orchestration', 'orchestration-chat-completion-stream-chunk-response-tool-call.json' ) @@ -39,8 +39,8 @@ describe('Orchestration chat completion stream chunk response', () => { deltaContentResponse: new OrchestrationStreamChunkResponse( mockResponses.deltaContentResponse ), - toolCallResponse: new OrchestrationStreamChunkResponse( - mockResponses.toolCallResponse + deltaToolCallResponse: new OrchestrationStreamChunkResponse( + mockResponses.deltaToolCallResponse ) }; }); @@ -83,7 +83,7 @@ describe('Orchestration chat completion stream chunk response', () => { it('should return delta tool call chunks with default index 0', () => { const toolCallChunks = - orchestrationStreamChunkResponses.toolCallResponse.getDeltaToolCalls(); + orchestrationStreamChunkResponses.deltaToolCallResponse.getDeltaToolCalls(); expect(toolCallChunks).toBeDefined(); expect(toolCallChunks).toHaveLength(1); @@ -97,7 +97,7 @@ describe('Orchestration chat completion stream chunk response', () => { it('should find choice by valid index', () => { const choice = - orchestrationStreamChunkResponses.toolCallResponse.findChoiceByIndex(0); + orchestrationStreamChunkResponses.deltaToolCallResponse.findChoiceByIndex(0); expect(choice).toBeDefined(); expect(choice?.index).toBe(0); From 69b66e935361c8f1122960ddcd760dbeec7ed99b Mon Sep 17 00:00:00 2001 From: cloud-sdk-js Date: Fri, 23 May 2025 09:29:07 +0000 Subject: [PATCH 67/67] fix: Changes from lint --- .../src/orchestration-stream-chunk-response.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts index c82a1a9eb..c0f5c5fe2 100644 --- a/packages/orchestration/src/orchestration-stream-chunk-response.test.ts +++ b/packages/orchestration/src/orchestration-stream-chunk-response.test.ts @@ -97,7 +97,9 @@ describe('Orchestration chat completion stream chunk response', () => { it('should find choice by valid index', () => { const choice = - orchestrationStreamChunkResponses.deltaToolCallResponse.findChoiceByIndex(0); + orchestrationStreamChunkResponses.deltaToolCallResponse.findChoiceByIndex( + 0 + ); expect(choice).toBeDefined(); expect(choice?.index).toBe(0);