Skip to content

Commit 3d37106

Browse files
xrmxxrmx
authored
botocore: handle tool calls with Amazon nova with Bedrock InvokeModel* APIs (#3385)
* botocore: test invokemodel tool calls against amazon nova * botocore: handle amazon nova tool calls events for InvokeModelWithResponseStream * Update documentation * Update Changelog * Please pylint * Cleanup * Reduce branches in tool calls tests Instead pass down a per-model object that implements the peculiar part. And fix recording of amazon.nova stream no content test. * Move stream content extraction for tool calls tests to a couple of helpers
1 parent dde065b commit 3d37106

14 files changed

+3209
-1134
lines changed

‎CHANGELOG.md

+4
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1111
1212
## Unreleased
1313

14+
### Added
15+
1416
- `opentelemetry-instrumentation-asyncclick`: new instrumentation to trace asyncclick commands
1517
([#3319](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3319))
18+
- `opentelemetry-instrumentation-botocore` Add support for GenAI tool events using Amazon Nova models and `InvokeModel*` APIs
19+
([#3385](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3385))
1620

1721
### Fixed
1822

‎instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/__init__.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,9 @@ def response_hook(span, service_name, operation_name, result):
109109
- Amazon Nova models
110110
- Anthropic Claude
111111
112-
There is no support for tool calls with Amazon Models for the InvokeModel and InvokeModelWithResponseStream APIs.
112+
Tool calls with InvokeModel and InvokeModelWithResponseStream APIs are supported with:
113+
- Amazon Nova models
114+
- Anthropic Claude 3+
113115
"""
114116

115117
import logging

‎instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock_utils.py

+23-1
Original file line numberDiff line numberDiff line change
@@ -216,25 +216,47 @@ def _process_amazon_titan_chunk(self, chunk):
216216

217217
def _process_amazon_nova_chunk(self, chunk):
218218
# pylint: disable=too-many-branches
219-
# TODO: handle tool calls!
220219
if "messageStart" in chunk:
221220
# {'messageStart': {'role': 'assistant'}}
222221
if chunk["messageStart"].get("role") == "assistant":
223222
self._record_message = True
224223
self._message = {"role": "assistant", "content": []}
225224
return
226225

226+
if "contentBlockStart" in chunk:
227+
# {'contentBlockStart': {'start': {'toolUse': {'toolUseId': 'id', 'name': 'name'}}, 'contentBlockIndex': 31}}
228+
if self._record_message:
229+
self._message["content"].append(self._content_block)
230+
231+
start = chunk["contentBlockStart"].get("start", {})
232+
if "toolUse" in start:
233+
self._content_block = start
234+
else:
235+
self._content_block = {}
236+
return
237+
227238
if "contentBlockDelta" in chunk:
228239
# {'contentBlockDelta': {'delta': {'text': "Hello"}, 'contentBlockIndex': 0}}
240+
# {'contentBlockDelta': {'delta': {'toolUse': {'input': '{"location":"San Francisco"}'}}, 'contentBlockIndex': 31}}
229241
if self._record_message:
230242
delta = chunk["contentBlockDelta"].get("delta", {})
231243
if "text" in delta:
232244
self._content_block.setdefault("text", "")
233245
self._content_block["text"] += delta["text"]
246+
elif "toolUse" in delta:
247+
self._content_block.setdefault("toolUse", {})
248+
self._content_block["toolUse"]["input"] = json.loads(
249+
delta["toolUse"]["input"]
250+
)
234251
return
235252

236253
if "contentBlockStop" in chunk:
237254
# {'contentBlockStop': {'contentBlockIndex': 0}}
255+
if self._record_message:
256+
# create a new content block only for tools
257+
if "toolUse" in self._content_block:
258+
self._message["content"].append(self._content_block)
259+
self._content_block = {}
238260
return
239261

240262
if "messageStop" in chunk:

‎instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_invoke_model_no_content_tool_call[amazon.nova].yaml

+291
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
interactions:
2+
- request:
3+
body: |-
4+
{
5+
"messages": [
6+
{
7+
"role": "user",
8+
"content": [
9+
{
10+
"text": "What is the weather in Seattle and San Francisco today? Please expect one tool call for Seattle and one for San Francisco"
11+
}
12+
]
13+
}
14+
],
15+
"inferenceConfig": {
16+
"max_new_tokens": 1000
17+
},
18+
"schemaVersion": "messages-v1",
19+
"toolConfig": {
20+
"tools": [
21+
{
22+
"toolSpec": {
23+
"name": "get_current_weather",
24+
"description": "Get the current weather in a given location.",
25+
"inputSchema": {
26+
"json": {
27+
"type": "object",
28+
"properties": {
29+
"location": {
30+
"type": "string",
31+
"description": "The name of the city"
32+
}
33+
},
34+
"required": [
35+
"location"
36+
]
37+
}
38+
}
39+
}
40+
}
41+
]
42+
}
43+
}
44+
headers:
45+
Content-Length:
46+
- '552'
47+
User-Agent:
48+
- Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64
49+
lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56
50+
X-Amz-Date:
51+
- 20250325T130901Z
52+
X-Amz-Security-Token:
53+
- test_aws_security_token
54+
X-Amzn-Trace-Id:
55+
- Root=1-2541300b-2dbeda14c09ccd3d3b041879;Parent=1fc006fd156b7383;Sampled=1
56+
amz-sdk-invocation-id:
57+
- 09f5a7ed-4163-46ec-a974-000c5662062a
58+
amz-sdk-request:
59+
- attempt=1
60+
authorization:
61+
- Bearer test_aws_authorization
62+
method: POST
63+
uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.nova-micro-v1%3A0/invoke
64+
response:
65+
body:
66+
string: |-
67+
{
68+
"output": {
69+
"message": {
70+
"content": [
71+
{
72+
"text": "<thinking> To provide the current weather information for both Seattle and San Francisco, I will need to use the `get_current_weather` tool separately for each city. I will start by calling the tool for Seattle and then proceed to call it for San Francisco.</thinking>\n"
73+
},
74+
{
75+
"toolUse": {
76+
"name": "get_current_weather",
77+
"toolUseId": "4d8acf92-6a04-42b6-aaf7-f68937a339ef",
78+
"input": {
79+
"location": "Seattle"
80+
}
81+
}
82+
},
83+
{
84+
"toolUse": {
85+
"name": "get_current_weather",
86+
"toolUseId": "db27bd9d-9f7d-4783-aa8b-59871e00e0bc",
87+
"input": {
88+
"location": "San Francisco"
89+
}
90+
}
91+
}
92+
],
93+
"role": "assistant"
94+
}
95+
},
96+
"stopReason": "tool_use",
97+
"usage": {
98+
"inputTokens": 427,
99+
"outputTokens": 172,
100+
"totalTokens": 599,
101+
"cacheReadInputTokenCount": 0,
102+
"cacheWriteInputTokenCount": 0
103+
}
104+
}
105+
headers:
106+
Connection:
107+
- keep-alive
108+
Content-Type:
109+
- application/json
110+
Date:
111+
- Tue, 25 Mar 2025 13:09:02 GMT
112+
Set-Cookie: test_set_cookie
113+
X-Amzn-Bedrock-Cache-Read-Input-Token-Count:
114+
- '0'
115+
X-Amzn-Bedrock-Cache-Write-Input-Token-Count:
116+
- '0'
117+
X-Amzn-Bedrock-Input-Token-Count:
118+
- '427'
119+
X-Amzn-Bedrock-Invocation-Latency:
120+
- '430'
121+
X-Amzn-Bedrock-Output-Token-Count:
122+
- '172'
123+
x-amzn-RequestId:
124+
- adebeba4-5940-4c34-87a1-46cd45646bbe
125+
status:
126+
code: 200
127+
message: OK
128+
- request:
129+
body: |-
130+
{
131+
"messages": [
132+
{
133+
"role": "user",
134+
"content": [
135+
{
136+
"text": "What is the weather in Seattle and San Francisco today? Please expect one tool call for Seattle and one for San Francisco"
137+
}
138+
]
139+
},
140+
{
141+
"content": [
142+
{
143+
"text": "<thinking> To provide the current weather information for both Seattle and San Francisco, I will need to use the `get_current_weather` tool separately for each city. I will start by calling the tool for Seattle and then proceed to call it for San Francisco.</thinking>\n"
144+
},
145+
{
146+
"toolUse": {
147+
"name": "get_current_weather",
148+
"toolUseId": "4d8acf92-6a04-42b6-aaf7-f68937a339ef",
149+
"input": {
150+
"location": "Seattle"
151+
}
152+
}
153+
},
154+
{
155+
"toolUse": {
156+
"name": "get_current_weather",
157+
"toolUseId": "db27bd9d-9f7d-4783-aa8b-59871e00e0bc",
158+
"input": {
159+
"location": "San Francisco"
160+
}
161+
}
162+
}
163+
],
164+
"role": "assistant"
165+
},
166+
{
167+
"role": "user",
168+
"content": [
169+
{
170+
"toolResult": {
171+
"toolUseId": "4d8acf92-6a04-42b6-aaf7-f68937a339ef",
172+
"content": [
173+
{
174+
"json": {
175+
"weather": "50 degrees and raining"
176+
}
177+
}
178+
]
179+
}
180+
},
181+
{
182+
"toolResult": {
183+
"toolUseId": "db27bd9d-9f7d-4783-aa8b-59871e00e0bc",
184+
"content": [
185+
{
186+
"json": {
187+
"weather": "70 degrees and sunny"
188+
}
189+
}
190+
]
191+
}
192+
}
193+
]
194+
}
195+
],
196+
"inferenceConfig": {
197+
"max_new_tokens": 1000
198+
},
199+
"schemaVersion": "messages-v1",
200+
"toolConfig": {
201+
"tools": [
202+
{
203+
"toolSpec": {
204+
"name": "get_current_weather",
205+
"description": "Get the current weather in a given location.",
206+
"inputSchema": {
207+
"json": {
208+
"type": "object",
209+
"properties": {
210+
"location": {
211+
"type": "string",
212+
"description": "The name of the city"
213+
}
214+
},
215+
"required": [
216+
"location"
217+
]
218+
}
219+
}
220+
}
221+
}
222+
]
223+
}
224+
}
225+
headers:
226+
Content-Length:
227+
- '1439'
228+
User-Agent:
229+
- Boto3/1.35.56 md/Botocore#1.35.56 ua/2.0 os/linux#6.1.0-1034-oem md/arch#x86_64
230+
lang/python#3.10.12 md/pyimpl#CPython cfg/retry-mode#legacy Botocore/1.35.56
231+
X-Amz-Date:
232+
- 20250325T130902Z
233+
X-Amz-Security-Token:
234+
- test_aws_security_token
235+
X-Amzn-Trace-Id:
236+
- Root=1-2592d804-d8515e92a6320e1d293c3643;Parent=5406f9af558aaabe;Sampled=1
237+
amz-sdk-invocation-id:
238+
- 4ad9ba7e-df5a-4a01-a3a1-40fce5d29ef2
239+
amz-sdk-request:
240+
- attempt=1
241+
authorization:
242+
- Bearer test_aws_authorization
243+
method: POST
244+
uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.nova-micro-v1%3A0/invoke
245+
response:
246+
body:
247+
string: |-
248+
{
249+
"output": {
250+
"message": {
251+
"content": [
252+
{
253+
"text": "<thinking> I have received the weather information for both cities. Now I will compile this information and present it to the User.</thinking>\n\nThe current weather in Seattle is 50 degrees and it is raining. In San Francisco, the weather is 70 degrees and sunny."
254+
}
255+
],
256+
"role": "assistant"
257+
}
258+
},
259+
"stopReason": "end_turn",
260+
"usage": {
261+
"inputTokens": 576,
262+
"outputTokens": 57,
263+
"totalTokens": 633,
264+
"cacheReadInputTokenCount": 0,
265+
"cacheWriteInputTokenCount": 0
266+
}
267+
}
268+
headers:
269+
Connection:
270+
- keep-alive
271+
Content-Type:
272+
- application/json
273+
Date:
274+
- Tue, 25 Mar 2025 13:09:03 GMT
275+
Set-Cookie: test_set_cookie
276+
X-Amzn-Bedrock-Cache-Read-Input-Token-Count:
277+
- '0'
278+
X-Amzn-Bedrock-Cache-Write-Input-Token-Count:
279+
- '0'
280+
X-Amzn-Bedrock-Input-Token-Count:
281+
- '576'
282+
X-Amzn-Bedrock-Invocation-Latency:
283+
- '417'
284+
X-Amzn-Bedrock-Output-Token-Count:
285+
- '57'
286+
x-amzn-RequestId:
287+
- cedb9b10-f794-4ec4-ac67-f3e237511e62
288+
status:
289+
code: 200
290+
message: OK
291+
version: 1

0 commit comments

Comments
 (0)