Skip to content

Commit 07dfd73

Browse files
kylecarbskylecarbs
committed
Add support for images
1 parent f82a685 commit 07dfd73

8 files changed

Lines changed: 309 additions & 288 deletions

File tree

‎anthropic.go‎

Lines changed: 103 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package aisdk
22

33
import (
4+
"encoding/base64"
45
"encoding/json"
56
"fmt"
7+
"strings"
68

79
"github.com/anthropics/anthropic-sdk-go"
810
"github.com/anthropics/anthropic-sdk-go/packages/ssestream"
@@ -48,156 +50,158 @@ func ToolsToAnthropic(tools []Tool) []anthropic.ToolUnionParam {
4850
// and user tool_result blocks.
4951
func MessagesToAnthropic(messages []Message) ([]anthropic.MessageParam, []anthropic.TextBlockParam, error) {
5052
anthropicMessages := []anthropic.MessageParam{}
51-
systemPrompts := []anthropic.TextBlockParam{}
5253

53-
// Iterate through messages and process them
54-
for i := 0; i < len(messages); i++ {
55-
message := messages[i]
54+
var systemPrompt []anthropic.TextBlockParam
5655

57-
if message.Role == "system" {
58-
// Handle system messages
59-
systemPrompts = append(systemPrompts, anthropic.TextBlockParam{Text: message.Content})
56+
for _, message := range messages {
57+
role := anthropic.MessageParamRoleAssistant
58+
content := []anthropic.ContentBlockParamUnion{}
59+
60+
switch message.Role {
61+
case "system":
62+
if len(systemPrompt) > 0 {
63+
return nil, nil, fmt.Errorf("multiple system messages found")
64+
}
6065
for _, part := range message.Parts {
6166
if part.Type == PartTypeText && part.Text != "" {
62-
systemPrompts = append(systemPrompts, anthropic.TextBlockParam{Text: part.Text})
67+
systemPrompt = append(systemPrompt, anthropic.TextBlockParam{
68+
Text: part.Text,
69+
})
6370
}
64-
// Ignore other parts in system messages for now
6571
}
66-
continue
67-
}
68-
69-
var role anthropic.MessageParamRole
70-
var currentContent []anthropic.ContentBlockParamUnion
71-
72-
if message.Role == "assistant" {
73-
role = anthropic.MessageParamRoleAssistant
74-
// Process parts for assistant message
72+
break
73+
case "assistant":
7574
for _, part := range message.Parts {
7675
switch part.Type {
7776
case PartTypeText:
78-
if part.Text != "" {
79-
currentContent = append(currentContent, anthropic.ContentBlockParamUnion{
80-
OfRequestTextBlock: &anthropic.TextBlockParam{Text: part.Text},
81-
})
82-
}
77+
content = append(content, anthropic.ContentBlockParamUnion{
78+
OfRequestTextBlock: &anthropic.TextBlockParam{
79+
Text: part.Text,
80+
},
81+
})
8382
case PartTypeToolInvocation:
8483
if part.ToolInvocation == nil {
8584
return nil, nil, fmt.Errorf("assistant message part has type tool-invocation but nil ToolInvocation field (ID: %s)", message.ID)
8685
}
87-
88-
// Add the tool *call* part
8986
argsJSON, err := json.Marshal(part.ToolInvocation.Args)
9087
if err != nil {
9188
return nil, nil, fmt.Errorf("marshalling tool input for call %s: %w", part.ToolInvocation.ToolCallID, err)
9289
}
93-
currentContent = append(currentContent, anthropic.ContentBlockParamUnion{
90+
content = append(content, anthropic.ContentBlockParamUnion{
9491
OfRequestToolUseBlock: &anthropic.ToolUseBlockParam{
9592
ID: part.ToolInvocation.ToolCallID,
96-
Name: part.ToolInvocation.ToolName,
9793
Input: json.RawMessage(argsJSON),
94+
Name: part.ToolInvocation.ToolName,
9895
},
9996
})
10097

101-
// If the state is Result, we need to immediately add a user message with the result
102-
if part.ToolInvocation.State == ToolInvocationStateResult {
103-
// Flush the current assistant message first
104-
if len(currentContent) > 0 {
105-
anthropicMessages = append(anthropicMessages, anthropic.MessageParam{
106-
Role: role,
107-
Content: currentContent,
98+
if part.ToolInvocation.State != ToolInvocationStateResult {
99+
continue
100+
}
101+
102+
// Tool Results are sent as a separate message, so we need to flush existing content here.
103+
anthropicMessages = append(anthropicMessages, anthropic.MessageParam{
104+
Role: role,
105+
Content: content,
106+
})
107+
content = nil
108+
109+
resultContent := []anthropic.ToolResultBlockParamContentUnion{}
110+
resultParts, err := toolResultToParts(part.ToolInvocation.Result)
111+
if err != nil {
112+
return nil, nil, fmt.Errorf("failed to convert tool call result to parts: %w", err)
113+
}
114+
for _, resultPart := range resultParts {
115+
switch resultPart.Type {
116+
case PartTypeText:
117+
resultContent = append(resultContent, anthropic.ToolResultBlockParamContentUnion{
118+
OfRequestTextBlock: &anthropic.TextBlockParam{Text: resultPart.Text},
119+
})
120+
case PartTypeFile:
121+
resultContent = append(resultContent, anthropic.ToolResultBlockParamContentUnion{
122+
OfRequestImageBlock: &anthropic.ImageBlockParam{
123+
Source: anthropic.ImageBlockParamSourceUnion{
124+
OfBase64ImageSource: &anthropic.Base64ImageSourceParam{
125+
Data: base64.StdEncoding.EncodeToString(resultPart.Data),
126+
MediaType: anthropic.Base64ImageSourceMediaType(resultPart.MimeType),
127+
},
128+
},
129+
},
108130
})
109-
currentContent = nil // Reset for the potential next message
110131
}
132+
}
111133

112-
// Now create the user message with the tool result
113-
resultBytes, err := json.Marshal(part.ToolInvocation.Result)
114-
if err != nil {
115-
return nil, nil, fmt.Errorf("marshalling tool result for call %s: %w", part.ToolInvocation.ToolCallID, err)
116-
}
117-
userResultContent := []anthropic.ContentBlockParamUnion{
134+
// Send the tool result as a separate message with the role as user.
135+
anthropicMessages = append(anthropicMessages, anthropic.MessageParam{
136+
Role: anthropic.MessageParamRoleUser,
137+
Content: []anthropic.ContentBlockParamUnion{
118138
{
119139
OfRequestToolResultBlock: &anthropic.ToolResultBlockParam{
120140
ToolUseID: part.ToolInvocation.ToolCallID,
121-
Content: []anthropic.ToolResultBlockParamContentUnion{
122-
{
123-
OfRequestTextBlock: &anthropic.TextBlockParam{Text: string(resultBytes)},
124-
},
125-
},
141+
Content: resultContent,
126142
},
127143
},
128-
}
129-
anthropicMessages = append(anthropicMessages, anthropic.MessageParam{
130-
Role: anthropic.MessageParamRoleUser, // Result must be in user role
131-
Content: userResultContent,
132-
})
133-
// Since we added the user message, effectively skip adding the current assistant message later
134-
role = "" // Mark role as processed
135-
}
136-
// TODO: Add support for other part types
144+
},
145+
})
146+
content = nil
137147
}
138148
}
139-
} else if message.Role == "user" || message.Role == "tool" {
149+
case "user":
140150
role = anthropic.MessageParamRoleUser
141-
// Process parts for user/tool message
142151
for _, part := range message.Parts {
143152
switch part.Type {
144153
case PartTypeText:
145-
if part.Text != "" {
146-
currentContent = append(currentContent, anthropic.ContentBlockParamUnion{
147-
OfRequestTextBlock: &anthropic.TextBlockParam{Text: part.Text},
148-
})
149-
}
150-
case PartTypeToolInvocation:
151-
if part.ToolInvocation == nil {
152-
return nil, nil, fmt.Errorf("user/tool message part has type tool-invocation but nil ToolInvocation field (ID: %s)", message.ID)
153-
}
154-
// User/tool role should only contain results
155-
if part.ToolInvocation.State != ToolInvocationStateResult {
156-
return nil, nil, fmt.Errorf("non-result tool invocation found in user/tool message (ID: %s, State: %s)", message.ID, part.ToolInvocation.State)
157-
}
158-
resultBytes, err := json.Marshal(part.ToolInvocation.Result)
159-
if err != nil {
160-
return nil, nil, fmt.Errorf("marshalling tool result for call %s: %w", part.ToolInvocation.ToolCallID, err)
161-
}
162-
currentContent = append(currentContent, anthropic.ContentBlockParamUnion{
163-
OfRequestToolResultBlock: &anthropic.ToolResultBlockParam{
164-
ToolUseID: part.ToolInvocation.ToolCallID,
165-
Content: []anthropic.ToolResultBlockParamContentUnion{
166-
{
167-
OfRequestTextBlock: &anthropic.TextBlockParam{Text: string(resultBytes)},
154+
content = append(content, anthropic.ContentBlockParamUnion{
155+
OfRequestTextBlock: &anthropic.TextBlockParam{Text: part.Text},
156+
})
157+
case PartTypeFile:
158+
content = append(content, anthropic.ContentBlockParamUnion{
159+
OfRequestImageBlock: &anthropic.ImageBlockParam{
160+
Source: anthropic.ImageBlockParamSourceUnion{
161+
OfBase64ImageSource: &anthropic.Base64ImageSourceParam{
162+
Data: base64.StdEncoding.EncodeToString(part.Data),
163+
MediaType: anthropic.Base64ImageSourceMediaType(part.MimeType),
168164
},
169165
},
170166
},
171167
})
172-
// TODO: Add support for other part types
168+
case PartTypeToolInvocation:
169+
return nil, nil, fmt.Errorf("user message part has type tool-invocation (ID: %s)", message.ID)
173170
}
174171
}
175-
// Add plain content as text block if no parts were processed
176-
if len(currentContent) == 0 && message.Content != "" {
177-
currentContent = append(currentContent, anthropic.ContentBlockParamUnion{
178-
OfRequestTextBlock: &anthropic.TextBlockParam{Text: message.Content},
179-
})
180-
}
181-
} else {
172+
default:
182173
return nil, nil, fmt.Errorf("unsupported message role encountered: %s", message.Role)
183174
}
184175

185-
// Add the processed message if it has content and role wasn't handled by tool result splitting
186-
if len(currentContent) > 0 && role != "" {
187-
// Check if the last message was of the same role, if so, merge content
188-
lastIdx := len(anthropicMessages) - 1
189-
if lastIdx >= 0 && anthropicMessages[lastIdx].Role == role {
190-
anthropicMessages[lastIdx].Content = append(anthropicMessages[lastIdx].Content, currentContent...)
191-
} else {
192-
anthropicMessages = append(anthropicMessages, anthropic.MessageParam{
193-
Role: role,
194-
Content: currentContent,
176+
if len(message.Attachments) > 0 {
177+
for _, attachment := range message.Attachments {
178+
// URLs typically have the mime prefixing as a URL.
179+
parts := strings.SplitN(attachment.URL, ",", 2)
180+
if len(parts) != 2 {
181+
return nil, nil, fmt.Errorf("invalid attachment URL: %s", attachment.URL)
182+
}
183+
content = append(content, anthropic.ContentBlockParamUnion{
184+
OfRequestImageBlock: &anthropic.ImageBlockParam{
185+
Source: anthropic.ImageBlockParamSourceUnion{
186+
OfBase64ImageSource: &anthropic.Base64ImageSourceParam{
187+
Data: parts[1],
188+
MediaType: anthropic.Base64ImageSourceMediaType(attachment.ContentType),
189+
},
190+
},
191+
},
195192
})
196193
}
197194
}
195+
if len(content) > 0 {
196+
anthropicMessages = append(anthropicMessages, anthropic.MessageParam{
197+
Role: role,
198+
Content: content,
199+
})
200+
content = nil
201+
}
198202
}
199203

200-
return anthropicMessages, systemPrompts, nil
204+
return anthropicMessages, systemPrompt, nil
201205
}
202206

203207
// AnthropicToDataStream pipes an Anthropic stream to a DataStream.

‎anthropic_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ data: {"type":"message_stop" }`
7575

7676
var acc aisdk.DataStreamAccumulator
7777
stream := aisdk.AnthropicToDataStream(typedStream)
78-
stream = stream.WithToolCalling(func(toolCall aisdk.ToolCall) any {
78+
stream = stream.WithToolCalling(func(toolCall aisdk.ToolCall) aisdk.ToolCallResult {
7979
return map[string]any{"message": "Message printed to the console"}
8080
})
8181
stream = stream.WithAccumulator(&acc)

‎demo/index.tsx‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ type model = keyof typeof modelToProvider;
1414
const Chat = () => {
1515
const [model, setModel] = useState<model>("gpt-4o");
1616
const [thinking, setThinking] = useState(false);
17+
const [files, setFiles] = useState<FileList | null>(null);
1718
const { messages, input, handleInputChange, handleSubmit, error } = useChat({
1819
api: "/api/chat",
1920
body: {
@@ -23,6 +24,17 @@ const Chat = () => {
2324
},
2425
});
2526

27+
const handleSubmitWithFiles = (e: React.FormEvent<HTMLFormElement>) => {
28+
e.preventDefault();
29+
if (files) {
30+
handleSubmit(e, {
31+
experimental_attachments: files,
32+
});
33+
} else {
34+
handleSubmit(e);
35+
}
36+
};
37+
2638
console.log(messages);
2739

2840
return (
@@ -163,7 +175,12 @@ const Chat = () => {
163175
})}
164176
</div>
165177

166-
<form onSubmit={handleSubmit}>
178+
<form onSubmit={handleSubmitWithFiles}>
179+
<input
180+
type="file"
181+
multiple
182+
onChange={(e) => setFiles(e.target.files || null)}
183+
/>
167184
<input
168185
value={input}
169186
onChange={handleInputChange}

‎demo/server.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func run(ctx context.Context) error {
6363
return
6464
}
6565

66-
handleToolCall := func(toolCall aisdk.ToolCall) any {
66+
handleToolCall := func(toolCall aisdk.ToolCall) aisdk.ToolCallResult {
6767
return map[string]string{
6868
"message": "It worked!",
6969
}

0 commit comments

Comments
 (0)