build(deps): bump github.com/docker/docker from 25.0.5+incompatible to 25.0.6+incompatible

go.mod

@@ -65,7 +65,7 @@ require (
 	github.com/deepmap/oapi-codegen/v2 v2.1.0 // indirect
 	github.com/distribution/reference v0.5.0 // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
-	github.com/docker/docker v25.0.5+incompatible // indirect
+	github.com/docker/docker v25.0.6+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -171,8 +171,8 @@ github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5
 github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
 github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE=
-github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg=
+github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=

llms/anthropic/anthropicllm.go

@@ -211,11 +211,33 @@ func generateMessagesContent(ctx context.Context, o *LLM, messages []llms.Messag
 func toolsToTools(tools []llms.Tool) []anthropicclient.Tool {
 	toolReq := make([]anthropicclient.Tool, len(tools))
 	for i, tool := range tools {
-		toolReq[i] = anthropicclient.Tool{
+		anthropicTool := anthropicclient.Tool{
 			Name:        tool.Function.Name,
 			Description: tool.Function.Description,
 			InputSchema: tool.Function.Parameters,
+
+			Type: tool.Type,
+		}
+		if tool.Function.ProviderSpecific != nil {
+			if displayHeightPx, ok := tool.Function.ProviderSpecific["display_height_px"]; ok {
+				anthropicTool.DisplayHeightPx = displayHeightPx.(int)
+			}
+			if displayWidthPx, ok := tool.Function.ProviderSpecific["display_width_px"]; ok {
+				anthropicTool.DisplayWidthPx = displayWidthPx.(int)
+			}
+			if displayNumber, ok := tool.Function.ProviderSpecific["display_number"]; ok {
+				anthropicTool.DisplayNumber = displayNumber.(int)
+			}
+			if cacheControl, ok := tool.Function.ProviderSpecific["cache_control"]; ok {
+				anthropicTool.CacheControl = struct {
+					Type string "json:\"type,omitempty\""
+				}{
+					Type: cacheControl.(string),
+				}
+			}
 		}
+
+		toolReq[i] = anthropicTool
 	}
 	return toolReq
 }
@@ -320,6 +342,10 @@ func handleToolMessage(msg llms.MessageContent) (anthropicclient.ChatMessage, er
 			Content:   toolCallResponse.Content,
 		}
 
+		if toolCallResponse.MultiContent != nil {
+			toolContent.MultiContent = toolCallResponse.MultiContent
+		}
+
 		return anthropicclient.ChatMessage{
 			Role:    RoleUser,
 			Content: []anthropicclient.Content{toolContent},

llms/anthropic/internal/anthropicclient/messages.go

@@ -11,6 +11,8 @@ import (
 	"log"
 	"net/http"
 	"strings"
+
+	"github.com/tmc/langchaingo/llms"
 )
 
 var (
@@ -27,8 +29,8 @@ var (
 )
 
 type ChatMessage struct {
-	Role    string      `json:"role"`
-	Content interface{} `json:"content"`
+	Role    string `json:"role"`
+	Content any    `json:"content"`
 }
 
 type messagePayload struct {
@@ -50,6 +52,16 @@ type Tool struct {
 	Name        string `json:"name"`
 	Description string `json:"description,omitempty"`
 	InputSchema any    `json:"input_schema,omitempty"`
+
+	// The fields below are used for the built-in tools:
+	// https://docs.anthropic.com/en/docs/build-with-claude/computer-use#understand-anthropic-defined-tools
+	Type            string `json:"type"`
+	DisplayHeightPx int    `json:"display_height_px,omitempty"`
+	DisplayWidthPx  int    `json:"display_width_px,omitempty"`
+	DisplayNumber   int    `json:"display_number,omitempty"`
+	CacheControl    struct {
+		Type string `json:"type,omitempty"` // valid value: "ephemeral"
+	} `json:"cache_control,omitempty"`
 }
 
 // Content can be TextContent or ToolUseContent depending on the type.
@@ -80,7 +92,46 @@ func (tuc ToolUseContent) GetType() string {
 type ToolResultContent struct {
 	Type      string `json:"type"`
 	ToolUseID string `json:"tool_use_id"`
-	Content   string `json:"content"`
+
+	// The content of the message.
+	// This field is mutually exclusive with MultiContent.
+	Content string `json:"-"`
+
+	MultiContent []llms.ToolResultContentPart `json:"-"`
+}
+
+// json marshal ToolResultContent such that either Content or MultiContent is set, but not both.
+// the json key for both is "content"
+func (trc ToolResultContent) MarshalJSON() ([]byte, error) {
+	if trc.Content != "" && len(trc.MultiContent) > 0 {
+		return nil, fmt.Errorf("both Content and MultiContent cannot be set in ToolResultContents")
+	}
+
+	type alias ToolResultContent
+
+	if len(trc.MultiContent) > 0 {
+		result, err := json.Marshal(struct {
+			alias
+			Content []llms.ToolResultContentPart `json:"content"`
+		}{
+			alias:   (alias)(trc),
+			Content: trc.MultiContent,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("marshal multi content: %w", err)
+		}
+		// fmt.Printf("ToolResultContent json: %s\n", string(result))
+
+		return result, nil
+	}
+
+	return json.Marshal(struct {
+		alias
+		Content string `json:"content"`
+	}{
+		alias:   (alias)(trc),
+		Content: trc.Content,
+	})
 }
 
 func (trc ToolResultContent) GetType() string {

llms/generatecontent.go

@@ -113,7 +113,12 @@ type ToolCallResponse struct {
 	// Name is the name of the tool that was called.
 	Name string `json:"name"`
 	// Content is the textual content of the response.
+	// This field is mutually exclusive with MultiContent.
 	Content string `json:"content"`
+
+	// MultiContent is a list of content parts to use in the response.
+	// This may not be supported by all providers.
+	MultiContent []ToolResultContentPart
 }
 
 func (ToolCallResponse) isPart() {}
@@ -178,10 +183,31 @@ func ShowMessageContents(w io.Writer, msgs []MessageContent) {
 			case ToolCall:
 				fmt.Fprintf(w, "ToolCall ID=%v, Type=%v, Func=%v(%v)\n", pp.ID, pp.Type, pp.FunctionCall.Name, pp.FunctionCall.Arguments)
 			case ToolCallResponse:
-				fmt.Fprintf(w, "ToolCallResponse ID=%v, Name=%v, Content=%v\n", pp.ToolCallID, pp.Name, pp.Content)
+				fmt.Fprintf(w, "ToolCallResponse ID=%v, Name=%v, Content=%v, MultiContent=%v\n", pp.ToolCallID, pp.Name, pp.Content, pp.MultiContent)
 			default:
 				fmt.Fprintf(w, "unknown type %T\n", pp)
 			}
 		}
 	}
 }
+
+type ToolResultContentPart interface {
+	isToolResultContentPart()
+}
+
+type ImageToolContent struct {
+	Type   string             `json:"type"` // always "image"
+	Source ImageSourceContent `json:"source"`
+}
+
+func (ImageToolContent) isToolResultContentPart() {}
+
+type ImageSourceContent struct {
+	Type      string `json:"type"`       // always "base64"
+	MediaType string `json:"media_type"` // mime type, conventionally prefixed with "image/"
+	Data      string `json:"data"`
+}
+
+func (imc ImageSourceContent) String() string {
+	return "data:" + imc.MediaType + ";base64," + imc.Data
+}

llms/options.go

@@ -86,6 +86,8 @@ type FunctionDefinition struct {
 	Parameters any `json:"parameters,omitempty"`
 	// Strict is a flag to indicate if the function should be called strictly. Only used for openai llm structured output.
 	Strict bool `json:"strict,omitempty"`
+	// ProviderSpecific is a map of provider specific options.
+	ProviderSpecific map[string]any `json:"provider_specific,omitempty"`
 }
 
 // ToolChoice is a specific tool to use.