diff --git a/providers/openai/language_model_hooks.go b/providers/openai/language_model_hooks.go index c4ee8efaf..d5169cddd 100644 --- a/providers/openai/language_model_hooks.go +++ b/providers/openai/language_model_hooks.go @@ -297,7 +297,13 @@ func DefaultStreamProviderMetadataFunc(choice openai.ChatCompletionChoice, metad func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletionMessageParamUnion, []fantasy.CallWarning) { var messages []openai.ChatCompletionMessageParamUnion var warnings []fantasy.CallWarning + // All tool replies in a batch must precede synthetic user media. + var pendingToolMedia []openai.ChatCompletionMessageParamUnion for _, msg := range prompt { + if msg.Role != fantasy.MessageRoleTool { + messages = append(messages, pendingToolMedia...) + pendingToolMedia = nil + } switch msg.Role { case fantasy.MessageRoleSystem: var systemPromptParts []string @@ -585,7 +591,8 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio // OpenAI Chat Completions tool messages cannot carry image // or audio content directly; see ToolResultMediaMessages. mediaMessages, mediaWarnings := ToolResultMediaMessages(output, toolResultPart.ToolCallID) - messages = append(messages, mediaMessages...) + messages = append(messages, mediaMessages[0]) + pendingToolMedia = append(pendingToolMedia, mediaMessages[1:]...) warnings = append(warnings, mediaWarnings...) default: warnings = append(warnings, fantasy.CallWarning{ @@ -596,7 +603,7 @@ func DefaultToPrompt(prompt fantasy.Prompt, _, _ string) ([]openai.ChatCompletio } } } - return messages, warnings + return append(messages, pendingToolMedia...), warnings } // ToolResultMediaMessages maps a tool-result media output to the chat diff --git a/providers/openai/tool_result_media_order_test.go b/providers/openai/tool_result_media_order_test.go new file mode 100644 index 000000000..ebd501eeb --- /dev/null +++ b/providers/openai/tool_result_media_order_test.go @@ -0,0 +1,104 @@ +package openai_test + +import ( + "testing" + + "charm.land/fantasy" + "charm.land/fantasy/providers/openai" + "charm.land/fantasy/providers/openaicompat" + openaisdk "github.com/openai/openai-go/v3" + "github.com/stretchr/testify/require" +) + +func TestToolMediaFollowsToolResultBatch(t *testing.T) { + t.Parallel() + + for _, provider := range []struct { + name string + toPrompt func(fantasy.Prompt, string, string) ([]openaisdk.ChatCompletionMessageParamUnion, []fantasy.CallWarning) + }{ + {name: "OpenAI", toPrompt: openai.DefaultToPrompt}, + {name: "OpenAICompatible", toPrompt: openaicompat.ToPromptFunc}, + } { + t.Run(provider.name, func(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + separateMessages bool + secondMedia bool + followup bool + }{ + {name: "SeparateMessagesMixed", separateMessages: true}, + {name: "SeparateMessagesTwoMedia", separateMessages: true, secondMedia: true}, + {name: "SameMessageMixed"}, + {name: "SameMessageTwoMedia", secondMedia: true}, + {name: "BeforeFollowingMessages", separateMessages: true, secondMedia: true, followup: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + media := fantasy.ToolResultOutputContentMedia{Data: "AAEC", MediaType: "image/png", Text: "first result"} + results := []fantasy.MessagePart{ + fantasy.ToolResultPart{ToolCallID: "call-a", Output: media}, + fantasy.ToolResultPart{ToolCallID: "call-b", Output: fantasy.ToolResultOutputContentText{Text: "second result"}}, + } + imageURLs := []string{"data:image/png;base64,AAEC"} + if tc.secondMedia { + media.Data, media.Text = "AwQF", "second result" + results[1] = fantasy.ToolResultPart{ToolCallID: "call-b", Output: media} + imageURLs = append(imageURLs, "data:image/png;base64,AwQF") + } + prompt := fantasy.Prompt{{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ToolCallID: "call-a", ToolName: "first", Input: "{}"}, + fantasy.ToolCallPart{ToolCallID: "call-b", ToolName: "second", Input: "{}"}, + }, + }} + if tc.separateMessages { + for _, result := range results { + prompt = append(prompt, fantasy.Message{Role: fantasy.MessageRoleTool, Content: []fantasy.MessagePart{result}}) + } + } else { + prompt = append(prompt, fantasy.Message{Role: fantasy.MessageRoleTool, Content: results}) + } + if tc.followup { + prompt = append(prompt, + fantasy.Message{Role: fantasy.MessageRoleAssistant, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "done"}}}, + fantasy.Message{Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "thanks"}}}, + ) + } + + messages, warnings := provider.toPrompt(prompt, "", "") + require.Empty(t, warnings) + wantLen := 3 + len(imageURLs) + if tc.followup { + wantLen += 2 + } + require.Len(t, messages, wantLen) + require.NotNil(t, messages[0].OfAssistant) + require.Len(t, messages[0].OfAssistant.ToolCalls, 2) + for i, want := range []struct{ id, text string }{{"call-a", "first result"}, {"call-b", "second result"}} { + tool := messages[i+1].OfTool + require.NotNil(t, tool, "all tool results must precede synthetic user media") + require.Equal(t, want.id, tool.ToolCallID) + require.Equal(t, want.text, tool.Content.OfString.Value) + } + for i, url := range imageURLs { + user := messages[3+i].OfUser + require.NotNil(t, user) + require.Len(t, user.Content.OfArrayOfContentParts, 1) + image := user.Content.OfArrayOfContentParts[0].OfImageURL + require.NotNil(t, image) + require.Equal(t, url, image.ImageURL.URL) + } + if tc.followup { + require.NotNil(t, messages[wantLen-2].OfAssistant) + require.Equal(t, "done", messages[wantLen-2].OfAssistant.Content.OfString.Value) + require.NotNil(t, messages[wantLen-1].OfUser) + require.Equal(t, "thanks", messages[wantLen-1].OfUser.Content.OfString.Value) + } + }) + } + }) + } +} diff --git a/providers/openaicompat/language_model_hooks.go b/providers/openaicompat/language_model_hooks.go index b7d7ff5d1..daa616d5a 100644 --- a/providers/openaicompat/language_model_hooks.go +++ b/providers/openaicompat/language_model_hooks.go @@ -160,8 +160,14 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio var messages []openaisdk.ChatCompletionMessageParamUnion var warnings []fantasy.CallWarning hasReasoning := false + // All tool replies in a batch must precede synthetic user media. + var pendingToolMedia []openaisdk.ChatCompletionMessageParamUnion for _, msg := range prompt { + if msg.Role != fantasy.MessageRoleTool { + messages = append(messages, pendingToolMedia...) + pendingToolMedia = nil + } switch msg.Role { case fantasy.MessageRoleSystem: var blocks []openaisdk.ChatCompletionContentPartTextParam @@ -502,7 +508,8 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio // helper, which emits a text tool message plus a synthetic // user message holding the media. mediaMessages, mediaWarnings := openai.ToolResultMediaMessages(output, toolResultPart.ToolCallID) - messages = append(messages, mediaMessages...) + messages = append(messages, mediaMessages[0]) + pendingToolMedia = append(pendingToolMedia, mediaMessages[1:]...) warnings = append(warnings, mediaWarnings...) default: warnings = append(warnings, fantasy.CallWarning{ @@ -513,7 +520,7 @@ func ToPromptFunc(prompt fantasy.Prompt, _, _ string) ([]openaisdk.ChatCompletio } } } - return messages, warnings + return append(messages, pendingToolMedia...), warnings } // toolResultMediaUserPart maps a tool-result media output to an OpenAI chat