From 17da4b95dd3a074ccf341c8db7f2dfef36fc412b Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Mon, 26 Oct 2026 20:23:07 -0400 Subject: [PATCH] fix: forward Claude image blocks to OpenAI chat and Responses requests Co-Authored-By: Claude Sonnet 5.5 --- .../transformer/convert/claude_images_test.go | 116 ++++++++++++++++++ internal/transformer/convert/claude_openai.go | 18 ++- .../transformer/convert/claude_openai2.go | 4 + internal/transformer/convert/common.go | 22 ++++ 4 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 internal/transformer/convert/claude_images_test.go diff --git a/internal/transformer/convert/claude_images_test.go b/internal/transformer/convert/claude_images_test.go new file mode 100644 index 00000000..06bc2dbf --- /dev/null +++ b/internal/transformer/convert/claude_images_test.go @@ -0,0 +1,116 @@ +package convert + +import ( + "encoding/json" + "testing" +) + +const claudeReqWithImage = `{ + "model": "claude-sonnet-4-6", + "max_tokens": 1024, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "what is in this screenshot?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "QUJD"}} + ]}] +}` + +func TestClaudeReqToOpenAISendsImagesAsImageURLParts(t *testing.T) { + out, err := ClaudeReqToOpenAI([]byte(claudeReqWithImage), "gpt-test") + if err != nil { + t.Fatalf("ClaudeReqToOpenAI failed: %v", err) + } + var body struct { + Messages []struct { + Role string `json:"role"` + Content interface{} `json:"content"` + } `json:"messages"` + } + if err := json.Unmarshal(out, &body); err != nil { + t.Fatalf("converted request is not JSON: %v", err) + } + + parts, ok := body.Messages[0].Content.([]interface{}) + if !ok || len(parts) != 2 { + t.Fatalf("expected a content list with a text and an image part, got %v", body.Messages[0].Content) + } + text := parts[0].(map[string]interface{}) + if text["type"] != "text" || text["text"] != "what is in this screenshot?" { + t.Errorf("expected the text part first, got %v", text) + } + image := parts[1].(map[string]interface{}) + url, _ := image["image_url"].(map[string]interface{})["url"].(string) + if image["type"] != "image_url" || url != "data:image/png;base64,QUJD" { + t.Errorf("expected an image_url part with a data URL, got %v", image) + } +} + +func TestClaudeReqToOpenAIKeepsStringContentWithoutImages(t *testing.T) { + out, err := ClaudeReqToOpenAI([]byte(`{"model": "m", "messages": [{"role": "user", "content": [ + {"type": "text", "text": "hello "}, {"type": "text", "text": "world"} + ]}]}`), "gpt-test") + if err != nil { + t.Fatalf("ClaudeReqToOpenAI failed: %v", err) + } + var body struct { + Messages []struct { + Content interface{} `json:"content"` + } `json:"messages"` + } + if err := json.Unmarshal(out, &body); err != nil { + t.Fatalf("converted request is not JSON: %v", err) + } + if body.Messages[0].Content != "hello world" { + t.Errorf("expected the text joined into a string as before, got %v", body.Messages[0].Content) + } +} + +func TestClaudeReqToOpenAI2SendsImagesAsInputImage(t *testing.T) { + out, err := ClaudeReqToOpenAI2([]byte(claudeReqWithImage), "gpt-test") + if err != nil { + t.Fatalf("ClaudeReqToOpenAI2 failed: %v", err) + } + var body struct { + Input []struct { + Role string `json:"role"` + Content []map[string]interface{} `json:"content"` + } `json:"input"` + } + if err := json.Unmarshal(out, &body); err != nil { + t.Fatalf("converted request is not JSON: %v", err) + } + + if len(body.Input) != 1 || len(body.Input[0].Content) != 2 { + t.Fatalf("expected one message with a text and an image part, got %v", body.Input) + } + if body.Input[0].Content[0]["type"] != "input_text" { + t.Errorf("expected the text part first, got %v", body.Input[0].Content[0]) + } + image := body.Input[0].Content[1] + if image["type"] != "input_image" || image["image_url"] != "data:image/png;base64,QUJD" { + t.Errorf("expected an input_image part with a data URL, got %v", image) + } +} + +func TestClaudeImageURLSourceIsForwarded(t *testing.T) { + out, err := ClaudeReqToOpenAI([]byte(`{"model": "m", "messages": [{"role": "user", "content": [ + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}} + ]}]}`), "gpt-test") + if err != nil { + t.Fatalf("ClaudeReqToOpenAI failed: %v", err) + } + var body struct { + Messages []struct { + Content []map[string]interface{} `json:"content"` + } `json:"messages"` + } + if err := json.Unmarshal(out, &body); err != nil { + t.Fatalf("converted request is not JSON: %v", err) + } + if len(body.Messages) != 1 || len(body.Messages[0].Content) != 1 { + t.Fatalf("expected one message with one image part, got %v", body.Messages) + } + url, _ := body.Messages[0].Content[0]["image_url"].(map[string]interface{})["url"].(string) + if url != "https://example.com/a.png" { + t.Errorf("expected the image URL to be forwarded, got %q", url) + } +} diff --git a/internal/transformer/convert/claude_openai.go b/internal/transformer/convert/claude_openai.go index d9b23916..39b936dd 100644 --- a/internal/transformer/convert/claude_openai.go +++ b/internal/transformer/convert/claude_openai.go @@ -35,9 +35,11 @@ func ClaudeReqToOpenAI(claudeReq []byte, model string) ([]byte, error) { case []interface{}: // Check for tool_result blocks var textParts []string + var contentParts []interface{} // text and image parts in order, used when the message has images var toolCalls []transformer.OpenAIToolCall var toolResults []transformer.OpenAIMessage hasThinking := false + hasImage := false for _, block := range content { m, ok := block.(map[string]interface{}) @@ -48,6 +50,15 @@ func ClaudeReqToOpenAI(claudeReq []byte, model string) ([]byte, error) { case "text": if text, ok := m["text"].(string); ok { textParts = append(textParts, text) + contentParts = append(contentParts, map[string]interface{}{"type": "text", "text": text}) + } + case "image": + if url := claudeImageURL(m); url != "" && msg.Role == "user" { + hasImage = true + contentParts = append(contentParts, map[string]interface{}{ + "type": "image_url", + "image_url": map[string]interface{}{"url": url}, + }) } case "thinking": // Skip thinking blocks - they are Claude's internal reasoning @@ -86,9 +97,12 @@ func ClaudeReqToOpenAI(claudeReq []byte, model string) ([]byte, error) { } // Add main message if has text or tool_calls - if len(textParts) > 0 || len(toolCalls) > 0 { + if len(textParts) > 0 || len(toolCalls) > 0 || hasImage { openaiMsg := transformer.OpenAIMessage{Role: msg.Role} - if len(textParts) > 0 { + if hasImage { + // Images need the content-parts form; plain text keeps the string form. + openaiMsg.Content = contentParts + } else if len(textParts) > 0 { openaiMsg.Content = strings.Join(textParts, "") } if len(toolCalls) > 0 { diff --git a/internal/transformer/convert/claude_openai2.go b/internal/transformer/convert/claude_openai2.go index be8eb950..31449c71 100644 --- a/internal/transformer/convert/claude_openai2.go +++ b/internal/transformer/convert/claude_openai2.go @@ -748,6 +748,10 @@ func convertClaudeMessageToOpenAI2Items(content []interface{}, role string) []ma case "text": text, _ := m["text"].(string) messageParts = append(messageParts, map[string]interface{}{"type": textType, "text": text}) + case "image": + if url := claudeImageURL(m); url != "" && role == "user" { + messageParts = append(messageParts, map[string]interface{}{"type": "input_image", "image_url": url}) + } case "thinking": // Skip thinking blocks - they are Claude's internal reasoning continue diff --git a/internal/transformer/convert/common.go b/internal/transformer/convert/common.go index 0d9ec45e..6459f29a 100644 --- a/internal/transformer/convert/common.go +++ b/internal/transformer/convert/common.go @@ -131,3 +131,25 @@ func extractSystemText(system interface{}) string { } return "" } + +// claudeImageURL returns the URL an OpenAI-style image part needs for a Claude image block: +// a data URL for a base64 source, the URL itself for a url source, "" for anything else. +func claudeImageURL(block map[string]interface{}) string { + source, ok := block["source"].(map[string]interface{}) + if !ok { + return "" + } + switch source["type"] { + case "base64": + mediaType, _ := source["media_type"].(string) + data, _ := source["data"].(string) + if mediaType == "" || data == "" { + return "" + } + return "data:" + mediaType + ";base64," + data + case "url": + url, _ := source["url"].(string) + return url + } + return "" +}