Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 116 additions & 0 deletions internal/transformer/convert/claude_images_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package convert

import (
"encoding/json"
"testing"
)

const claudeReqWithImage = `{
"model": "claude-sonnet-4-6",
"max_tokens": 1024,
"messages": [{"role": "user", "content": [
{"type": "text", "text": "what is in this screenshot?"},
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "QUJD"}}
]}]
}`

func TestClaudeReqToOpenAISendsImagesAsImageURLParts(t *testing.T) {
out, err := ClaudeReqToOpenAI([]byte(claudeReqWithImage), "gpt-test")
if err != nil {
t.Fatalf("ClaudeReqToOpenAI failed: %v", err)
}
var body struct {
Messages []struct {
Role string `json:"role"`
Content interface{} `json:"content"`
} `json:"messages"`
}
if err := json.Unmarshal(out, &body); err != nil {
t.Fatalf("converted request is not JSON: %v", err)
}

parts, ok := body.Messages[0].Content.([]interface{})
if !ok || len(parts) != 2 {
t.Fatalf("expected a content list with a text and an image part, got %v", body.Messages[0].Content)
}
text := parts[0].(map[string]interface{})
if text["type"] != "text" || text["text"] != "what is in this screenshot?" {
t.Errorf("expected the text part first, got %v", text)
}
image := parts[1].(map[string]interface{})
url, _ := image["image_url"].(map[string]interface{})["url"].(string)
if image["type"] != "image_url" || url != "data:image/png;base64,QUJD" {
t.Errorf("expected an image_url part with a data URL, got %v", image)
}
}

func TestClaudeReqToOpenAIKeepsStringContentWithoutImages(t *testing.T) {
out, err := ClaudeReqToOpenAI([]byte(`{"model": "m", "messages": [{"role": "user", "content": [
{"type": "text", "text": "hello "}, {"type": "text", "text": "world"}
]}]}`), "gpt-test")
if err != nil {
t.Fatalf("ClaudeReqToOpenAI failed: %v", err)
}
var body struct {
Messages []struct {
Content interface{} `json:"content"`
} `json:"messages"`
}
if err := json.Unmarshal(out, &body); err != nil {
t.Fatalf("converted request is not JSON: %v", err)
}
if body.Messages[0].Content != "hello world" {
t.Errorf("expected the text joined into a string as before, got %v", body.Messages[0].Content)
}
}

func TestClaudeReqToOpenAI2SendsImagesAsInputImage(t *testing.T) {
out, err := ClaudeReqToOpenAI2([]byte(claudeReqWithImage), "gpt-test")
if err != nil {
t.Fatalf("ClaudeReqToOpenAI2 failed: %v", err)
}
var body struct {
Input []struct {
Role string `json:"role"`
Content []map[string]interface{} `json:"content"`
} `json:"input"`
}
if err := json.Unmarshal(out, &body); err != nil {
t.Fatalf("converted request is not JSON: %v", err)
}

if len(body.Input) != 1 || len(body.Input[0].Content) != 2 {
t.Fatalf("expected one message with a text and an image part, got %v", body.Input)
}
if body.Input[0].Content[0]["type"] != "input_text" {
t.Errorf("expected the text part first, got %v", body.Input[0].Content[0])
}
image := body.Input[0].Content[1]
if image["type"] != "input_image" || image["image_url"] != "data:image/png;base64,QUJD" {
t.Errorf("expected an input_image part with a data URL, got %v", image)
}
}

func TestClaudeImageURLSourceIsForwarded(t *testing.T) {
out, err := ClaudeReqToOpenAI([]byte(`{"model": "m", "messages": [{"role": "user", "content": [
{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}
]}]}`), "gpt-test")
if err != nil {
t.Fatalf("ClaudeReqToOpenAI failed: %v", err)
}
var body struct {
Messages []struct {
Content []map[string]interface{} `json:"content"`
} `json:"messages"`
}
if err := json.Unmarshal(out, &body); err != nil {
t.Fatalf("converted request is not JSON: %v", err)
}
if len(body.Messages) != 1 || len(body.Messages[0].Content) != 1 {
t.Fatalf("expected one message with one image part, got %v", body.Messages)
}
url, _ := body.Messages[0].Content[0]["image_url"].(map[string]interface{})["url"].(string)
if url != "https://example.com/a.png" {
t.Errorf("expected the image URL to be forwarded, got %q", url)
}
}
18 changes: 16 additions & 2 deletions internal/transformer/convert/claude_openai.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,11 @@ func ClaudeReqToOpenAI(claudeReq []byte, model string) ([]byte, error) {
case []interface{}:
// Check for tool_result blocks
var textParts []string
var contentParts []interface{} // text and image parts in order, used when the message has images
var toolCalls []transformer.OpenAIToolCall
var toolResults []transformer.OpenAIMessage
hasThinking := false
hasImage := false

for _, block := range content {
m, ok := block.(map[string]interface{})
Expand All @@ -48,6 +50,15 @@ func ClaudeReqToOpenAI(claudeReq []byte, model string) ([]byte, error) {
case "text":
if text, ok := m["text"].(string); ok {
textParts = append(textParts, text)
contentParts = append(contentParts, map[string]interface{}{"type": "text", "text": text})
}
case "image":
if url := claudeImageURL(m); url != "" && msg.Role == "user" {
hasImage = true
contentParts = append(contentParts, map[string]interface{}{
"type": "image_url",
"image_url": map[string]interface{}{"url": url},
})
}
case "thinking":
// Skip thinking blocks - they are Claude's internal reasoning
Expand Down Expand Up @@ -86,9 +97,12 @@ func ClaudeReqToOpenAI(claudeReq []byte, model string) ([]byte, error) {
}

// Add main message if has text or tool_calls
if len(textParts) > 0 || len(toolCalls) > 0 {
if len(textParts) > 0 || len(toolCalls) > 0 || hasImage {
openaiMsg := transformer.OpenAIMessage{Role: msg.Role}
if len(textParts) > 0 {
if hasImage {
// Images need the content-parts form; plain text keeps the string form.
openaiMsg.Content = contentParts
} else if len(textParts) > 0 {
openaiMsg.Content = strings.Join(textParts, "")
}
if len(toolCalls) > 0 {
Expand Down
4 changes: 4 additions & 0 deletions internal/transformer/convert/claude_openai2.go
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,10 @@ func convertClaudeMessageToOpenAI2Items(content []interface{}, role string) []ma
case "text":
text, _ := m["text"].(string)
messageParts = append(messageParts, map[string]interface{}{"type": textType, "text": text})
case "image":
if url := claudeImageURL(m); url != "" && role == "user" {
messageParts = append(messageParts, map[string]interface{}{"type": "input_image", "image_url": url})
}
case "thinking":
// Skip thinking blocks - they are Claude's internal reasoning
continue
Expand Down
22 changes: 22 additions & 0 deletions internal/transformer/convert/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,3 +131,25 @@ func extractSystemText(system interface{}) string {
}
return ""
}

// claudeImageURL returns the URL an OpenAI-style image part needs for a Claude image block:
// a data URL for a base64 source, the URL itself for a url source, "" for anything else.
func claudeImageURL(block map[string]interface{}) string {
source, ok := block["source"].(map[string]interface{})
if !ok {
return ""
}
switch source["type"] {
case "base64":
mediaType, _ := source["media_type"].(string)
data, _ := source["data"].(string)
if mediaType == "" || data == "" {
return ""
}
return "data:" + mediaType + ";base64," + data
case "url":
url, _ := source["url"].(string)
return url
}
return ""
}
Loading