Skip to content

Commit 504b470

Browse files
committed
gateway: images a tool returned reach a Chat model after the tool messages (charan-rathore in #167: Claude Code's tool_result images were dropped on the way to Chat Completions)
A tool result holding images, as Claude Code's screenshot tools and its Read of a .png send it (an Anthropic tool_result with image blocks) or as a Responses function_call_output with input_image parts, lost its images whenever the request was translated: the result was read as its text only, so a Chat model was told "Read shot.png" and never shown it, and a result that was only an image came through empty. #167 turned such a request away with a 400; now the images go through. The shared request keeps a tool result's images beside its text. Chat's tool messages hold text only, so a tool message says "[The tool returned an image; it follows in the next message.]" after its text, and the images go right after the run of tool messages, each result's labelled with the tool and call it came from ("[From the result of Read (tool call toolu_1):]"), as image_url parts (a data: URL for inline ones). When the user's own message comes next, as Claude Code's system reminder beside a result does, the images open that message rather than being a second user message in a row, which some models' chat templates turn away; otherwise they are a user message of their own, before the assistant's next turn. Tool messages still directly follow the assistant's tool_calls. An Anthropic upstream gets the images inside its tool_result, a Responses one in the function_call_output's output list, Code Assist as inlineData after the functionResponse, as Gemini CLI sends a file it read; Kiro, Cursor and Devin take the text as before. A model that can't see still has each image described in its place before translating, so nothing follows its tool message. Tested end to end against fake upstreams: Claude Code's tool_use then a tool_result of text and a base64 PNG to a Chat model that sees (with and without the user's text after it), two parallel results with three images (base64 and a URL) giving tool, tool, then one user message with all three labelled, a screenshot from an older turn kept before the assistant's answer, Codex's function_call_output with input_image to Chat, Claude Code to a Responses provider, and a text-only model still given the description in the tool message. The four Chat tests fail without the change. go build, vet and the gateway tests pass; windows and linux (CGO off) build.
1 parent bdf3d11 commit 504b470

6 files changed

Lines changed: 461 additions & 18 deletions

File tree

‎internal/gateway/anthropic.go‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,8 @@ func parseAnthropic(body []byte) (*Request, error) {
105105
case "tool_use":
106106
msg.Parts = append(msg.Parts, Part{Kind: ToolCall, ID: b.ID, Name: b.Name, Args: b.Input})
107107
case "tool_result":
108-
msg.Parts = append(msg.Parts, Part{Kind: ToolResult, CallID: b.ToolUseID, Text: stringOrText(b.Content), IsError: b.IsError})
108+
out, images := toolOutput(b.Content)
109+
msg.Parts = append(msg.Parts, Part{Kind: ToolResult, CallID: b.ToolUseID, Text: out, Images: images, IsError: b.IsError})
109110
case "thinking":
110111
msg.Parts = append(msg.Parts, Part{Kind: Thinking, Text: b.Thinking, Signature: b.Signature})
111112
}
@@ -235,6 +236,23 @@ func withOutputEffort(body []byte, levels []string) []byte {
235236
return withFields(body, map[string]any{"output_config": oc})
236237
}
237238

239+
// imageBlock is an image as an Anthropic block: inline, or by its URL.
240+
func imageBlock(p Part) aBlock {
241+
b := aBlock{Type: "image"}
242+
b.Source = &struct {
243+
Type string `json:"type"`
244+
MediaType string `json:"media_type"`
245+
Data string `json:"data"`
246+
URL string `json:"url"`
247+
}{}
248+
if p.URL != "" && p.Data == "" {
249+
b.Source.Type, b.Source.URL = "url", p.URL
250+
} else {
251+
b.Source.Type, b.Source.MediaType, b.Source.Data = "base64", p.MediaType, p.Data
252+
}
253+
return b
254+
}
255+
238256
// buildAnthropic renders a request for an Anthropic-style upstream.
239257
func buildAnthropic(r *Request, model string) []byte {
240258
type msg struct {
@@ -263,23 +281,22 @@ func buildAnthropic(r *Request, model string) []byte {
263281
case File:
264282
rest = append(rest, aBlock{Type: "text", Text: attachmentText(p)})
265283
case Image:
266-
b := aBlock{Type: "image"}
267-
b.Source = &struct {
268-
Type string `json:"type"`
269-
MediaType string `json:"media_type"`
270-
Data string `json:"data"`
271-
URL string `json:"url"`
272-
}{}
273-
if p.URL != "" && p.Data == "" {
274-
b.Source.Type, b.Source.URL = "url", p.URL
275-
} else {
276-
b.Source.Type, b.Source.MediaType, b.Source.Data = "base64", p.MediaType, p.Data
277-
}
278-
rest = append(rest, b)
284+
rest = append(rest, imageBlock(p))
279285
case ToolCall:
280286
rest = append(rest, aBlock{Type: "tool_use", ID: p.ID, Name: p.Name, Input: argsOf(p)})
281287
case ToolResult:
282288
c, _ := json.Marshal(p.Text)
289+
if len(p.Images) > 0 {
290+
// a tool_result holds images beside its text
291+
var blocks []aBlock
292+
if strings.TrimSpace(p.Text) != "" {
293+
blocks = append(blocks, aBlock{Type: "text", Text: p.Text})
294+
}
295+
for _, im := range p.Images {
296+
blocks = append(blocks, imageBlock(im))
297+
}
298+
c, _ = json.Marshal(blocks)
299+
}
283300
results = append(results, aBlock{Type: "tool_result", ToolUseID: p.CallID, Content: c, IsError: p.IsError})
284301
case Thinking:
285302
if p.Signature != "" {

‎internal/gateway/chat.go‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ func parseChat(body []byte) (*Request, error) {
8888
}
8989
r.Messages = append(r.Messages, msg)
9090
case "tool":
91-
r.Messages = append(r.Messages, Message{Role: "user", Parts: []Part{{Kind: ToolResult, CallID: m.ToolCallID, Text: stringOrText(m.Content)}}})
91+
out, images := toolOutput(m.Content)
92+
r.Messages = append(r.Messages, Message{Role: "user", Parts: []Part{{Kind: ToolResult, CallID: m.ToolCallID, Text: out, Images: images}}})
9293
}
9394
}
9495
r.System = strings.Join(sys, "\n\n")
@@ -173,8 +174,36 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {
173174
msgs = append(msgs, map[string]any{"role": "system", "content": r.System})
174175
}
175176
deepseek := strings.Contains(host, "deepseek")
177+
// A tool message holds text only, so the images tools returned go to
178+
// the model in a user message after the tool messages, as the start of
179+
// the user's own message when one comes next: some models' chat
180+
// templates turn away two user messages in a row.
181+
names := map[string]string{}
182+
var seen []map[string]any
183+
seeLater := func(p Part) {
184+
if len(p.Images) == 0 {
185+
return
186+
}
187+
of := "tool call " + p.CallID
188+
if name := names[p.CallID]; name != "" {
189+
of = name + " (" + of + ")"
190+
} else if p.Name != "" {
191+
of = p.Name + " (" + of + ")"
192+
}
193+
seen = append(seen, map[string]any{"type": "text", "text": "[From the result of " + of + ":]"})
194+
for _, im := range p.Images {
195+
seen = append(seen, map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURL(im)}})
196+
}
197+
}
198+
showSeen := func() {
199+
if len(seen) > 0 {
200+
msgs = append(msgs, map[string]any{"role": "user", "content": seen})
201+
seen = nil
202+
}
203+
}
176204
for _, m := range r.Messages {
177205
if m.Role == "assistant" {
206+
showSeen()
178207
am := map[string]any{"role": "assistant"}
179208
var calls []map[string]any
180209
var think string
@@ -185,6 +214,7 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {
185214
if id == "" {
186215
id = "call_" + newID()
187216
}
217+
names[id] = p.Name
188218
calls = append(calls, map[string]any{"id": id, "type": "function",
189219
"function": map[string]any{"name": p.Name, "arguments": argsString(p)}})
190220
case Thinking:
@@ -210,6 +240,10 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {
210240
if len(content) == 0 {
211241
return
212242
}
243+
if len(seen) > 0 {
244+
content, plain = append(seen, content...), false
245+
seen = nil
246+
}
213247
if plain {
214248
var b strings.Builder
215249
for _, c := range content {
@@ -232,11 +266,24 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {
232266
content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": dataURL(p)}})
233267
case ToolResult:
234268
flush()
235-
msgs = append(msgs, map[string]any{"role": "tool", "tool_call_id": p.CallID, "content": p.Text})
269+
out := p.Text
270+
if n := len(p.Images); n > 0 {
271+
note := fmt.Sprintf("[The tool returned %d images; they follow in the next message.]", n)
272+
if n == 1 {
273+
note = "[The tool returned an image; it follows in the next message.]"
274+
}
275+
if strings.TrimSpace(out) != "" {
276+
out += "\n\n"
277+
}
278+
out += note
279+
}
280+
msgs = append(msgs, map[string]any{"role": "tool", "tool_call_id": p.CallID, "content": out})
281+
seeLater(p)
236282
}
237283
}
238284
flush()
239285
}
286+
showSeen()
240287
out := map[string]any{"model": model, "messages": msgs, "stream": r.Stream}
241288
if r.CacheKey != "" {
242289
out["prompt_cache_key"] = r.CacheKey

‎internal/gateway/codeassist.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,15 @@ func buildCodeAssist(r *Request, model, agent string) []byte {
126126
res["id"] = id
127127
}
128128
parts = append(parts, map[string]any{"functionResponse": res})
129+
// the images a tool returned follow its response, as Gemini
130+
// CLI sends a file it read
131+
for _, im := range p.Images {
132+
if im.Data != "" {
133+
parts = append(parts, map[string]any{"inlineData": map[string]any{"mimeType": im.MediaType, "data": im.Data}})
134+
} else if im.URL != "" {
135+
parts = append(parts, map[string]any{"fileData": map[string]any{"mimeType": im.MediaType, "fileUri": im.URL}})
136+
}
137+
}
129138
}
130139
// thinking isn't sent back: its signatures belong to whoever
131140
// made them, and Google turns away ones it didn't

‎internal/gateway/ir.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ type Part struct {
4545
// tool_result
4646
CallID string
4747
IsError bool
48+
Images []Part // the images the tool returned beside its text
4849

4950
// thinking
5051
Signature string
@@ -345,6 +346,51 @@ func stringOrText(raw json.RawMessage) string {
345346
return ""
346347
}
347348

349+
// toolOutput reads a tool's result: its text, as stringOrText reads it,
350+
// and the images in it. Anthropic's tool_result holds image blocks, a
351+
// Responses function_call_output input_image parts and a Chat tool message,
352+
// from clients that send them, image_url parts. An image named only by a
353+
// vendor's file id has nothing to carry and is left out.
354+
func toolOutput(raw json.RawMessage) (string, []Part) {
355+
text := stringOrText(raw)
356+
var blocks []struct {
357+
Type string `json:"type"`
358+
Source *struct {
359+
MediaType string `json:"media_type"`
360+
Data string `json:"data"`
361+
URL string `json:"url"`
362+
} `json:"source"`
363+
ImageURL json.RawMessage `json:"image_url"`
364+
}
365+
if len(raw) == 0 || raw[0] != '[' || json.Unmarshal(raw, &blocks) != nil {
366+
return text, nil
367+
}
368+
var images []Part
369+
for _, b := range blocks {
370+
switch b.Type {
371+
case "image":
372+
if s := b.Source; s != nil && s.Data != "" {
373+
images = append(images, Part{Kind: Image, MediaType: s.MediaType, Data: s.Data})
374+
} else if s != nil && s.URL != "" {
375+
images = append(images, Part{Kind: Image, URL: s.URL})
376+
}
377+
case "input_image", "image_url":
378+
var u string
379+
if json.Unmarshal(b.ImageURL, &u) != nil {
380+
var o struct {
381+
URL string `json:"url"`
382+
}
383+
json.Unmarshal(b.ImageURL, &o)
384+
u = o.URL
385+
}
386+
if u != "" {
387+
images = append(images, imagePart(u))
388+
}
389+
}
390+
}
391+
return text, images
392+
}
393+
348394
// effortOf normalises the reasoning effort names the APIs use.
349395
func effortOf(s string) string {
350396
switch strings.ToLower(strings.TrimSpace(s)) {

‎internal/gateway/responses.go‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,8 @@ func parseResponses(body []byte) (*Request, error) {
131131
}
132132
r.Messages = append(r.Messages, Message{Role: "assistant", Parts: []Part{{Kind: ToolCall, ID: it.CallID, Name: name, Args: parseArgs(it.Arguments)}}})
133133
case it.Type == "function_call_output":
134-
r.Messages = append(r.Messages, Message{Role: "user", Parts: []Part{{Kind: ToolResult, CallID: it.CallID, Text: stringOrText(it.Output)}}})
134+
out, images := toolOutput(it.Output)
135+
r.Messages = append(r.Messages, Message{Role: "user", Parts: []Part{{Kind: ToolResult, CallID: it.CallID, Text: out, Images: images}}})
135136
case it.Type == "reasoning":
136137
var b strings.Builder
137138
for _, s := range it.Summary {
@@ -306,7 +307,19 @@ func buildResponses(r *Request, model, host string, rejectTemp bool) []byte {
306307
input = append(input, map[string]any{"type": "function_call", "call_id": id, "name": p.Name, "arguments": argsString(p)})
307308
case ToolResult:
308309
flushMsg()
309-
input = append(input, map[string]any{"type": "function_call_output", "call_id": p.CallID, "output": p.Text})
310+
var output any = p.Text
311+
if len(p.Images) > 0 {
312+
// an output can be a list of text and images
313+
var items []map[string]any
314+
if strings.TrimSpace(p.Text) != "" {
315+
items = append(items, map[string]any{"type": "input_text", "text": p.Text})
316+
}
317+
for _, im := range p.Images {
318+
items = append(items, map[string]any{"type": "input_image", "image_url": dataURL(im)})
319+
}
320+
output = items
321+
}
322+
input = append(input, map[string]any{"type": "function_call_output", "call_id": p.CallID, "output": output})
310323
}
311324
}
312325
flushMsg()

0 commit comments

Comments
 (0)