Skip to content

Commit 04da4c6

Browse files
committed
fix(task): keep repaired tool_result images top-level so every provider sends them
Nesting the image into the preceding tool_result dropped it for the OpenAI chat, AI SDK, and OpenAI Codex converters, and hid it from maybeRemoveImageBlocks for text-only models. Partition results first instead, the shape live turns already store (#1307).
1 parent ec7057c commit 04da4c6

3 files changed

Lines changed: 162 additions & 110 deletions

File tree

‎src/core/task/__tests__/Task.spec.ts‎

Lines changed: 77 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -3027,80 +3027,86 @@ describe("Cline", () => {
30273027
expect(Object.keys(cleanConversationHistory[0]!)).toEqual(["role", "content"])
30283028
})
30293029

3030-
it("sends tool_result blocks first when stored history has images between them (#1774)", async () => {
3031-
const cline = new Task({
3032-
provider: mockProvider,
3033-
apiConfiguration: mockApiConfig,
3034-
task: "test task",
3035-
startTask: false,
3036-
})
3037-
vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt")
3038-
// Image-capable model so the image block is kept in the outgoing request.
3039-
vi.spyOn(cline.api, "getModel").mockReturnValue({
3040-
id: "claude-3-sonnet",
3041-
info: {
3042-
supportsImages: true,
3043-
supportsPromptCache: false,
3044-
contextWindow: 200000,
3045-
maxTokens: 4096,
3046-
} as ModelInfo,
3047-
})
3048-
const mockStream = {
3049-
async *[Symbol.asyncIterator]() {
3050-
yield { type: "text", text: "response" }
3051-
},
3052-
} as AsyncGenerator<ApiStreamChunk>
3053-
const createMessageSpy = vi.spyOn(cline.api, "createMessage").mockReturnValue(mockStream)
3030+
it.each([
3031+
{
3032+
supportsImages: true,
3033+
imageBlock: { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } },
3034+
},
3035+
{
3036+
// A text-only model must not receive the image, including after the repair.
3037+
supportsImages: false,
3038+
imageBlock: { type: "text", text: "[Referenced image in conversation]" },
3039+
},
3040+
])(
3041+
"sends tool_result blocks first when stored history has images between them (#1774, supportsImages: $supportsImages)",
3042+
async ({ supportsImages, imageBlock }) => {
3043+
const cline = new Task({
3044+
provider: mockProvider,
3045+
apiConfiguration: mockApiConfig,
3046+
task: "test task",
3047+
startTask: false,
3048+
})
3049+
vi.spyOn(getTaskTestAccess(cline), "getSystemPrompt").mockResolvedValue("mock system prompt")
3050+
vi.spyOn(cline.api, "getModel").mockReturnValue({
3051+
id: "claude-3-sonnet",
3052+
info: {
3053+
supportsImages,
3054+
supportsPromptCache: false,
3055+
contextWindow: 200000,
3056+
maxTokens: 4096,
3057+
} as ModelInfo,
3058+
})
3059+
const mockStream = {
3060+
async *[Symbol.asyncIterator]() {
3061+
yield { type: "text", text: "response" }
3062+
},
3063+
} as AsyncGenerator<ApiStreamChunk>
3064+
const createMessageSpy = vi.spyOn(cline.api, "createMessage").mockReturnValue(mockStream)
30543065

3055-
// History as persisted by a turn that read an image alongside another tool call.
3056-
cline.apiConversationHistory = [
3057-
{ role: "user", content: [{ type: "text", text: "read the logo and list files" }], ts: 1 },
3058-
{
3059-
role: "assistant",
3060-
content: [
3061-
{ type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } },
3062-
{ type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } },
3063-
],
3064-
ts: 2,
3065-
},
3066-
{
3067-
role: "user",
3068-
content: [
3069-
{ type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" },
3070-
{ type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } },
3071-
{ type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" },
3072-
{ type: "text", text: "<environment_details>" },
3073-
],
3074-
ts: 3,
3075-
},
3076-
]
3066+
// History as persisted by a turn that read an image alongside another tool call.
3067+
cline.apiConversationHistory = [
3068+
{ role: "user", content: [{ type: "text", text: "read the logo and list files" }], ts: 1 },
3069+
{
3070+
role: "assistant",
3071+
content: [
3072+
{ type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } },
3073+
{ type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } },
3074+
],
3075+
ts: 2,
3076+
},
3077+
{
3078+
role: "user",
3079+
content: [
3080+
{ type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" },
3081+
{ type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } },
3082+
{ type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" },
3083+
{ type: "text", text: "<environment_details>" },
3084+
],
3085+
ts: 3,
3086+
},
3087+
]
30773088

3078-
const iterator = cline.attemptApiRequest(0)
3079-
await iterator.next()
3089+
const iterator = cline.attemptApiRequest(0)
3090+
await iterator.next()
30803091

3081-
const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0])
3082-
const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }>
3083-
// Both results lead; the image sits inside the read_file result; the environment details follow.
3084-
expect(lastUserContent).toEqual([
3085-
{
3086-
type: "tool_result",
3087-
tool_use_id: "read",
3088-
content: [
3089-
{ type: "text", text: "File: logo.jpg" },
3090-
{ type: "image", source: { type: "base64", media_type: "image/jpeg", data: "abc" } },
3091-
],
3092-
},
3093-
{ type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" },
3094-
{ type: "text", text: "<environment_details>" },
3095-
])
3096-
// The stored history keeps its original shape.
3097-
expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([
3098-
{ type: "tool_result" },
3099-
{ type: "image" },
3100-
{ type: "tool_result" },
3101-
{ type: "text" },
3102-
])
3103-
})
3092+
const [, sentHistory] = requireDefined(createMessageSpy.mock.calls[0])
3093+
const lastUserContent = requireDefined(sentHistory.at(-1)).content as Array<{ type: string }>
3094+
// Both results lead; the image (or its text stand-in) and the environment details follow.
3095+
expect(lastUserContent).toEqual([
3096+
{ type: "tool_result", tool_use_id: "read", content: "File: logo.jpg" },
3097+
{ type: "tool_result", tool_use_id: "cmd", content: "Exit code: 0" },
3098+
imageBlock,
3099+
{ type: "text", text: "<environment_details>" },
3100+
])
3101+
// The stored history keeps its original shape.
3102+
expect(requireDefined(cline.apiConversationHistory.at(-1)).content).toMatchObject([
3103+
{ type: "tool_result" },
3104+
{ type: "image" },
3105+
{ type: "tool_result" },
3106+
{ type: "text" },
3107+
])
3108+
},
3109+
)
31043110

31053111
it("should shape image blocks for API compatibility before request construction", async () => {
31063112
const conversationHistory = [

‎src/core/task/__tests__/orderToolResultsFirst.spec.ts‎

Lines changed: 79 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
import type { Anthropic } from "@anthropic-ai/sdk"
44

5+
import type { ApiHandler } from "../../../api"
6+
import { OpenAiCodexHandler } from "../../../api/providers/openai-codex"
7+
import { convertToAiSdkMessages } from "../../../api/transform/ai-sdk"
8+
import { maybeRemoveImageBlocks } from "../../../api/transform/image-cleaning"
9+
import { convertToOpenAiMessages } from "../../../api/transform/openai-format"
510
import type { ApiMessage } from "../../task-persistence"
611
import { mergeConsecutiveApiMessages } from "../mergeConsecutiveApiMessages"
712
import { orderToolResultsFirst } from "../orderToolResultsFirst"
@@ -22,10 +27,24 @@ const nested = (id: string, content: Anthropic.Messages.ToolResultBlockParam["co
2227
content,
2328
})
2429

30+
// [tool_result, image, tool_result, text] as saved when read_file on an image ran alongside
31+
// another tool call in the same turn (#1774).
32+
const stuckTurn = (): ApiMessage[] => [
33+
{
34+
role: "assistant",
35+
content: [
36+
{ type: "tool_use", id: "read", name: "read_file", input: { path: "logo.jpg" } },
37+
{ type: "tool_use", id: "cmd", name: "execute_command", input: { command: "ls" } },
38+
],
39+
},
40+
{
41+
role: "user",
42+
content: [toolResult("read"), image("jpeg"), toolResult("cmd"), text("<environment_details>")],
43+
},
44+
]
45+
2546
describe("orderToolResultsFirst", () => {
2647
it("repairs the persisted #1774 shape so every tool_result leads the message", () => {
27-
// [tool_result, image, tool_result, text] as saved when read_file on an image ran
28-
// alongside another tool call in the same turn.
2948
const stuck: ApiMessage = {
3049
role: "user",
3150
ts: 16,
@@ -34,10 +53,11 @@ describe("orderToolResultsFirst", () => {
3453

3554
const [repaired] = orderToolResultsFirst([stuck])
3655

37-
// The image moves into the read_file result, the shape the Anthropic API expects.
56+
// The results lead; the image stays a top-level block, as live turns store it (#1307).
3857
expect(repaired.content).toEqual([
39-
nested("read", [text("result read"), image("jpeg")]),
58+
toolResult("read"),
4059
toolResult("cmd"),
60+
image("jpeg"),
4161
text("<environment_details>"),
4262
])
4363
expect(repaired.ts).toBe(16)
@@ -60,23 +80,24 @@ describe("orderToolResultsFirst", () => {
6080

6181
expect(repaired.content).toEqual([
6282
toolResult("a"),
63-
nested("b", [text("result b"), image("1")]),
64-
nested("c", [text("result c"), image("2")]),
83+
toolResult("b"),
84+
toolResult("c"),
85+
image("1"),
86+
image("2"),
6587
text("t"),
6688
])
6789
})
6890

69-
it("appends an image to a result whose content is already an array", () => {
91+
it("leaves the content of every result block untouched", () => {
7092
const withArray = nested("a", [text("first"), text("second")])
7193

7294
const [repaired] = orderToolResultsFirst([{ role: "user", content: [withArray, image("1"), toolResult("b")] }])
7395

74-
expect(repaired.content).toEqual([nested("a", [text("first"), text("second"), image("1")]), toolResult("b")])
75-
// The original result block is not mutated.
96+
expect(repaired.content).toEqual([withArray, toolResult("b"), image("1")])
7697
expect(withArray.content).toEqual([text("first"), text("second")])
7798
})
7899

79-
it("keeps an image standalone when no result directly precedes it", () => {
100+
it("moves leading and mid-message blocks after the results in their original order", () => {
80101
const [repaired] = orderToolResultsFirst([
81102
{
82103
role: "user",
@@ -112,4 +133,52 @@ describe("orderToolResultsFirst", () => {
112133

113134
out.forEach((message, index) => expect(message).toBe([ordered, plain, noResults, assistant][index]))
114135
})
136+
137+
describe("repaired images reach the provider request", () => {
138+
const dataUrl = "data:image/jpeg;base64,jpeg"
139+
140+
it("OpenAI chat format sends the image as an image_url part", () => {
141+
const converted = convertToOpenAiMessages(orderToolResultsFirst(stuckTurn()))
142+
143+
expect(converted.map((message) => message.role)).toEqual(["assistant", "tool", "tool", "user"])
144+
expect(converted.at(-1)?.content).toContainEqual({ type: "image_url", image_url: { url: dataUrl } })
145+
})
146+
147+
it("AI SDK format sends the image as an image part", () => {
148+
const converted = convertToAiSdkMessages(orderToolResultsFirst(stuckTurn()))
149+
const user = converted.find((message) => message.role === "user")
150+
151+
expect(user?.content).toContainEqual({ type: "image", image: dataUrl, mimeType: "image/jpeg" })
152+
})
153+
154+
it("OpenAI Codex Responses input sends the image as an input_image part", () => {
155+
const handler = new OpenAiCodexHandler({ apiModelId: "gpt-5.1-codex" })
156+
const input: unknown[] = handler["formatFullConversation"]("", orderToolResultsFirst(stuckTurn()))
157+
158+
expect(input).toContainEqual({
159+
role: "user",
160+
content: [
161+
{ type: "input_image", image_url: dataUrl },
162+
{ type: "input_text", text: "<environment_details>" },
163+
],
164+
})
165+
})
166+
167+
it("a model without image support gets no image block anywhere in the request", () => {
168+
const textOnly: ApiHandler = {
169+
getModel: vi.fn().mockReturnValue({ id: "text-only", info: { supportsImages: false } }),
170+
createMessage: vi.fn(),
171+
countTokens: vi.fn(),
172+
getModelCacheScope: vi.fn(),
173+
}
174+
const [, user] = maybeRemoveImageBlocks(orderToolResultsFirst(stuckTurn()), textOnly)
175+
176+
expect(user.content).toEqual([
177+
toolResult("read"),
178+
toolResult("cmd"),
179+
text("[Referenced image in conversation]"),
180+
text("<environment_details>"),
181+
])
182+
})
183+
})
115184
})
Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,11 @@
1-
import type { Anthropic } from "@anthropic-ai/sdk"
2-
31
import type { ApiMessage } from "../task-persistence"
42

5-
type Block = Anthropic.Messages.ContentBlockParam
6-
type ToolResultBlock = Anthropic.Messages.ToolResultBlockParam
7-
8-
function toContentArray(content: ToolResultBlock["content"]): Exclude<ToolResultBlock["content"], string | undefined> {
9-
if (typeof content === "string") return [{ type: "text", text: content }]
10-
return content ?? []
11-
}
12-
133
/**
144
* Moves `tool_result` blocks to the front of each user message, keeping the relative order of
15-
* the results and of the remaining blocks. An image that directly follows a result (as when
16-
* `read_file` returns an image) is nested into that result's `content`, where Anthropic expects it.
5+
* the results and of the remaining blocks (images, text). This is the same partition
6+
* `Task.pushToolResultToUserContent` applies to live turns (#1307), so repaired history has the
7+
* shape every provider converter already handles: images stay top-level blocks after the results,
8+
* where image-capable converters forward them and `maybeRemoveImageBlocks` can strip them.
179
*
1810
* Anthropic requires the results for the previous turn's `tool_use` blocks to lead the next user
1911
* message. History saved with image blocks between results (#1307, #1774), or produced by merging
@@ -34,23 +26,8 @@ export function orderToolResultsFirst(messages: ApiMessage[]): ApiMessage[] {
3426
return message
3527
}
3628

37-
const results: ToolResultBlock[] = []
38-
const rest: Block[] = []
39-
// Index of the result an immediately following image belongs to; reset by any other block.
40-
let owner = -1
41-
for (const block of message.content) {
42-
if (block.type === "tool_result") {
43-
results.push(block)
44-
owner = results.length - 1
45-
} else if (block.type === "image" && owner !== -1) {
46-
const current = results[owner]
47-
results[owner] = { ...current, content: [...toContentArray(current.content), block] }
48-
} else {
49-
owner = -1
50-
rest.push(block)
51-
}
52-
}
53-
29+
const results = message.content.filter((block) => block.type === "tool_result")
30+
const rest = message.content.filter((block) => block.type !== "tool_result")
5431
return { ...message, content: [...results, ...rest] }
5532
})
5633
}

0 commit comments

Comments
 (0)