Skip to content

Commit e58142f

Browse files
author
Zoo (VP)
committed
fix(anthropic): keep tool_result blocks contiguous by grouping image blocks separately
When a turn issued multiple tool_use blocks and tool results carried image blocks (read_file on images, screenshots, feedback images), presentAssistantMessage pushed each tool_result followed immediately by its images, yielding tool_result/image interleaving in userMessageContent. Task.ts then consumed the array verbatim, and the Anthropic API rejected the request with a 400: "tool_use ids were found without tool_result blocks immediately after" (#1307). pushToolResultToUserContent is the single choke point every tool_result push flows through, so it now maintains the invariant: when a new tool_result is appended, image blocks already accumulated are moved after it. The resulting user message is tool_results first (in tool_use order), then images, then other blocks — matching the shape the issue expects. A single tool result with images and image-free messages are unchanged.
1 parent bac8adc commit e58142f

3 files changed

Lines changed: 208 additions & 8 deletions

File tree

‎src/core/assistant-message/__tests__/presentAssistantMessage-images.spec.ts‎

Lines changed: 135 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { Anthropic } from "@anthropic-ai/sdk"
55
import { presentAssistantMessage } from "../presentAssistantMessage"
66
import { Task } from "../../task/Task"
77

8+
type UserContentBlock = Anthropic.TextBlockParam | Anthropic.ImageBlockParam | Anthropic.ToolResultBlockParam
9+
810
// Mock dependencies
911
vi.mock("../../task/Task")
1012
vi.mock("../../tools/validateToolUse", () => ({
@@ -62,7 +64,8 @@ describe("presentAssistantMessage - Image Handling in Native Tool Calling", () =
6264
ask: vi.fn().mockResolvedValue({ response: "yesButtonClicked" }),
6365
}
6466

65-
// Add pushToolResultToUserContent method after mockTask is created so it can reference mockTask
67+
// Mirrors Task.pushToolResultToUserContent: duplicate guard plus keeping
68+
// tool_result blocks contiguous by moving image blocks after the new result.
6669
mockTask.pushToolResultToUserContent = vi.fn().mockImplementation((toolResult: any) => {
6770
const existingResult = mockTask.userMessageContent.find(
6871
(block: any) => block.type === "tool_result" && block.tool_use_id === toolResult.tool_use_id,
@@ -71,6 +74,14 @@ describe("presentAssistantMessage - Image Handling in Native Tool Calling", () =
7174
return false
7275
}
7376
mockTask.userMessageContent.push(toolResult)
77+
const images = mockTask.userMessageContent.filter((block: UserContentBlock) => block.type === "image")
78+
if (images.length > 0) {
79+
const withoutImages = mockTask.userMessageContent.filter(
80+
(block: UserContentBlock) => block.type !== "image",
81+
)
82+
withoutImages.push(...images)
83+
mockTask.userMessageContent.splice(0, mockTask.userMessageContent.length, ...withoutImages)
84+
}
7485
return true
7586
})
7687
})
@@ -319,4 +330,127 @@ describe("presentAssistantMessage - Image Handling in Native Tool Calling", () =
319330
expect(toolResult.content).toContain("was interrupted and not executed")
320331
})
321332
})
333+
334+
describe("tool_result contiguity (GitHub #1307)", () => {
335+
it("groups images after all tool_results when parallel tools return images", async () => {
336+
const toolCallId1 = "tool_call_img_1"
337+
const toolCallId2 = "tool_call_img_2"
338+
339+
mockTask.assistantMessageContent = [
340+
{
341+
type: "tool_use",
342+
id: toolCallId1,
343+
name: "ask_followup_question",
344+
params: { question: "Q1?", follow_up: [] },
345+
nativeArgs: { question: "Q1?", follow_up: [] },
346+
},
347+
{
348+
type: "tool_use",
349+
id: toolCallId2,
350+
name: "ask_followup_question",
351+
params: { question: "Q2?", follow_up: [] },
352+
nativeArgs: { question: "Q2?", follow_up: [] },
353+
},
354+
]
355+
356+
// The tool asks the followup question via task.ask("followup", ...);
357+
// distinct image data per question so ordering is observable.
358+
mockTask.ask = vi.fn().mockImplementation((type: string, message: string) => {
359+
if (type !== "followup") {
360+
return Promise.resolve({ response: "yesButtonClicked", text: "approved", images: undefined })
361+
}
362+
const prefix = message.includes("Q1") ? "a" : "b"
363+
return Promise.resolve({
364+
response: "yesButtonClicked",
365+
text: `answer ${prefix}`,
366+
images: [`data:image/png;base64,${prefix}-result`],
367+
})
368+
})
369+
370+
await presentAssistantMessage(mockTask)
371+
372+
// Anthropic rejects user messages where image blocks sit between
373+
// tool_result blocks, so images from both tools must be grouped
374+
// after the last tool_result instead of interleaved.
375+
expect(mockTask.userMessageContent.map((block: UserContentBlock) => block.type)).toEqual([
376+
"tool_result",
377+
"tool_result",
378+
"image",
379+
"image",
380+
])
381+
expect(
382+
mockTask.userMessageContent
383+
.filter(
384+
(block: UserContentBlock): block is Anthropic.ToolResultBlockParam =>
385+
block.type === "tool_result",
386+
)
387+
.map((block: Anthropic.ToolResultBlockParam) => block.tool_use_id),
388+
).toEqual([toolCallId1, toolCallId2])
389+
expect(
390+
mockTask.userMessageContent
391+
.filter((block: UserContentBlock): block is Anthropic.ImageBlockParam => block.type === "image")
392+
.map(
393+
(block: Anthropic.ImageBlockParam) =>
394+
(block.source as { type: "base64"; media_type: string; data: string }).data,
395+
),
396+
).toEqual(["a-result", "b-result"])
397+
})
398+
399+
it("relocates an earlier tool's images after a later tool_result with no images", async () => {
400+
const toolCallId1 = "tool_call_img_3"
401+
const toolCallId2 = "tool_call_img_4"
402+
403+
mockTask.assistantMessageContent = [
404+
{
405+
type: "tool_use",
406+
id: toolCallId1,
407+
name: "ask_followup_question",
408+
params: { question: "Q1?", follow_up: [] },
409+
nativeArgs: { question: "Q1?", follow_up: [] },
410+
},
411+
{
412+
type: "tool_use",
413+
id: toolCallId2,
414+
name: "ask_followup_question",
415+
params: { question: "Q2?", follow_up: [] },
416+
nativeArgs: { question: "Q2?", follow_up: [] },
417+
},
418+
]
419+
420+
mockTask.ask = vi.fn().mockImplementation((type: string, message: string) => {
421+
if (type === "followup" && message.includes("Q1")) {
422+
return Promise.resolve({
423+
response: "yesButtonClicked",
424+
text: "answer a",
425+
images: ["data:image/png;base64,a-result"],
426+
})
427+
}
428+
return Promise.resolve({ response: "yesButtonClicked", text: "answer b", images: undefined })
429+
})
430+
431+
await presentAssistantMessage(mockTask)
432+
433+
expect(mockTask.userMessageContent.map((block: UserContentBlock) => block.type)).toEqual([
434+
"tool_result",
435+
"tool_result",
436+
"image",
437+
])
438+
expect(
439+
mockTask.userMessageContent
440+
.filter(
441+
(block: UserContentBlock): block is Anthropic.ToolResultBlockParam =>
442+
block.type === "tool_result",
443+
)
444+
.map((block: Anthropic.ToolResultBlockParam) => block.tool_use_id),
445+
).toEqual([toolCallId1, toolCallId2])
446+
expect(
447+
mockTask.userMessageContent
448+
.filter((block: UserContentBlock): block is Anthropic.ImageBlockParam => block.type === "image")
449+
.map(
450+
(block: Anthropic.ImageBlockParam) =>
451+
(block.source as { type: "base64"; media_type: string; data: string }).data,
452+
),
453+
).toEqual(["a-result"])
454+
})
455+
})
322456
})

‎src/core/task/Task.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,12 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
469469
* Push a tool_result block to userMessageContent, preventing duplicates.
470470
* Duplicate tool_use_ids cause API errors.
471471
*
472+
* Keeps all tool_result blocks contiguous, as the Anthropic API requires:
473+
* image blocks attached to earlier tool results must not sit between two
474+
* tool_results ("tool_use ids were found without tool_result blocks
475+
* immediately after"), so any images preceding the new result are moved
476+
* after it.
477+
*
472478
* @param toolResult - The tool_result block to add
473479
* @returns true if added, false if duplicate was skipped
474480
*/
@@ -484,6 +490,18 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
484490
return false
485491
}
486492
this.userMessageContent.push(toolResult)
493+
494+
// The new result is always appended at the end, so every image block
495+
// currently in the array precedes it and would split the tool_result
496+
// run once another result arrives. Move them after the new result.
497+
const images = this.userMessageContent.filter(
498+
(block): block is Anthropic.ImageBlockParam => block.type === "image",
499+
)
500+
if (images.length > 0) {
501+
const withoutImages = this.userMessageContent.filter((block) => block.type !== "image")
502+
this.userMessageContent.splice(0, this.userMessageContent.length, ...withoutImages, ...images)
503+
}
504+
487505
return true
488506
}
489507
didRejectTool = false

‎src/core/task/__tests__/Task.spec.ts‎

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6270,7 +6270,7 @@ describe("pushToolResultToUserContent", () => {
62706270
expect(task.userMessageContent[0]).toEqual(errorResult)
62716271
})
62726272

6273-
it("should not interfere with other content types in userMessageContent", () => {
6273+
it("should preserve other content types while keeping tool_results contiguous", () => {
62746274
const task = new Task({
62756275
provider: mockProvider,
62766276
apiConfiguration: mockApiConfig,
@@ -6279,10 +6279,11 @@ describe("pushToolResultToUserContent", () => {
62796279
})
62806280

62816281
// Add text and image blocks manually
6282-
task.userMessageContent.push(
6283-
{ type: "text", text: "Some text" },
6284-
{ type: "image", source: { type: "base64", media_type: "image/png", data: "base64data" } },
6285-
)
6282+
const image: Anthropic.ImageBlockParam = {
6283+
type: "image",
6284+
source: { type: "base64", media_type: "image/png", data: "base64data" },
6285+
}
6286+
task.userMessageContent.push({ type: "text", text: "Some text" }, image)
62866287

62876288
const toolResult: Anthropic.ToolResultBlockParam = {
62886289
type: "tool_result",
@@ -6294,8 +6295,55 @@ describe("pushToolResultToUserContent", () => {
62946295

62956296
expect(added).toBe(true)
62966297
expect(task.userMessageContent).toHaveLength(3)
6298+
// The image must not sit between two tool_results, so it moves after
6299+
// the new result; other blocks keep their relative order.
62976300
expect(task.userMessageContent[0].type).toBe("text")
6298-
expect(task.userMessageContent[1].type).toBe("image")
6299-
expect(task.userMessageContent[2]).toEqual(toolResult)
6301+
expect(task.userMessageContent[1]).toEqual(toolResult)
6302+
expect(task.userMessageContent[2]).toEqual(image)
6303+
})
6304+
6305+
it("keeps tool_result blocks contiguous when a later result has no images (GitHub #1307)", () => {
6306+
const task = new Task({
6307+
provider: mockProvider,
6308+
apiConfiguration: mockApiConfig,
6309+
task: "test task",
6310+
startTask: false,
6311+
})
6312+
6313+
const toolResult1: Anthropic.ToolResultBlockParam = {
6314+
type: "tool_result",
6315+
tool_use_id: "id-1",
6316+
content: "Result 1",
6317+
}
6318+
const toolResult2: Anthropic.ToolResultBlockParam = {
6319+
type: "tool_result",
6320+
tool_use_id: "id-2",
6321+
content: "Result 2",
6322+
}
6323+
const image1: Anthropic.ImageBlockParam = {
6324+
type: "image",
6325+
source: { type: "base64", media_type: "image/png", data: "img1" },
6326+
}
6327+
const image2: Anthropic.ImageBlockParam = {
6328+
type: "image",
6329+
source: { type: "base64", media_type: "image/png", data: "img2" },
6330+
}
6331+
6332+
// First tool result arrives with its images appended right after it (as
6333+
// presentAssistantMessage does); a second result without images follows.
6334+
task.pushToolResultToUserContent(toolResult1)
6335+
task.userMessageContent.push(image1, image2)
6336+
task.pushToolResultToUserContent(toolResult2)
6337+
6338+
expect(task.userMessageContent.map((block) => block.type)).toEqual([
6339+
"tool_result",
6340+
"tool_result",
6341+
"image",
6342+
"image",
6343+
])
6344+
expect(task.userMessageContent[0]).toEqual(toolResult1)
6345+
expect(task.userMessageContent[1]).toEqual(toolResult2)
6346+
expect(task.userMessageContent[2]).toEqual(image1)
6347+
expect(task.userMessageContent[3]).toEqual(image2)
63006348
})
63016349
})

0 commit comments

Comments
 (0)