Skip to content

Commit c284469

Browse files
committed
fix: ensure images are properly returned as tool results
1 parent fc0210c commit c284469

3 files changed

Lines changed: 64 additions & 47 deletions

File tree

‎packages/opencode/src/session/message-v2.ts‎

Lines changed: 55 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,40 @@ export namespace MessageV2 {
435435

436436
export function toModelMessages(input: WithParts[], model: Provider.Model): ModelMessage[] {
437437
const result: UIMessage[] = []
438+
const toolNames = new Set<string>()
439+
440+
const toModelOutput = (output: unknown) => {
441+
if (typeof output === "string") {
442+
return { type: "text", value: output }
443+
}
444+
445+
if (typeof output === "object") {
446+
const outputObject = output as {
447+
text: string
448+
attachments?: Array<{ mime: string; url: string }>
449+
}
450+
const attachments = (outputObject.attachments ?? []).filter((attachment) => {
451+
return attachment.url.startsWith("data:") && attachment.url.includes(",")
452+
})
453+
454+
return {
455+
type: "content",
456+
value: [
457+
{ type: "text", text: outputObject.text },
458+
...attachments.map((attachment) => ({
459+
type: "media",
460+
mediaType: attachment.mime,
461+
data: iife(() => {
462+
const commaIndex = attachment.url.indexOf(",")
463+
return commaIndex === -1 ? attachment.url : attachment.url.slice(commaIndex + 1)
464+
}),
465+
})),
466+
],
467+
}
468+
}
469+
470+
return { type: "json", value: output as never }
471+
}
438472

439473
for (const msg of input) {
440474
if (msg.parts.length === 0) continue
@@ -505,31 +539,24 @@ export namespace MessageV2 {
505539
type: "step-start",
506540
})
507541
if (part.type === "tool") {
542+
toolNames.add(part.tool)
508543
if (part.state.status === "completed") {
509-
if (part.state.attachments?.length) {
510-
result.push({
511-
id: Identifier.ascending("message"),
512-
role: "user",
513-
parts: [
514-
{
515-
type: "text",
516-
text: `The tool ${part.tool} returned the following attachments:`,
517-
},
518-
...part.state.attachments.map((attachment) => ({
519-
type: "file" as const,
520-
url: attachment.url,
521-
mediaType: attachment.mime,
522-
filename: attachment.filename,
523-
})),
524-
],
525-
})
526-
}
544+
const outputText = part.state.time.compacted ? "[Old tool result content cleared]" : part.state.output
545+
const attachments = part.state.time.compacted ? [] : (part.state.attachments ?? [])
546+
const output =
547+
attachments.length > 0
548+
? {
549+
text: outputText,
550+
attachments,
551+
}
552+
: outputText
553+
527554
assistantMessage.parts.push({
528555
type: ("tool-" + part.tool) as `tool-${string}`,
529556
state: "output-available",
530557
toolCallId: part.callID,
531558
input: part.state.input,
532-
output: part.state.time.compacted ? "[Old tool result content cleared]" : part.state.output,
559+
output,
533560
...(differentModel ? {} : { callProviderMetadata: part.metadata }),
534561
})
535562
}
@@ -568,7 +595,15 @@ export namespace MessageV2 {
568595
}
569596
}
570597

571-
return convertToModelMessages(result.filter((msg) => msg.parts.some((part) => part.type !== "step-start")))
598+
const tools = Object.fromEntries(Array.from(toolNames).map((toolName) => [toolName, { toModelOutput }]))
599+
600+
return convertToModelMessages(
601+
result.filter((msg) => msg.parts.some((part) => part.type !== "step-start")),
602+
{
603+
//@ts-expect-error (convertToModelMessages expects a ToolSet but only actually needs tools[name]?.toModelOutput)
604+
tools,
605+
},
606+
)
572607
}
573608

574609
export const stream = fn(Identifier.schema("session"), async function* (sessionID) {

‎packages/opencode/src/session/prompt.ts‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -722,12 +722,6 @@ export namespace SessionPrompt {
722722
)
723723
return result
724724
},
725-
toModelOutput(result) {
726-
return {
727-
type: "text",
728-
value: result.output,
729-
}
730-
},
731725
})
732726
}
733727

@@ -819,12 +813,6 @@ export namespace SessionPrompt {
819813
content: result.content, // directly return content to preserve ordering when outputting to model
820814
}
821815
}
822-
item.toModelOutput = (result) => {
823-
return {
824-
type: "text",
825-
value: result.output,
826-
}
827-
}
828816
tools[key] = item
829817
}
830818

‎packages/opencode/test/session/message-v2.test.ts‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ describe("session.message-v2.toModelMessage", () => {
262262
])
263263
})
264264

265-
test("converts assistant tool completion into tool-call + tool-result messages and emits attachment message", () => {
265+
test("converts assistant tool completion into tool-call + tool-result messages with attachments", () => {
266266
const userID = "m-user"
267267
const assistantID = "m-assistant"
268268

@@ -304,7 +304,7 @@ describe("session.message-v2.toModelMessage", () => {
304304
type: "file",
305305
mime: "image/png",
306306
filename: "attachment.png",
307-
url: "https://example.com/attachment.png",
307+
url: "data:image/png;base64,Zm9v",
308308
},
309309
],
310310
},
@@ -319,18 +319,6 @@ describe("session.message-v2.toModelMessage", () => {
319319
role: "user",
320320
content: [{ type: "text", text: "run tool" }],
321321
},
322-
{
323-
role: "user",
324-
content: [
325-
{ type: "text", text: "The tool bash returned the following attachments:" },
326-
{
327-
type: "file",
328-
mediaType: "image/png",
329-
filename: "attachment.png",
330-
data: "https://example.com/attachment.png",
331-
},
332-
],
333-
},
334322
{
335323
role: "assistant",
336324
content: [
@@ -352,7 +340,13 @@ describe("session.message-v2.toModelMessage", () => {
352340
type: "tool-result",
353341
toolCallId: "call-1",
354342
toolName: "bash",
355-
output: { type: "text", value: "ok" },
343+
output: {
344+
type: "content",
345+
value: [
346+
{ type: "text", text: "ok" },
347+
{ type: "media", mediaType: "image/png", data: "Zm9v" },
348+
],
349+
},
356350
providerOptions: { openai: { tool: "meta" } },
357351
},
358352
],

0 commit comments

Comments
 (0)