Skip to content

Commit eb81eaa

Browse files
authored
Merge pull request #4497 from lidge-jun/codex/260913-chat-tool-image-parts
fix(chat): accept Pi and Anthropic-shaped image parts on the chat wire
2 parents 248670e + 9b9675b commit eb81eaa

2 files changed

Lines changed: 34 additions & 4 deletions

File tree

‎src/chat/inbound.ts‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,35 @@ function contentToText(content: unknown): string {
4646
}
4747

4848
function imageUrlFromPart(part: Rec): string | null {
49-
if (part.type !== "image_url") return null;
50-
const imageUrl = part.image_url;
51-
if (typeof imageUrl === "string" && imageUrl.length > 0) return imageUrl;
52-
if (isRec(imageUrl) && typeof imageUrl.url === "string" && imageUrl.url.length > 0) return imageUrl.url;
49+
if (part.type === "image_url") {
50+
const imageUrl = part.image_url;
51+
if (typeof imageUrl === "string" && imageUrl.length > 0) return imageUrl;
52+
if (isRec(imageUrl) && typeof imageUrl.url === "string" && imageUrl.url.length > 0) return imageUrl.url;
53+
return null;
54+
}
55+
// Agent clients whose native wire shape is not OpenAI's still send images over
56+
// Chat Completions: Pi/MCP-style parts carry {type:"image", data, mimeType}
57+
// (Aside read_file tool results), Anthropic-shaped clients carry a source
58+
// object. Dropping either silently blinds a vision model, so normalize both
59+
// to the URL/data-URI form the Responses pipeline already understands.
60+
if (part.type === "image") {
61+
const data = part.data;
62+
if (typeof data === "string" && data.length > 0) {
63+
if (data.startsWith("data:")) return data;
64+
const media = typeof part.mimeType === "string" && part.mimeType.length > 0 ? part.mimeType
65+
: typeof part.mediaType === "string" && part.mediaType.length > 0 ? part.mediaType
66+
: "image/png";
67+
return "data:" + media + ";base64," + data;
68+
}
69+
const source = part.source;
70+
if (isRec(source)) {
71+
if (source.type === "base64" && typeof source.data === "string" && source.data.length > 0) {
72+
const media = typeof source.media_type === "string" && source.media_type.length > 0 ? source.media_type : "image/png";
73+
return "data:" + media + ";base64," + source.data;
74+
}
75+
if (source.type === "url" && typeof source.url === "string" && source.url.length > 0) return source.url;
76+
}
77+
}
5378
return null;
5479
}
5580

‎tests/responses/chat-completions-endpoint.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,11 @@ describe("chatCompletionsToResponsesBody image parts", () => {
242242
{ part: { type: "image_url", image_url: "https://example.com/image.png" }, expected: { type: "input_image", image_url: "https://example.com/image.png" } },
243243
{ part: { type: "image_url", image_url: "https://example.com/image.png", detail: "low" }, expected: { type: "input_image", image_url: "https://example.com/image.png", detail: "low" } },
244244
{ part: { type: "image_url", image_url: { url: "https://example.com/image.png" } }, expected: { type: "input_image", image_url: "https://example.com/image.png" } },
245+
{ part: { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, expected: { type: "input_image", image_url: "data:image/png;base64,aGVsbG8=" } },
246+
{ part: { type: "image", data: "aGVsbG8=", mediaType: "image/jpeg" }, expected: { type: "input_image", image_url: "data:image/jpeg;base64,aGVsbG8=" } },
247+
{ part: { type: "image", data: "data:image/webp;base64,aGVsbG8=", mimeType: "image/png" }, expected: { type: "input_image", image_url: "data:image/webp;base64,aGVsbG8=" } },
248+
{ part: { type: "image", source: { type: "base64", media_type: "image/jpeg", data: "aGVsbG8=" } }, expected: { type: "input_image", image_url: "data:image/jpeg;base64,aGVsbG8=" } },
249+
{ part: { type: "image", source: { type: "url", url: "https://example.com/claude.png" } }, expected: { type: "input_image", image_url: "https://example.com/claude.png" } },
245250
])("preserves user image shorthand and omitted detail: %j", ({ part, expected }) => {
246251
const body = chatCompletionsToResponsesBody({ model: "mock/test-model", messages: [{ role: "user", content: [part] }] });
247252
expect(body.input).toEqual([{ type: "message", role: "user", content: [expected] }]);

0 commit comments

Comments
 (0)