Skip to content

Commit 843d235

Browse files
committed
fix(transform): handle URL-sourced images and non-base64 blocks in format converters
1 parent fae44be commit 843d235

8 files changed

Lines changed: 302 additions & 7 deletions

File tree

‎src/api/transform/__tests__/mistral-format.spec.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,45 @@ describe("convertToMistralMessages", () => {
106106
})
107107
})
108108

109+
it("should handle user messages with URL image content", () => {
110+
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
111+
{
112+
role: "user",
113+
content: [
114+
{ type: "text", text: "Describe this:" },
115+
{
116+
type: "image",
117+
source: { type: "url", url: "https://example.com/photo.jpg" } as any,
118+
},
119+
],
120+
},
121+
]
122+
123+
const mistralMessages = convertToMistralMessages(anthropicMessages)
124+
const content = mistralMessages[0].content as Array<{ type: string; imageUrl?: { url: string }; text?: string }>
125+
126+
expect(content[1]).toEqual({ type: "image_url", imageUrl: { url: "https://example.com/photo.jpg" } })
127+
})
128+
129+
it("should fall back to [Image] placeholder for unsupported image source types", () => {
130+
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
131+
{
132+
role: "user",
133+
content: [
134+
{
135+
type: "image",
136+
source: { type: "file" } as any,
137+
},
138+
],
139+
},
140+
]
141+
142+
const mistralMessages = convertToMistralMessages(anthropicMessages)
143+
const content = mistralMessages[0].content as Array<{ type: string; text?: string }>
144+
145+
expect(content[0]).toEqual({ type: "text", text: "[Image]" })
146+
})
147+
109148
it("should handle user messages with only tool results", () => {
110149
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
111150
{

‎src/api/transform/__tests__/openai-format.spec.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,97 @@ describe("convertToOpenAiMessages", () => {
7676
})
7777
})
7878

79+
it("should handle messages with URL image content", () => {
80+
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
81+
{
82+
role: "user",
83+
content: [
84+
{ type: "text", text: "What is in this image?" },
85+
{
86+
type: "image",
87+
source: { type: "url", url: "https://example.com/image.png" } as any,
88+
},
89+
],
90+
},
91+
]
92+
93+
const openAiMessages = convertToOpenAiMessages(anthropicMessages)
94+
const content = openAiMessages[0].content as Array<{ type: string; text?: string; image_url?: { url: string } }>
95+
96+
expect(content[1]).toEqual({
97+
type: "image_url",
98+
image_url: { url: "https://example.com/image.png" },
99+
})
100+
})
101+
102+
it("should fall back to [Image] placeholder for unsupported image source types", () => {
103+
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
104+
{
105+
role: "user",
106+
content: [
107+
{
108+
type: "image",
109+
source: { type: "file" } as any,
110+
},
111+
],
112+
},
113+
]
114+
115+
const openAiMessages = convertToOpenAiMessages(anthropicMessages)
116+
const content = openAiMessages[0].content as Array<{ type: string; text?: string }>
117+
118+
expect(content[0]).toEqual({ type: "text", text: "[Image]" })
119+
})
120+
121+
it("should handle non-text/non-image blocks in tool results as empty string", () => {
122+
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
123+
{
124+
role: "user",
125+
content: [
126+
{
127+
type: "tool_result",
128+
tool_use_id: "tool-1",
129+
content: [
130+
{ type: "text", text: "some text" },
131+
{
132+
type: "document",
133+
source: { type: "base64", media_type: "application/pdf", data: "abc" },
134+
} as any,
135+
],
136+
},
137+
],
138+
},
139+
]
140+
141+
const openAiMessages = convertToOpenAiMessages(anthropicMessages)
142+
expect(openAiMessages[0].role).toBe("tool")
143+
expect((openAiMessages[0] as any).content).toBe("some text\n")
144+
})
145+
146+
it("should handle base64 image in tool result with placeholder and skip URL images", () => {
147+
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
148+
{
149+
role: "user",
150+
content: [
151+
{
152+
type: "tool_result",
153+
tool_use_id: "tool-1",
154+
content: [
155+
{
156+
type: "image",
157+
source: { type: "url", url: "https://example.com/img.png" } as any,
158+
},
159+
],
160+
},
161+
],
162+
},
163+
]
164+
165+
const openAiMessages = convertToOpenAiMessages(anthropicMessages)
166+
expect(openAiMessages[0].role).toBe("tool")
167+
expect((openAiMessages[0] as any).content).toBe("[Image]")
168+
})
169+
79170
it("should handle assistant messages with tool use (no normalization without normalizeToolCallId)", () => {
80171
const anthropicMessages: Anthropic.Messages.MessageParam[] = [
81172
{

‎src/api/transform/__tests__/vscode-lm-format.spec.ts‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,48 @@ describe("convertToVsCodeLmMessages", () => {
178178
const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart
179179
expect(imagePlaceholder.value).toContain("[Image (base64): image/png not supported by VSCode LM API]")
180180
})
181+
182+
it("should produce correct placeholder for URL image in non-tool messages", () => {
183+
const messages: Anthropic.Messages.MessageParam[] = [
184+
{
185+
role: "user",
186+
content: [
187+
{
188+
type: "image",
189+
source: { type: "url", url: "https://example.com/img.png" } as any,
190+
},
191+
],
192+
},
193+
]
194+
195+
const result = convertToVsCodeLmMessages(messages)
196+
const imagePlaceholder = result[0].content[0] as MockLanguageModelTextPart
197+
expect(imagePlaceholder.value).toContain("[Image (url): not supported by VSCode LM API]")
198+
})
199+
200+
it("should produce correct placeholder for URL image inside tool result", () => {
201+
const messages: Anthropic.Messages.MessageParam[] = [
202+
{
203+
role: "user",
204+
content: [
205+
{
206+
type: "tool_result",
207+
tool_use_id: "tool-1",
208+
content: [
209+
{
210+
type: "image",
211+
source: { type: "url", url: "https://example.com/img.png" } as any,
212+
},
213+
],
214+
},
215+
],
216+
},
217+
]
218+
219+
const result = convertToVsCodeLmMessages(messages)
220+
const toolResult = result[0].content[0] as any
221+
expect(toolResult.content[0].value).toContain("[Image (url): not supported by VSCode LM API]")
222+
})
181223
})
182224

183225
describe("convertToAnthropicRole", () => {
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// npx vitest run api/transform/__tests__/zai-format.spec.ts
2+
3+
import { Anthropic } from "@anthropic-ai/sdk"
4+
5+
import { convertToZAiFormat } from "../zai-format"
6+
7+
describe("convertToZAiFormat", () => {
8+
it("should convert simple user text messages", () => {
9+
const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }]
10+
11+
const result = convertToZAiFormat(messages)
12+
expect(result).toHaveLength(1)
13+
expect(result[0]).toEqual({ role: "user", content: "Hello" })
14+
})
15+
16+
it("should handle base64 image in user message", () => {
17+
const messages: Anthropic.Messages.MessageParam[] = [
18+
{
19+
role: "user",
20+
content: [
21+
{ type: "text", text: "Describe this:" },
22+
{
23+
type: "image",
24+
source: { type: "base64", media_type: "image/png", data: "abc123" },
25+
},
26+
],
27+
},
28+
]
29+
30+
const result = convertToZAiFormat(messages)
31+
expect(result).toHaveLength(1)
32+
const content = (result[0] as any).content as Array<{
33+
type: string
34+
image_url?: { url: string }
35+
text?: string
36+
}>
37+
expect(content[0]).toEqual({ type: "text", text: "Describe this:" })
38+
expect(content[1]).toEqual({ type: "image_url", image_url: { url: "data:image/png;base64,abc123" } })
39+
})
40+
41+
it("should skip non-base64 images in user messages", () => {
42+
const messages: Anthropic.Messages.MessageParam[] = [
43+
{
44+
role: "user",
45+
content: [
46+
{ type: "text", text: "Hello" },
47+
{
48+
type: "image",
49+
source: { type: "url", url: "https://example.com/img.png" } as any,
50+
},
51+
],
52+
},
53+
]
54+
55+
const result = convertToZAiFormat(messages)
56+
expect(result).toHaveLength(1)
57+
// URL image is skipped — only text remains, so content is a plain string
58+
expect((result[0] as any).content).toBe("Hello")
59+
})
60+
61+
it("should convert tool_result content", () => {
62+
const messages: Anthropic.Messages.MessageParam[] = [
63+
{
64+
role: "user",
65+
content: [{ type: "tool_result", tool_use_id: "tool-1", content: "Result text" }],
66+
},
67+
]
68+
69+
const result = convertToZAiFormat(messages)
70+
expect(result[0]).toEqual({ role: "tool", tool_call_id: "tool-1", content: "Result text" })
71+
})
72+
73+
it("should convert assistant messages with text", () => {
74+
const messages: Anthropic.Messages.MessageParam[] = [{ role: "assistant", content: "I can help with that." }]
75+
76+
const result = convertToZAiFormat(messages)
77+
expect(result[0]).toEqual({ role: "assistant", content: "I can help with that." })
78+
})
79+
})

‎src/api/transform/mistral-format.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,12 @@ export function convertToMistralMessages(anthropicMessages: Anthropic.Messages.M
112112
},
113113
}
114114
}
115+
if (part.source.type === "url") {
116+
return {
117+
type: "image_url",
118+
imageUrl: { url: part.source.url },
119+
}
120+
}
115121
return { type: "text", text: "[Image]" }
116122
}
117123
if (part.type === "text") {

‎src/api/transform/openai-format.ts‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -358,8 +358,11 @@ export function convertToOpenAiMessages(
358358
toolMessage.content
359359
?.map((part) => {
360360
if (part.type === "image") {
361-
toolResultImages.push(part)
362-
return "(see following user message for image)"
361+
if (part.source.type === "base64") {
362+
toolResultImages.push(part)
363+
return "(see following user message for image)"
364+
}
365+
return "[Image]"
363366
}
364367
if (part.type === "text") {
365368
return part.text
@@ -428,7 +431,15 @@ export function convertToOpenAiMessages(
428431
if (part.source.type === "base64") {
429432
return {
430433
type: "image_url",
431-
image_url: { url: `data:${part.source.media_type};base64,${part.source.data}` },
434+
image_url: {
435+
url: `data:${part.source.media_type};base64,${part.source.data}`,
436+
},
437+
}
438+
}
439+
if (part.source.type === "url") {
440+
return {
441+
type: "image_url",
442+
image_url: { url: part.source.url },
432443
}
433444
}
434445
return { type: "text", text: "[Image]" }

‎src/api/transform/vscode-lm-format.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,11 @@ export function convertToVsCodeLmMessages(
7474
if (part.type === "image") {
7575
if (part.source.type === "base64") {
7676
return new vscode.LanguageModelTextPart(
77-
`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
77+
`[Image (base64): ${part.source.media_type} not supported by VSCode LM API]`,
7878
)
7979
}
8080
return new vscode.LanguageModelTextPart(
81-
`[Image (${part.source?.type || "Unknown source-type"}): not supported by VSCode LM API]`,
81+
`[Image (${part.source.type}): not supported by VSCode LM API]`,
8282
)
8383
}
8484
if (part.type === "text") {
@@ -95,11 +95,11 @@ export function convertToVsCodeLmMessages(
9595
if (part.type === "image") {
9696
if (part.source.type === "base64") {
9797
return new vscode.LanguageModelTextPart(
98-
`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
98+
`[Image (base64): ${part.source.media_type} not supported by VSCode LM API]`,
9999
)
100100
}
101101
return new vscode.LanguageModelTextPart(
102-
`[Image (${part.source?.type || "Unknown source-type"}): not supported by VSCode LM API]`,
102+
`[Image (${part.source.type}): not supported by VSCode LM API]`,
103103
)
104104
}
105105
if (part.type === "text") {

‎src/integrations/misc/__tests__/export-markdown.spec.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,5 +72,32 @@ describe("export-markdown", () => {
7272
const block = { type: "unknown_type" as const } as any
7373
expect(formatContentBlockToMarkdown(block)).toBe("[Unexpected content type: unknown_type]")
7474
})
75+
76+
it("should format document blocks", () => {
77+
const block = {
78+
type: "document",
79+
source: { type: "base64", media_type: "application/pdf", data: "abc" },
80+
} as any as ExtendedContentBlock
81+
expect(formatContentBlockToMarkdown(block)).toBe("[Document]")
82+
})
83+
84+
it("should format search_result blocks", () => {
85+
const block = {
86+
type: "search_result",
87+
source: "https://example.com",
88+
title: "Example",
89+
content: [],
90+
} as any as ExtendedContentBlock
91+
expect(formatContentBlockToMarkdown(block)).toBe("[Search Result]")
92+
})
93+
94+
it("should format tool_reference blocks", () => {
95+
const block = {
96+
type: "tool_reference",
97+
id: "tool-1",
98+
name: "read_file",
99+
} as any as ExtendedContentBlock
100+
expect(formatContentBlockToMarkdown(block)).toBe("[Tool Reference]")
101+
})
75102
})
76103
})

0 commit comments

Comments
 (0)