Skip to content

Commit caa4bd3

Browse files
author
Bertan Ari
committed
feat(vscode-lm): support image input conversion and shim data parts
1 parent 64908ab commit caa4bd3

7 files changed

Lines changed: 125 additions & 25 deletions

File tree

‎packages/vscode-shim/src/__tests__/Additional.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
CodeActionKind,
88
CodeLens,
99
LanguageModelTextPart,
10+
LanguageModelDataPart,
1011
LanguageModelToolCallPart,
1112
LanguageModelToolResultPart,
1213
FileSystemError,
@@ -261,6 +262,54 @@ describe("LanguageModelTextPart", () => {
261262
})
262263
})
263264

265+
describe("LanguageModelDataPart", () => {
266+
const decode = (data: Uint8Array) => new TextDecoder().decode(data)
267+
268+
it("should create data part via constructor", () => {
269+
const data = new Uint8Array([1, 2, 3])
270+
const part = new LanguageModelDataPart(data, "image/png")
271+
272+
expect(part.data).toBe(data)
273+
expect(part.mimeType).toBe("image/png")
274+
})
275+
276+
describe("image()", () => {
277+
it("should create image part preserving bytes and mime type", () => {
278+
const data = new Uint8Array([137, 80, 78, 71])
279+
const part = LanguageModelDataPart.image(data, "image/png")
280+
281+
expect(part).toBeInstanceOf(LanguageModelDataPart)
282+
expect(part.data).toEqual(data)
283+
expect(part.mimeType).toBe("image/png")
284+
})
285+
286+
it("should round-trip base64-decoded image data", () => {
287+
const base64 = Buffer.from("fake-image-bytes").toString("base64")
288+
const part = LanguageModelDataPart.image(new Uint8Array(Buffer.from(base64, "base64")), "image/jpeg")
289+
290+
expect(decode(part.data)).toBe("fake-image-bytes")
291+
expect(part.mimeType).toBe("image/jpeg")
292+
})
293+
})
294+
})
295+
296+
describe("LanguageModelDataPart text and JSON", () => {
297+
it.each([undefined, "text/html"])("encodes text with MIME type %s", (mimeType) => {
298+
const part = LanguageModelDataPart.text("Hello 🌍", mimeType)
299+
expect(part).toBeInstanceOf(LanguageModelDataPart)
300+
expect(new TextDecoder().decode(part.data)).toBe("Hello 🌍")
301+
expect(part.mimeType).toBe(mimeType ?? "text/plain")
302+
})
303+
304+
it.each([undefined, "application/example+json"])("encodes JSON with MIME type %s", (mimeType) => {
305+
const value = { message: "Hello 🌍", count: 2 }
306+
const part = LanguageModelDataPart.json(value, mimeType)
307+
expect(part).toBeInstanceOf(LanguageModelDataPart)
308+
expect(JSON.parse(new TextDecoder().decode(part.data))).toEqual(value)
309+
expect(part.mimeType).toBe(mimeType ?? "application/json")
310+
})
311+
})
312+
264313
describe("LanguageModelToolCallPart", () => {
265314
it("should create tool call part", () => {
266315
const part = new LanguageModelToolCallPart("call-123", "searchFiles", { query: "test" })

‎packages/vscode-shim/src/api/create-vscode-api-mock.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
CodeActionKind,
2222
CodeLens,
2323
LanguageModelTextPart,
24+
LanguageModelDataPart,
2425
LanguageModelToolCallPart,
2526
LanguageModelToolResultPart,
2627
FileSystemError,
@@ -161,6 +162,7 @@ export function createVSCodeAPIMock(
161162
CancellationTokenSource,
162163
CodeLens,
163164
LanguageModelTextPart,
165+
LanguageModelDataPart,
164166
LanguageModelToolCallPart,
165167
LanguageModelToolResultPart,
166168
ExtensionContext: ExtensionContextImpl,

‎packages/vscode-shim/src/classes/Additional.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,25 @@ export class LanguageModelTextPart {
112112
constructor(public value: string) {}
113113
}
114114

115+
export class LanguageModelDataPart {
116+
constructor(
117+
public data: Uint8Array,
118+
public mimeType: string,
119+
) {}
120+
121+
static image(data: Uint8Array, mimeType: string): LanguageModelDataPart {
122+
return new LanguageModelDataPart(data, mimeType)
123+
}
124+
125+
static text(value: string, mimeType: string = "text/plain"): LanguageModelDataPart {
126+
return new LanguageModelDataPart(new TextEncoder().encode(value), mimeType)
127+
}
128+
129+
static json(value: unknown, mimeType: string = "application/json"): LanguageModelDataPart {
130+
return new LanguageModelDataPart(new TextEncoder().encode(JSON.stringify(value)), mimeType)
131+
}
132+
}
133+
115134
export class LanguageModelToolCallPart {
116135
constructor(
117136
public callId: string,

‎packages/vscode-shim/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export {
3030
CancellationTokenSource,
3131
CodeLens,
3232
LanguageModelTextPart,
33+
LanguageModelDataPart,
3334
LanguageModelToolCallPart,
3435
LanguageModelToolResultPart,
3536
FileSystemError,

‎packages/vscode-shim/src/vscode.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export {
2323
CodeActionKind,
2424
CodeLens,
2525
LanguageModelTextPart,
26+
LanguageModelDataPart,
2627
LanguageModelToolCallPart,
2728
LanguageModelToolResultPart,
2829
FileSystemError,

‎src/api/transform/__tests__/vscode-lm-format.spec.ts‎

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -174,23 +174,48 @@ describe("convertToVsCodeLmMessages", () => {
174174
expect(part.content[0]).toEqual({ data: Buffer.from("image"), mimeType: "image/jpeg" })
175175
})
176176

177-
it("does not silently replace unsupported image URLs when image input is enabled", () => {
178-
expect(() =>
179-
convertToVsCodeLmMessages(
180-
[
181-
{
182-
role: "user",
183-
content: [
184-
{
185-
type: "image",
186-
source: { type: "url", url: "https://example.com/image.png" },
187-
},
188-
],
189-
},
190-
],
191-
true,
192-
),
193-
).toThrow("requires base64 image data")
177+
describe.each(["message", "tool_result"] as const)("image conversion in %s", (path) => {
178+
function convertImage(source: Anthropic.ImageBlockParam["source"]) {
179+
const image: Anthropic.ImageBlockParam = { type: "image", source }
180+
const content: Anthropic.Messages.ContentBlockParam[] =
181+
path === "message" ? [image] : [{ type: "tool_result", tool_use_id: "image-tool", content: [image] }]
182+
const result = convertToVsCodeLmMessages([{ role: "user", content }], true)
183+
const part = result[0].content[0]
184+
return part instanceof vscode.LanguageModelToolResultPart ? part.content[0] : part
185+
}
186+
187+
it.each(["image/png", "image/jpeg", "image/gif", "image/webp"] as const)(
188+
"sends decoded bytes for %s",
189+
(mediaType) => {
190+
const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0xff, 0x00])
191+
expect(convertImage({ type: "base64", media_type: mediaType, data: bytes.toString("base64") })).toEqual(
192+
{ data: bytes, mimeType: mediaType },
193+
)
194+
},
195+
)
196+
197+
it("keeps a placeholder for URL images", () => {
198+
expect(convertImage({ type: "url", url: "https://example.com/image.png" })).toEqual(
199+
new vscode.LanguageModelTextPart("[Image (url): not supported by VSCode LM API]"),
200+
)
201+
})
202+
203+
it.each(["", "!!!"])("keeps a placeholder when base64 %j decodes to no bytes", (data) => {
204+
expect(convertImage({ type: "base64", media_type: "image/png", data })).toEqual(
205+
new vscode.LanguageModelTextPart("[Image (base64): image/png not supported by VSCode LM API]"),
206+
)
207+
})
208+
209+
it("keeps a placeholder for unsupported media types", () => {
210+
expect(
211+
convertImage({
212+
type: "base64",
213+
// Exercise runtime input outside the SDK's supported media-type union.
214+
media_type: "image/bmp" as "image/png",
215+
data: "aW1hZ2U=",
216+
}),
217+
).toEqual(new vscode.LanguageModelTextPart("[Image (base64): image/bmp not supported by VSCode LM API]"))
218+
})
194219
})
195220

196221
it("should convert simple string messages", () => {

‎src/api/transform/vscode-lm-format.ts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -93,16 +93,19 @@ export function decodeToolNameSurrogates(name: string): string {
9393
return decoded
9494
}
9595

96+
const SUPPORTED_IMAGE_MEDIA_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"])
97+
9698
function convertImageToVsCodeLmPart(image: Anthropic.Messages.ImageBlockParam, supportsImages: boolean) {
97-
if (supportsImages) {
98-
if (image.source.type !== "base64") {
99-
throw new Error("GitHub Copilot image input requires base64 image data, not an image URL.")
100-
}
101-
const part = createImagePart(Buffer.from(image.source.data, "base64"), image.source.media_type)
102-
if (!part) {
103-
throw new Error("Image input requires a newer version of VS Code that can send image data.")
99+
if (supportsImages && image.source.type === "base64" && SUPPORTED_IMAGE_MEDIA_TYPES.has(image.source.media_type)) {
100+
// Pass decoded bytes, not the base64 string, to avoid encoding the image twice.
101+
const bytes = Buffer.from(image.source.data, "base64")
102+
if (bytes.length > 0) {
103+
const part = createImagePart(bytes, image.source.media_type)
104+
if (!part) {
105+
throw new Error("Image input requires a newer version of VS Code that can send image data.")
106+
}
107+
return part
104108
}
105-
return part
106109
}
107110
return new vscode.LanguageModelTextPart(
108111
image.source.type === "base64"

0 commit comments

Comments
 (0)