Skip to content

Commit 6646ee3

Browse files
author
Bertan Ari
committed
fix(vscode-lm): replace unpaired UTF-16 surrogates before sending requests
A lone surrogate cannot be encoded as UTF-8, so the VS Code LM backend rejects the entire request with a 400. Sanitize message text, tool_result content, nested tool-call arguments, and the system prompt at the conversion boundary.
1 parent 01c7357 commit 6646ee3

4 files changed

Lines changed: 168 additions & 10 deletions

File tree

‎src/api/providers/__tests__/vscode-lm.spec.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,27 @@ describe("VsCodeLmHandler", () => {
272272
})
273273
})
274274

275+
describe("system prompt sanitization", () => {
276+
it("sanitizes lone surrogates in the system prompt", async () => {
277+
mockLanguageModelChat.sendRequest.mockResolvedValueOnce({
278+
stream: (async function* () {
279+
yield new vscode.LanguageModelTextPart("ok")
280+
return
281+
})(),
282+
text: (async function* () {
283+
yield "ok"
284+
return
285+
})(),
286+
})
287+
const stream = handler.createMessage("sys\uD800tem", [{ role: "user" as const, content: "hi" }])
288+
for await (const _chunk of stream) {
289+
// drain
290+
}
291+
292+
expect(vscode.LanguageModelChatMessage.Assistant).toHaveBeenCalledWith("sys\uFFFDtem")
293+
})
294+
})
295+
275296
it("should handle native tool calls when tools are provided", async () => {
276297
const systemPrompt = "You are a helpful assistant"
277298
const messages: Anthropic.Messages.MessageParam[] = [

‎src/api/providers/vscode-lm.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import { SELECTOR_SEPARATOR, stringifyVsCodeLmModelSelector } from "../../shared
1515
import { normalizeToolSchema } from "../../utils/json-schema"
1616

1717
import { ApiStream } from "../transform/stream"
18-
import { convertToVsCodeLmMessages, extractTextCountFromMessage } from "../transform/vscode-lm-format"
18+
import { convertToVsCodeLmMessages, extractTextCountFromMessage, sanitizeSurrogates } from "../transform/vscode-lm-format"
1919

2020
import { BaseProvider } from "./base-provider"
2121
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index"
@@ -391,7 +391,7 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
391391

392392
// Convert Anthropic messages to VS Code LM messages
393393
const vsCodeLmMessages: vscode.LanguageModelChatMessage[] = [
394-
vscode.LanguageModelChatMessage.Assistant(systemPrompt),
394+
vscode.LanguageModelChatMessage.Assistant(sanitizeSurrogates(systemPrompt)),
395395
...convertToVsCodeLmMessages(cleanedMessages),
396396
]
397397

‎src/api/transform/__tests__/vscode-lm-format.spec.ts‎

Lines changed: 97 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,12 @@
33
import { Anthropic } from "@anthropic-ai/sdk"
44
import * as vscode from "vscode"
55

6-
import { convertToVsCodeLmMessages, convertToAnthropicRole, extractTextCountFromMessage } from "../vscode-lm-format"
6+
import {
7+
convertToVsCodeLmMessages,
8+
convertToAnthropicRole,
9+
extractTextCountFromMessage,
10+
sanitizeSurrogates,
11+
} from "../vscode-lm-format"
712

813
// Mock crypto using Vitest
914
vitest.stubGlobal("crypto", {
@@ -325,6 +330,97 @@ describe("convertToVsCodeLmMessages", () => {
325330
})
326331
})
327332

333+
describe("sanitizeSurrogates", () => {
334+
it("leaves plain ASCII unchanged", () => {
335+
expect(sanitizeSurrogates("hello world")).toBe("hello world")
336+
})
337+
338+
it("leaves valid surrogate pairs unchanged", () => {
339+
// 😀 U+1F600 and 𐀀 U+10000 are astral-plane code points encoded as surrogate pairs.
340+
expect(sanitizeSurrogates("a\uD83D\uDE00b\uD800\uDC00c")).toBe("a\uD83D\uDE00b\uD800\uDC00c")
341+
})
342+
343+
it("replaces a lone high surrogate with U+FFFD", () => {
344+
expect(sanitizeSurrogates("a\uD800b")).toBe("a\uFFFDb")
345+
})
346+
347+
it("replaces a lone low surrogate with U+FFFD", () => {
348+
expect(sanitizeSurrogates("a\uDC00b")).toBe("a\uFFFDb")
349+
})
350+
351+
it("replaces a trailing lone high surrogate", () => {
352+
expect(sanitizeSurrogates("abc\uD800")).toBe("abc\uFFFD")
353+
})
354+
355+
it("replaces a reversed (low-then-high) pair as two lone surrogates", () => {
356+
expect(sanitizeSurrogates("\uDC00\uD800")).toBe("\uFFFD\uFFFD")
357+
})
358+
359+
it("returns empty input unchanged", () => {
360+
expect(sanitizeSurrogates("")).toBe("")
361+
})
362+
})
363+
364+
describe("convertToVsCodeLmMessages surrogate sanitization", () => {
365+
const lone = "bad\uD800end"
366+
const sanitized = "bad\uFFFDend"
367+
368+
const textValues = (message: { content: unknown }) =>
369+
(message.content as MockLanguageModelTextPart[]).map((part) => part.value)
370+
371+
it("sanitizes a simple string message", () => {
372+
const result = convertToVsCodeLmMessages([{ role: "user", content: lone }])
373+
expect(textValues(result[0])).toEqual([sanitized])
374+
})
375+
376+
it("sanitizes string tool_result content", () => {
377+
const result = convertToVsCodeLmMessages([
378+
{ role: "user", content: [{ type: "tool_result", tool_use_id: "tool-1", content: lone }] },
379+
])
380+
const toolResult = result[0].content[0] as MockLanguageModelToolResultPart
381+
expect(toolResult.content[0].value).toBe(sanitized)
382+
})
383+
384+
it("sanitizes tool_result text blocks", () => {
385+
const result = convertToVsCodeLmMessages([
386+
{
387+
role: "user",
388+
content: [{ type: "tool_result", tool_use_id: "tool-1", content: [{ type: "text", text: lone }] }],
389+
},
390+
])
391+
const toolResult = result[0].content[0] as MockLanguageModelToolResultPart
392+
expect(toolResult.content[0].value).toBe(sanitized)
393+
})
394+
395+
it("sanitizes user text blocks", () => {
396+
const result = convertToVsCodeLmMessages([{ role: "user", content: [{ type: "text", text: lone }] }])
397+
expect(textValues(result[0])).toContain(sanitized)
398+
})
399+
400+
it("sanitizes strings nested in tool_use input", () => {
401+
const result = convertToVsCodeLmMessages([
402+
{
403+
role: "assistant",
404+
content: [
405+
{
406+
type: "tool_use",
407+
id: "tool-1",
408+
name: "read_file",
409+
input: { path: lone, nested: { list: [lone] } },
410+
},
411+
],
412+
},
413+
])
414+
const toolCall = result[0].content[0] as MockLanguageModelToolCallPart
415+
expect(toolCall.input).toEqual({ path: sanitized, nested: { list: [sanitized] } })
416+
})
417+
418+
it("sanitizes assistant text blocks", () => {
419+
const result = convertToVsCodeLmMessages([{ role: "assistant", content: [{ type: "text", text: lone }] }])
420+
expect(textValues(result[0])).toContain(sanitized)
421+
})
422+
})
423+
328424
describe("convertToAnthropicRole", () => {
329425
it("should convert assistant role correctly", () => {
330426
const result = convertToAnthropicRole(vscode.LanguageModelChatMessageRole.Assistant)

‎src/api/transform/vscode-lm-format.ts‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,46 @@ function asObjectSafe(value: unknown): object {
2828
}
2929
}
3030

31+
/**
32+
* Replaces unpaired UTF-16 surrogate code units with the Unicode replacement character (U+FFFD).
33+
*
34+
* The VS Code LM backend forwards requests to model APIs that require valid UTF-8. A lone surrogate
35+
* — e.g. left behind when some upstream step slices a string through an astral-plane character
36+
* (emoji, CJK extension, etc.) — cannot be encoded as UTF-8, so the backend rejects the entire
37+
* request with a 400 ("string contains an unpaired UTF-16 surrogate code point and cannot be
38+
* encoded as valid UTF-8"). Valid surrogate pairs are matched by the lookahead/lookbehind and left
39+
* untouched. The regex intentionally omits the `u` flag so it operates on UTF-16 code units.
40+
*/
41+
export function sanitizeSurrogates(text: string): string {
42+
if (!text) {
43+
return text
44+
}
45+
return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g, "\uFFFD")
46+
}
47+
48+
/**
49+
* Applies {@link sanitizeSurrogates} to every string nested in a tool-call argument object. The
50+
* backend rejects the whole request for a lone surrogate anywhere in the JSON payload, so a tool
51+
* argument carrying a sliced astral character fails the request just as message text would.
52+
*/
53+
function sanitizeSurrogatesDeep(value: unknown): unknown {
54+
if (typeof value === "string") {
55+
return sanitizeSurrogates(value)
56+
}
57+
if (Array.isArray(value)) {
58+
return value.map(sanitizeSurrogatesDeep)
59+
}
60+
if (value && typeof value === "object") {
61+
return Object.fromEntries(
62+
Object.entries(value as Record<string, unknown>).map(([key, nested]) => [
63+
sanitizeSurrogates(key),
64+
sanitizeSurrogatesDeep(nested),
65+
]),
66+
)
67+
}
68+
return value
69+
}
70+
3171
export function convertToVsCodeLmMessages(
3272
anthropicMessages: Anthropic.Messages.MessageParam[],
3373
): vscode.LanguageModelChatMessage[] {
@@ -36,10 +76,11 @@ export function convertToVsCodeLmMessages(
3676
for (const anthropicMessage of anthropicMessages) {
3777
// Handle simple string messages
3878
if (typeof anthropicMessage.content === "string") {
79+
const safeContent = sanitizeSurrogates(anthropicMessage.content)
3980
vsCodeLmMessages.push(
4081
anthropicMessage.role === "assistant"
41-
? vscode.LanguageModelChatMessage.Assistant(anthropicMessage.content)
42-
: vscode.LanguageModelChatMessage.User(anthropicMessage.content),
82+
? vscode.LanguageModelChatMessage.Assistant(safeContent)
83+
: vscode.LanguageModelChatMessage.User(safeContent),
4384
)
4485
continue
4586
}
@@ -69,7 +110,7 @@ export function convertToVsCodeLmMessages(
69110
// Process tool result content into TextParts
70111
const toolContentParts: vscode.LanguageModelTextPart[] =
71112
typeof toolMessage.content === "string"
72-
? [new vscode.LanguageModelTextPart(toolMessage.content)]
113+
? [new vscode.LanguageModelTextPart(sanitizeSurrogates(toolMessage.content))]
73114
: (toolMessage.content?.map((part) => {
74115
if (part.type === "image") {
75116
if (part.source.type === "base64") {
@@ -82,7 +123,7 @@ export function convertToVsCodeLmMessages(
82123
)
83124
}
84125
if (part.type === "text") {
85-
return new vscode.LanguageModelTextPart(part.text)
126+
return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
86127
}
87128
return new vscode.LanguageModelTextPart("")
88129
}) ?? [new vscode.LanguageModelTextPart("")])
@@ -102,7 +143,7 @@ export function convertToVsCodeLmMessages(
102143
`[Image (${part.source.type}): not supported by VSCode LM API]`,
103144
)
104145
}
105-
return new vscode.LanguageModelTextPart(part.text)
146+
return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
106147
}),
107148
]
108149

@@ -135,7 +176,7 @@ export function convertToVsCodeLmMessages(
135176
if (part.type === "image") {
136177
return new vscode.LanguageModelTextPart("[Image generation not supported by VSCode LM API]")
137178
}
138-
return new vscode.LanguageModelTextPart(part.text)
179+
return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text))
139180
}),
140181

141182
// Convert tool messages to ToolCallParts after text
@@ -144,7 +185,7 @@ export function convertToVsCodeLmMessages(
144185
new vscode.LanguageModelToolCallPart(
145186
toolMessage.id,
146187
toolMessage.name,
147-
asObjectSafe(toolMessage.input),
188+
sanitizeSurrogatesDeep(asObjectSafe(toolMessage.input)) as object,
148189
),
149190
),
150191
]

0 commit comments

Comments
 (0)