Skip to content

Commit 306976d

Browse files
author
Bertan Ari
committed
test(vscode-lm): cover leaked tool-call salvage and tool_result truncation paths
Raises patch coverage on the new vscode-lm reliability code above the 80%% codecov/patch gate by exercising the streaming salvage state machine (marker split across chunks, multi-chunk buffering, unknown-tool passthrough, carried tail) and the tool_result truncation helpers (array-form content, surrogate-safe middle-out, guard clauses).
1 parent b4e1727 commit 306976d

1 file changed

Lines changed: 216 additions & 0 deletions

File tree

‎src/api/providers/__tests__/vscode-lm.spec.ts‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,108 @@ describe("VsCodeLmHandler", () => {
276276
})
277277
})
278278

279+
describe("leaked tool-call recovery during streaming", () => {
280+
const salvageTools = [
281+
{
282+
type: "function" as const,
283+
function: {
284+
name: "calculator",
285+
description: "A simple calculator",
286+
parameters: { type: "object", properties: { operation: { type: "string" } } },
287+
},
288+
},
289+
]
290+
291+
const streamTextParts = (parts: string[]) => {
292+
mockLanguageModelChat.sendRequest.mockResolvedValueOnce({
293+
stream: (async function* () {
294+
for (const part of parts) {
295+
yield new vscode.LanguageModelTextPart(part)
296+
}
297+
return
298+
})(),
299+
text: (async function* () {
300+
yield parts.join("")
301+
return
302+
})(),
303+
})
304+
}
305+
306+
const collect = async (parts: string[]) => {
307+
streamTextParts(parts)
308+
const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], {
309+
taskId: "test-task",
310+
tools: salvageTools,
311+
})
312+
const chunks = []
313+
for await (const chunk of stream) {
314+
chunks.push(chunk)
315+
}
316+
return chunks
317+
}
318+
319+
it("recovers a tool call the model streamed as raw invoke XML", async () => {
320+
const chunks = await collect([
321+
"Thinking. ",
322+
'<invoke name="calculator"><parameter name="operation">add</parameter></invoke>',
323+
])
324+
325+
expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "Thinking. " }])
326+
expect(chunks.filter((chunk) => chunk.type === "tool_call")).toEqual([
327+
{
328+
type: "tool_call",
329+
id: expect.stringContaining("vscodelm-salvaged-"),
330+
name: "calculator",
331+
arguments: JSON.stringify({ operation: "add" }),
332+
},
333+
])
334+
})
335+
336+
it("detects a marker split across stream chunks", async () => {
337+
const chunks = await collect([
338+
"abc <inv",
339+
'oke name="calculator"><parameter name="operation">sub</parameter></invoke>',
340+
])
341+
342+
expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "abc " }])
343+
expect(chunks.filter((chunk) => chunk.type === "tool_call")).toMatchObject([
344+
{ name: "calculator", arguments: JSON.stringify({ operation: "sub" }) },
345+
])
346+
})
347+
348+
it("emits a carried tail as plain text when it never becomes a marker", async () => {
349+
const chunks = await collect(["hello <par"])
350+
351+
expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([
352+
{ type: "text", text: "hello " },
353+
{ type: "text", text: "<par" },
354+
])
355+
expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false)
356+
})
357+
358+
it("buffers across chunks that arrive after the marker", async () => {
359+
const chunks = await collect([
360+
'prose <invoke name="calculator">',
361+
'<parameter name="operation">',
362+
"mul</parameter>",
363+
"</invoke>",
364+
])
365+
366+
expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "prose " }])
367+
expect(chunks.filter((chunk) => chunk.type === "tool_call")).toMatchObject([
368+
{ name: "calculator", arguments: JSON.stringify({ operation: "mul" }) },
369+
])
370+
})
371+
372+
it("keeps an invoke block for an unknown tool as literal text", async () => {
373+
const block = '<invoke name="not_our_tool"><parameter name="a">1</parameter></invoke>'
374+
const chunks = await collect([block])
375+
376+
expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: block }])
377+
expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false)
378+
})
379+
})
380+
279381
it("should handle native tool calls when tools are provided", async () => {
280382
const systemPrompt = "You are a helpful assistant"
281383
const messages: Anthropic.Messages.MessageParam[] = [
@@ -1166,6 +1268,15 @@ describe("context-window tool_result truncation", () => {
11661268
it("returns an empty string for a non-positive limit", () => {
11671269
expect(middleOutTruncate("anything", 0)).toBe("")
11681270
})
1271+
1272+
// A lone surrogate cannot be encoded as UTF-8 and 400s the whole request.
1273+
it("never splits a surrogate pair across the removed middle", () => {
1274+
const pair = "\u{1F600}" // one astral char = high + low surrogate
1275+
const text = pair.repeat(400)
1276+
const result = middleOutTruncate(text, 200)
1277+
1278+
expect(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/.test(result)).toBe(false)
1279+
})
11691280
})
11701281

11711282
describe("truncateToolResultsToFitWindow", () => {
@@ -1202,6 +1313,111 @@ describe("context-window tool_result truncation", () => {
12021313
expect(messages).toEqual(before)
12031314
})
12041315

1316+
it("returns messages untouched when the budget is not a usable number", () => {
1317+
const messages: Anthropic.Messages.MessageParam[] = [
1318+
toolUseMessage("t1"),
1319+
toolResultMessage("t1", "Y".repeat(50_000)),
1320+
]
1321+
const before = JSON.parse(JSON.stringify(messages))
1322+
1323+
expect(truncateToolResultsToFitWindow(messages, 0)).toBe(messages)
1324+
expect(truncateToolResultsToFitWindow(messages, Number.NaN)).toBe(messages)
1325+
expect(messages).toEqual(before)
1326+
})
1327+
1328+
it("truncates array-form tool_result content and preserves non-text parts", () => {
1329+
const messages: Anthropic.Messages.MessageParam[] = [
1330+
toolUseMessage("t1"),
1331+
{
1332+
role: "user",
1333+
content: [
1334+
{
1335+
type: "tool_result",
1336+
tool_use_id: "t1",
1337+
content: [
1338+
{ type: "text", text: "Z".repeat(50_000) },
1339+
{ type: "image", source: { type: "base64", media_type: "image/png", data: "abc" } },
1340+
],
1341+
},
1342+
],
1343+
} as unknown as Anthropic.Messages.MessageParam,
1344+
]
1345+
1346+
truncateToolResultsToFitWindow(messages, 10_000)
1347+
1348+
const toolResult = findBlock(messages[1], "tool_result")
1349+
const parts = toolResult.content as Array<{ type: string; text?: string }>
1350+
expect(parts[0].type).toBe("text")
1351+
expect(parts[0].text).toContain("characters truncated")
1352+
expect(parts.some((part) => part.type === "image")).toBe(true)
1353+
})
1354+
1355+
it("ignores string content and skips messages that cannot hold tool_result blocks", () => {
1356+
const messages: Anthropic.Messages.MessageParam[] = [
1357+
{ role: "user", content: "a plain string turn" },
1358+
toolUseMessage("t1"),
1359+
{
1360+
role: "user",
1361+
content: [
1362+
{ type: "tool_result", tool_use_id: "t1", content: "W".repeat(50_000) },
1363+
{ type: "image", source: { type: "base64", media_type: "image/png", data: "abc" } },
1364+
],
1365+
} as unknown as Anthropic.Messages.MessageParam,
1366+
]
1367+
1368+
truncateToolResultsToFitWindow(messages, 10_000)
1369+
1370+
expect(messages[0].content).toBe("a plain string turn")
1371+
expect(String(findBlock(messages[2], "tool_result").content)).toContain("characters truncated")
1372+
})
1373+
1374+
it("ignores a tool_result whose content is neither string nor array", () => {
1375+
const messages: Anthropic.Messages.MessageParam[] = [
1376+
toolUseMessage("t1"),
1377+
toolResultMessage("t1", "V".repeat(50_000)),
1378+
{
1379+
role: "user",
1380+
content: [{ type: "tool_result", tool_use_id: "t2", content: undefined }],
1381+
} as unknown as Anthropic.Messages.MessageParam,
1382+
]
1383+
1384+
truncateToolResultsToFitWindow(messages, 10_000)
1385+
1386+
expect(findBlock(messages[2], "tool_result").content).toBeUndefined()
1387+
expect(String(findBlock(messages[1], "tool_result").content)).toContain("characters truncated")
1388+
})
1389+
1390+
it("skips a tool_result already small enough to need no trimming", () => {
1391+
// Overage is tiny, so the largest block's target lands at its current length.
1392+
const messages: Anthropic.Messages.MessageParam[] = [
1393+
toolUseMessage("t1"),
1394+
toolResultMessage("t1", "U".repeat(3000)),
1395+
toolUseMessage("t2"),
1396+
toolResultMessage("t2", "T".repeat(2500)),
1397+
]
1398+
1399+
truncateToolResultsToFitWindow(messages, 5600)
1400+
1401+
const first = String(findBlock(messages[1], "tool_result").content)
1402+
const second = String(findBlock(messages[3], "tool_result").content)
1403+
expect(first.length + second.length).toBeLessThanOrEqual(5600)
1404+
})
1405+
1406+
it("leaves a tool_result at or below the minimum size alone", () => {
1407+
const shortResult = "S".repeat(1500)
1408+
const messages: Anthropic.Messages.MessageParam[] = [
1409+
toolUseMessage("t1"),
1410+
toolResultMessage("t1", shortResult),
1411+
toolUseMessage("t2"),
1412+
toolResultMessage("t2", shortResult),
1413+
]
1414+
1415+
truncateToolResultsToFitWindow(messages, 100)
1416+
1417+
expect(findBlock(messages[1], "tool_result").content).toBe(shortResult)
1418+
expect(findBlock(messages[3], "tool_result").content).toBe(shortResult)
1419+
})
1420+
12051421
it("shrinks an oversized tool_result so the conversation fits the budget", () => {
12061422
const messages: Anthropic.Messages.MessageParam[] = [
12071423
toolUseMessage("t1"),

0 commit comments

Comments
 (0)