Skip to content

Commit eeef7a3

Browse files
committed
fix: never split surrogate pairs at the compaction and kiro boundaries
Three more lone-surrogate U+FFFD producers found by the #904 coverage audit: the v1 compaction retained tail could begin on a lone low surrogate when the budget cut landed mid-pair; the kiro reasoning carry could emit a delta ending on a lone high surrogate; and kiro tool-description truncation could end on one. Each cut now shifts by one code unit to keep pairs whole, with boundary-pinned regressions (the 80,001-code-unit astral repro for compaction included).
1 parent 831283d commit eeef7a3

5 files changed

Lines changed: 68 additions & 4 deletions

File tree

‎src/adapters/kiro-thinking.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,16 @@ export class KiroThinkingParser {
8989
return events;
9090
}
9191
if (this.thinkingBuffer.length <= MAX_CLOSE_TAG) return [];
92-
const send = this.thinkingBuffer.slice(0, -MAX_CLOSE_TAG);
93-
this.replaceCarry("thinkingBuffer", this.thinkingBuffer.slice(-MAX_CLOSE_TAG));
92+
// Never split a surrogate pair at the send boundary: a lone high
93+
// surrogate at the end of one delta encodes as U+FFFD. Move the cut one
94+
// unit earlier so the whole pair stays in the carry.
95+
let cut = this.thinkingBuffer.length - MAX_CLOSE_TAG;
96+
if (cut > 0 && cut < this.thinkingBuffer.length) {
97+
const atCut = this.thinkingBuffer.charCodeAt(cut - 1);
98+
if (atCut >= 0xd800 && atCut <= 0xdbff) cut -= 1;
99+
}
100+
const send = this.thinkingBuffer.slice(0, cut);
101+
this.replaceCarry("thinkingBuffer", this.thinkingBuffer.slice(cut));
94102
return send ? [{ type: "reasoning_raw_delta", text: send }] : [];
95103
}
96104
}

‎src/adapters/kiro-tools.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,16 @@ function toolDescriptionLimit(modelId: string): number {
149149
function truncateDescription(description: string, limit: number): string {
150150
if (description.length <= limit) return description;
151151
if (limit <= 1) return description.slice(0, limit);
152-
return `${description.slice(0, limit - 1)}…`;
152+
let end = limit - 1;
153+
// Never end the kept text on a lone high surrogate; one step back keeps
154+
// the whole pair out instead of a U+FFFD-producing half.
155+
if (description.charCodeAt(end - 1) >= 0xd800 && description.charCodeAt(end - 1) <= 0xdbff) end -= 1;
156+
return `${description.slice(0, end)}…`;
157+
}
158+
159+
/** Test-only: exercise the surrogate-safe description truncation directly. */
160+
export function truncateDescriptionForTests(description: string, limit: number): string {
161+
return truncateDescription(description, limit);
153162
}
154163

155164
function serializedToolCatalogBytes(tools: readonly unknown[]): number {

‎src/responses/compaction.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,14 @@ export function buildCompactV1Output(userMessages: string[], summary: string): R
105105
remaining -= msg.length;
106106
} else {
107107
// Budget partially covers this older message: keep its tail (most recent context) and stop.
108-
selected.push(msg.slice(msg.length - remaining));
108+
let tailStart = msg.length - remaining;
109+
// Never start the retained tail on a lone LOW surrogate: the pair's
110+
// other half would be lost and encoding substitutes U+FFFD.
111+
if (tailStart > 0 && tailStart < msg.length) {
112+
const first = msg.charCodeAt(tailStart);
113+
if (first >= 0xdc00 && first <= 0xdfff) tailStart += 1;
114+
}
115+
selected.push(msg.slice(tailStart));
109116
break;
110117
}
111118
}

‎tests/kiro-stream.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1728,3 +1728,31 @@ describe("kiro adapter — parseResponse (web-search sidecar non-streaming path)
17281728
expect(usage?.contextTotalTokens).toBeGreaterThanOrEqual(builtEstimate);
17291729
});
17301730
});
1731+
1732+
describe("surrogate safety at kiro boundaries", () => {
1733+
test("the reasoning carry never emits a delta ending on a lone high surrogate", async () => {
1734+
const { KiroThinkingParser } = await import("../src/adapters/kiro-thinking");
1735+
const parser = new KiroThinkingParser();
1736+
// An astral char exactly at the carry/send boundary.
1737+
const events = parser.feed("<thinking>🎆aaaaaaaaaaa");
1738+
const emitted = JSON.stringify(events);
1739+
expect(emitted.includes("\uFFFD")).toBe(false);
1740+
for (const event of events) {
1741+
const text = (event as { text?: string }).text ?? "";
1742+
if (text.length === 0) continue;
1743+
const last = text.charCodeAt(text.length - 1);
1744+
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
1745+
}
1746+
});
1747+
1748+
test("a truncated tool description never ends on a lone high surrogate", async () => {
1749+
const { truncateDescriptionForTests } = await import("../src/adapters/kiro-tools");
1750+
const description = "a".repeat(1022) + "🎆cd";
1751+
const out = truncateDescriptionForTests(description, 1024);
1752+
expect(out.endsWith("…")).toBe(true);
1753+
const kept = out.slice(0, -1);
1754+
const last = kept.charCodeAt(kept.length - 1);
1755+
expect(last >= 0xd800 && last <= 0xdbff).toBe(false);
1756+
expect(out.includes("\uFFFD")).toBe(false);
1757+
});
1758+
});

‎tests/responses-compaction.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,4 +265,16 @@ describe("remote compaction v1 helpers (260707 Design-B sweep)", () => {
265265
expect(second.content[0].text).toBe(recent);
266266
expect(first.content[0].text.length).toBe(80_000 - recent.length);
267267
});
268+
269+
test("the retained tail never begins on a lone low surrogate", () => {
270+
// The reviewer's repro: an 80,001-code-unit message BEGINNING with an
271+
// astral character, so the 80k budget cut lands exactly inside the pair.
272+
const withAstral = "🎆" + "가".repeat(79_999);
273+
expect(withAstral.length).toBe(80_001);
274+
const output = buildCompactV1Output([withAstral], "summary");
275+
const retained = (output[output.length - 2] as { content: { text: string }[] }).content[0].text;
276+
const first = retained.charCodeAt(0);
277+
expect(first >= 0xdc00 && first <= 0xdfff).toBe(false);
278+
expect(retained.includes("\uFFFD")).toBe(false);
279+
});
268280
});

0 commit comments

Comments
 (0)