Skip to content

Commit a101c61

Browse files
authored
fix(read-file): preserve Unicode when clipping file lines (#1960)
1 parent ffb2435 commit a101c61

2 files changed

Lines changed: 149 additions & 1 deletion

File tree

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import { MAX_LINE_LENGTH } from "../../../core/prompts/tools/native-tools/read_file"
2+
import { formatWithLineNumbers, parseLines, readWithIndentation, readWithSlice } from "../indentation-reader"
3+
4+
describe("line formatting Unicode boundaries", () => {
5+
it("does not leave a dangling surrogate when clipping a long line", () => {
6+
const output = formatWithLineNumbers(parseLines("🔥".repeat(2000)))
7+
expect(output).toBe(`1 | ${"🔥".repeat(998)}...`)
8+
expect(output).toContain("...")
9+
expect(Buffer.from(output).toString("utf8")).not.toContain("�")
10+
expect(output.length).toBeLessThanOrEqual(MAX_LINE_LENGTH + 4)
11+
})
12+
13+
it.each([
14+
["pair crossing the boundary", "abcd🔥tail", "abcd..."],
15+
["complete pair before the boundary", "abc🔥tail", "abc🔥..."],
16+
["pair after the boundary", "abcde🔥tail", "abcde..."],
17+
["lowest high surrogate", `abcd${String.fromCodePoint(0x10000)}tail`, "abcd..."],
18+
["highest high surrogate", `abcd${String.fromCodePoint(0x10ffff)}tail`, "abcd..."],
19+
["BMP below the surrogate range", "abcd\ud7fftail", "abcd\ud7ff..."],
20+
["BMP above the surrogate range", "abcd\ue000tail", "abcd\ue000..."],
21+
["non-ASCII BMP text", "Жé文Ωßtail", "Жé文Ωß..."],
22+
["combining characters without normalization", "abcde\u0301tail", "abcde..."],
23+
["ZWJ sequence without grapheme segmentation", "ab👩\u200d💻tail", "ab👩\u200d..."],
24+
])("clips %s without corrupting UTF-16", (_name, content, expected) => {
25+
const output = formatWithLineNumbers(parseLines(content), 8)
26+
expect(output).toBe(`1 | ${expected}`)
27+
expect(Buffer.from(output).toString("utf8")).toBe(output)
28+
expect(output.slice(4).length).toBeLessThanOrEqual(8)
29+
})
30+
31+
it.each(["abc", "abcdefgh", "🔥🔥🔥🔥", "e\u0301👩\u200d💻"])(
32+
"preserves text at or below the limit: %s",
33+
(content) => {
34+
const output = formatWithLineNumbers(parseLines(content), 8)
35+
expect(output).toBe(`1 | ${content}`)
36+
expect(Buffer.from(output).toString("utf8")).toBe(output)
37+
},
38+
)
39+
40+
it.each([0, 1, 2, 3])("preserves the existing ellipsis behavior for a limit of %i", (limit) => {
41+
expect(formatWithLineNumbers(parseLines("abcdef"), limit)).toBe("1 | ...")
42+
})
43+
44+
it.each([
45+
[4, "..."],
46+
[5, "🔥..."],
47+
])("clips an emoji at the start with a limit of %i", (limit, expected) => {
48+
const output = formatWithLineNumbers(parseLines("🔥tail"), limit)
49+
expect(output).toBe(`1 | ${expected}`)
50+
expect(Buffer.from(output).toString("utf8")).toBe(output)
51+
expect(output.slice(4).length).toBeLessThanOrEqual(limit)
52+
})
53+
54+
it("preserves ASCII clipping and aligned line numbers alongside Unicode", () => {
55+
const lines = parseLines(`abcdefghij\n${"middle\n".repeat(8)}abcd🔥tail`)
56+
expect(formatWithLineNumbers([lines[0], lines[9]], 8)).toBe(" 1 | abcde...\n10 | abcd...")
57+
})
58+
59+
it("preserves empty formatting and the existing zero line-number fallback", () => {
60+
expect(formatWithLineNumbers([])).toBe("")
61+
expect(formatWithLineNumbers([{ ...parseLines("🔥")[0], lineNumber: 0 }])).toBe("0 | 🔥")
62+
})
63+
})
64+
65+
describe("Unicode clipping through the existing readers", () => {
66+
const longLine = "🔥".repeat(2000)
67+
const clippedLine = `${"🔥".repeat(998)}...`
68+
69+
it("preserves slice selection, ranges and truncation metadata", () => {
70+
const result = readWithSlice(`before\n${longLine}\nafter`, 1, 1)
71+
expect(result).toEqual({
72+
content: `2 | ${clippedLine}`,
73+
includedRanges: [[2, 2]],
74+
totalLines: 3,
75+
returnedLines: 1,
76+
wasTruncated: true,
77+
})
78+
expect(Buffer.from(result.content).toString("utf8")).toBe(result.content)
79+
})
80+
81+
it("preserves indentation selection and metadata with a single-line limit", () => {
82+
const result = readWithIndentation(`function example() {\n ${longLine}\n}`, {
83+
anchorLine: 2,
84+
limit: 1,
85+
})
86+
expect(result).toEqual({
87+
content: `2 | ${"🔥".repeat(996)}...`,
88+
includedRanges: [[2, 2]],
89+
totalLines: 3,
90+
returnedLines: 1,
91+
wasTruncated: true,
92+
})
93+
expect(Buffer.from(result.content).toString("utf8")).toBe(result.content)
94+
})
95+
96+
it("preserves indentation expansion, blank trimming and complete-file metadata", () => {
97+
const result = readWithIndentation(`\nfunction example() {\n ${longLine}\n}\n`, {
98+
anchorLine: 3,
99+
includeSiblings: true,
100+
})
101+
expect(result).toEqual({
102+
content: `2 | function example() {\n3 | ${"🔥".repeat(996)}...\n4 | }`,
103+
includedRanges: [[2, 4]],
104+
totalLines: 5,
105+
returnedLines: 3,
106+
wasTruncated: false,
107+
})
108+
expect(Buffer.from(result.content).toString("utf8")).toBe(result.content)
109+
})
110+
111+
it("preserves default slice limits and empty input", () => {
112+
expect(readWithSlice(longLine)).toEqual({
113+
content: `1 | ${clippedLine}`,
114+
includedRanges: [[1, 1]],
115+
totalLines: 1,
116+
returnedLines: 1,
117+
wasTruncated: false,
118+
})
119+
expect(readWithSlice("")).toEqual({
120+
content: "1 | ",
121+
includedRanges: [[1, 1]],
122+
totalLines: 1,
123+
returnedLines: 1,
124+
wasTruncated: false,
125+
})
126+
})
127+
128+
it("preserves reader errors without formatting or leaking the long line", () => {
129+
expect(readWithSlice(longLine, 1)).toEqual({
130+
content: "Error: offset 1 is beyond file end (1 lines)",
131+
includedRanges: [],
132+
totalLines: 1,
133+
returnedLines: 0,
134+
wasTruncated: false,
135+
})
136+
expect(readWithIndentation(longLine, { anchorLine: 0 })).toEqual({
137+
content: "Error: anchor_line 0 is out of range (1-1)",
138+
includedRanges: [],
139+
totalLines: 1,
140+
returnedLines: 0,
141+
wasTruncated: false,
142+
})
143+
})
144+
})

‎src/integrations/misc/indentation-reader.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,11 @@ export function formatWithLineNumbers(lines: LineRecord[], maxLineLength: number
238238

239239
// Truncate long lines
240240
if (content.length > maxLineLength) {
241-
content = content.substring(0, maxLineLength - 3) + "..."
241+
let end = maxLineLength - 3
242+
// Never split a UTF-16 surrogate pair before encoding the model result.
243+
const last = content.charCodeAt(end - 1)
244+
if (last >= 0xd800 && last <= 0xdbff) end--
245+
content = content.substring(0, end) + "..."
242246
}
243247

244248
return `${lineNum} | ${content}`

0 commit comments

Comments
 (0)