Skip to content

Commit 5c5acef

Browse files
authored
fix CJK extmarks sim offsets (#308)
Should fix OC anomalyco/opencode#3734
1 parent c3a4680 commit 5c5acef

3 files changed

Lines changed: 432 additions & 12 deletions

File tree

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
import { describe, expect, it, afterEach } from "bun:test"
2+
import { TextareaRenderable } from "../renderables/Textarea"
3+
import { createTestRenderer, type TestRenderer, type MockInput } from "../testing/test-renderer"
4+
import { type ExtmarksController } from "./extmarks"
5+
import { SyntaxStyle } from "../syntax-style"
6+
import { RGBA } from "./RGBA"
7+
8+
let currentRenderer: TestRenderer
9+
let renderOnce: () => Promise<void>
10+
let currentMockInput: MockInput
11+
let textarea: TextareaRenderable
12+
let extmarks: ExtmarksController
13+
14+
async function setup(initialValue: string = "Hello World") {
15+
const result = await createTestRenderer({ width: 80, height: 24 })
16+
currentRenderer = result.renderer
17+
renderOnce = result.renderOnce
18+
currentMockInput = result.mockInput
19+
20+
textarea = new TextareaRenderable(currentRenderer, {
21+
left: 0,
22+
top: 0,
23+
width: 40,
24+
height: 10,
25+
initialValue,
26+
})
27+
28+
currentRenderer.root.add(textarea)
29+
await renderOnce()
30+
31+
extmarks = textarea.extmarks
32+
33+
return { textarea, extmarks }
34+
}
35+
36+
describe("ExtmarksController - Multi-width Graphemes", () => {
37+
afterEach(() => {
38+
if (extmarks) extmarks.destroy()
39+
if (currentRenderer) currentRenderer.destroy()
40+
})
41+
42+
describe("Basic Multi-width Highlighting", () => {
43+
it("should correctly highlight text AFTER multi-width characters", async () => {
44+
// Text: "前后端分离 @git-committer"
45+
// Chinese chars are multi-width, @ onwards should highlight correctly
46+
await setup("前后端分离 @git-committer")
47+
48+
const style = SyntaxStyle.create()
49+
const styleId = style.registerStyle("mention", {
50+
fg: RGBA.fromValues(0, 0, 1, 1),
51+
bg: RGBA.fromValues(0.9, 0.9, 1, 1),
52+
})
53+
54+
textarea.syntaxStyle = style
55+
56+
const text = textarea.plainText
57+
58+
// Calculate CORRECT display-width offsets
59+
// "前" = 2 cols, "后" = 2 cols, "端" = 2 cols, "分" = 2 cols, "离" = 2 cols, " " = 1 col
60+
// Total before "@": 10 + 1 = 11 display-width columns
61+
let displayOffset = 0
62+
const atJsIndex = text.indexOf("@")
63+
for (let i = 0; i < atJsIndex; i++) {
64+
if (text[i] === "\n") {
65+
displayOffset += 1
66+
} else {
67+
displayOffset += Bun.stringWidth(text[i])
68+
}
69+
}
70+
71+
const mentionText = "@git-committer"
72+
const mentionDisplayWidth = Bun.stringWidth(mentionText)
73+
const mentionStart = displayOffset // Should be 11
74+
const mentionEnd = displayOffset + mentionDisplayWidth // Should be 25
75+
76+
extmarks.create({
77+
start: mentionStart,
78+
end: mentionEnd,
79+
styleId,
80+
})
81+
82+
const highlights = textarea.getLineHighlights(0)
83+
expect(highlights.length).toBe(1)
84+
expect(highlights[0].start).toBe(11)
85+
expect(highlights[0].end).toBe(25)
86+
})
87+
88+
it("should correctly highlight text BEFORE multi-width characters", async () => {
89+
await setup("hello 前后端分离")
90+
91+
const style = SyntaxStyle.create()
92+
const styleId = style.registerStyle("test", {
93+
fg: RGBA.fromValues(1, 0, 0, 1),
94+
})
95+
96+
textarea.syntaxStyle = style
97+
98+
// Highlight "hello" which is at offsets 0-5
99+
extmarks.create({
100+
start: 0,
101+
end: 5,
102+
styleId,
103+
})
104+
105+
const highlights = textarea.getLineHighlights(0)
106+
107+
expect(highlights.length).toBe(1)
108+
expect(highlights[0].start).toBe(0)
109+
expect(highlights[0].end).toBe(5)
110+
})
111+
112+
it("should correctly highlight BETWEEN multi-width characters", async () => {
113+
await setup("前后 test 端分离")
114+
115+
const style = SyntaxStyle.create()
116+
const styleId = style.registerStyle("test", {
117+
fg: RGBA.fromValues(1, 0, 0, 1),
118+
})
119+
120+
textarea.syntaxStyle = style
121+
122+
// "前后 test 端分离"
123+
// Offsets: 前=0, 后=1, space=2, t=3, e=4, s=5, t=6, space=7, 端=8, 分=9, 离=10
124+
const testStart = 3
125+
const testEnd = 7
126+
127+
128+
extmarks.create({
129+
start: testStart,
130+
end: testEnd,
131+
styleId,
132+
})
133+
134+
const highlights = textarea.getLineHighlights(0)
135+
136+
if (highlights.length > 0) {
137+
const lineText = textarea.plainText.split("\n")[0]
138+
const actualHighlightedText = lineText.substring(highlights[0].start, highlights[0].end)
139+
expect(actualHighlightedText).toBe("test")
140+
}
141+
142+
expect(highlights.length).toBe(1)
143+
})
144+
145+
it("should correctly highlight the multi-width characters themselves", async () => {
146+
await setup("hello 前后端分离 world")
147+
148+
const style = SyntaxStyle.create()
149+
const styleId = style.registerStyle("test", {
150+
fg: RGBA.fromValues(1, 0, 0, 1),
151+
})
152+
153+
textarea.syntaxStyle = style
154+
155+
// "hello 前后端分离 world"
156+
// Offsets: h=0,e=1,l=2,l=3,o=4,space=5,前=6,后=7,端=8,分=9,离=10,space=11,w=12...
157+
const chineseStart = 6
158+
const chineseEnd = 11
159+
160+
161+
extmarks.create({
162+
start: chineseStart,
163+
end: chineseEnd,
164+
styleId,
165+
})
166+
167+
const highlights = textarea.getLineHighlights(0)
168+
169+
if (highlights.length > 0) {
170+
const lineText = textarea.plainText.split("\n")[0]
171+
const actualHighlightedText = lineText.substring(highlights[0].start, highlights[0].end)
172+
expect(actualHighlightedText).toBe("前后端分离")
173+
}
174+
175+
expect(highlights.length).toBe(1)
176+
})
177+
})
178+
179+
describe("Complex Multi-width Scenarios", () => {
180+
it("should handle emoji and multi-width characters together", async () => {
181+
await setup("前后 🌟 test")
182+
183+
const style = SyntaxStyle.create()
184+
const styleId = style.registerStyle("test", {
185+
fg: RGBA.fromValues(1, 0, 0, 1),
186+
})
187+
188+
textarea.syntaxStyle = style
189+
190+
// Highlight "test" at the end
191+
const text = textarea.plainText
192+
const testPos = text.indexOf("test")
193+
194+
195+
extmarks.create({
196+
start: testPos,
197+
end: testPos + 4,
198+
styleId,
199+
})
200+
201+
const highlights = textarea.getLineHighlights(0)
202+
203+
expect(highlights.length).toBe(1)
204+
205+
const lineText = textarea.plainText.split("\n")[0]
206+
const actualHighlightedText = lineText.substring(highlights[0].start, highlights[0].end)
207+
expect(actualHighlightedText).toBe("test")
208+
})
209+
210+
it("should handle multiple highlights with multi-width characters", async () => {
211+
await setup("前后端 @user1 分离 @user2 end")
212+
213+
const style = SyntaxStyle.create()
214+
const styleId = style.registerStyle("mention", {
215+
fg: RGBA.fromValues(0, 0, 1, 1),
216+
})
217+
218+
textarea.syntaxStyle = style
219+
220+
const text = textarea.plainText
221+
222+
const user1Start = text.indexOf("@user1")
223+
const user1End = user1Start + 6
224+
const user2Start = text.indexOf("@user2")
225+
const user2End = user2Start + 6
226+
227+
228+
extmarks.create({
229+
start: user1Start,
230+
end: user1End,
231+
styleId,
232+
})
233+
234+
extmarks.create({
235+
start: user2Start,
236+
end: user2End,
237+
styleId,
238+
})
239+
240+
const highlights = textarea.getLineHighlights(0)
241+
242+
highlights.forEach((h, i) => {
243+
const lineText = textarea.plainText.split("\n")[0]
244+
const highlightedText = lineText.substring(h.start, h.end)
245+
})
246+
247+
expect(highlights.length).toBe(2)
248+
})
249+
})
250+
251+
describe("Cursor Movement with Multi-width Characters", () => {
252+
it("should correctly position cursor after multi-width characters", async () => {
253+
await setup("前后 test")
254+
255+
textarea.focus()
256+
textarea.cursorOffset = 0
257+
258+
// Text: "前后 test"
259+
// "前" = display width 2, "后" = display width 2, " " = display width 1
260+
// After 3 arrow right presses from position 0:
261+
// Press 1: move to display-width 2 (after "前")
262+
// Press 2: move to display-width 4 (after "后")
263+
// Press 3: move to display-width 5 (after " ")
264+
265+
for (let i = 0; i < 3; i++) {
266+
currentMockInput.pressArrow("right")
267+
}
268+
269+
const cursorPos = textarea.cursorOffset
270+
271+
// Cursor should be at display-width offset 5 (after "前后 ")
272+
expect(cursorPos).toBe(5)
273+
})
274+
})
275+
276+
describe("Visual vs Byte Offset Issues", () => {
277+
it("should demonstrate the offset to char offset conversion issue", async () => {
278+
// This is the CRITICAL test - offsetToCharOffset doesn't account for display width
279+
await setup("前后端分离 @git-committer")
280+
281+
const style = SyntaxStyle.create()
282+
const styleId = style.registerStyle("mention", {
283+
fg: RGBA.fromValues(0, 0, 1, 1),
284+
bg: RGBA.fromValues(0.9, 0.9, 1, 1),
285+
})
286+
287+
textarea.syntaxStyle = style
288+
289+
const text = textarea.plainText
290+
291+
// The @ symbol is at cursor offset 6
292+
const atPos = text.indexOf("@")
293+
294+
// We want to highlight from @ to the end of "committer"
295+
const start = atPos
296+
const end = atPos + 14 // "@git-committer" is 14 chars
297+
298+
299+
const extmarkId = extmarks.create({
300+
start: start,
301+
end: end,
302+
styleId,
303+
})
304+
305+
const extmark = extmarks.get(extmarkId)
306+
307+
const highlights = textarea.getLineHighlights(0)
308+
309+
if (highlights.length > 0) {
310+
const h = highlights[0]
311+
312+
// This is where the bug manifests:
313+
// The offsetToCharOffset in extmarks.ts doesn't account for multi-width display
314+
// So the highlight char offset will be wrong
315+
316+
const lineText = text.split("\n")[0]
317+
318+
// Try to extract what's actually highlighted using the char offsets
319+
const actualText = lineText.substring(h.start, Math.min(h.end, lineText.length))
320+
321+
// This will likely FAIL because the char offsets don't account for display width
322+
}
323+
324+
expect(highlights.length).toBe(1)
325+
})
326+
})
327+
})

0 commit comments

Comments
 (0)