Skip to content

Commit 9ec50ea

Browse files
committed
perf(mobile): bound diff syntax highlighting work
1 parent 082cab2 commit 9ec50ea

2 files changed

Lines changed: 248 additions & 18 deletions

File tree

‎apps/mobile/src/features/diffs/nativeReviewDiffHighlighter.test.ts‎

Lines changed: 151 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,47 @@
1-
import { describe, expect, it } from "vite-plus/test";
1+
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
22

33
import type { NativeReviewDiffRow } from "./nativeReviewDiffSurface";
44
import type { NativeReviewDiffFile } from "./nativeReviewDiffTypes";
5-
import { highlightNativeReviewDiffVisibleRows } from "./nativeReviewDiffHighlighter";
5+
import {
6+
highlightNativeReviewDiffVisibleRows,
7+
streamNativeReviewDiffTokens,
8+
type NativeReviewDiffTokenChunk,
9+
} from "./nativeReviewDiffHighlighter";
10+
11+
const tokenization = vi.hoisted(() => ({
12+
calls: [] as string[],
13+
afterCall: undefined as (() => void) | undefined,
14+
}));
15+
16+
vi.mock("@shikijs/core", async (importOriginal) => {
17+
const original = await importOriginal<typeof import("@shikijs/core")>();
18+
return {
19+
...original,
20+
createHighlighterCore: async (...args: Parameters<typeof original.createHighlighterCore>) => {
21+
const highlighter = await original.createHighlighterCore(...args);
22+
return {
23+
...highlighter,
24+
codeToTokensBase: (...input: Parameters<typeof highlighter.codeToTokensBase>) => {
25+
tokenization.calls.push(input[0]);
26+
const result = highlighter.codeToTokensBase(...input);
27+
tokenization.afterCall?.();
28+
return result;
29+
},
30+
};
31+
},
32+
};
33+
});
34+
35+
// Exercise the native entry path without requiring an iOS or Android runtime.
36+
vi.mock("react-native-shiki-engine", async () => {
37+
const { createJavaScriptRegexEngine } = await import("@shikijs/engine-javascript");
38+
return { isNativeEngineAvailable: () => true, createNativeEngine: createJavaScriptRegexEngine };
39+
});
40+
41+
afterEach(() => {
42+
tokenization.calls = [];
43+
tokenization.afterCall = undefined;
44+
});
645

746
const TYPESCRIPT_FILE: NativeReviewDiffFile = {
847
id: "file-1",
@@ -187,3 +226,113 @@ describe("highlightNativeReviewDiffVisibleRows", () => {
187226
);
188227
});
189228
});
229+
230+
describe.each(["native", "javascript"] as const)("%s highlighting budgets", (engine) => {
231+
const highlightRows = (rows: ReadonlyArray<NativeReviewDiffRow>, signal?: AbortSignal) =>
232+
highlightNativeReviewDiffVisibleRows({
233+
rows,
234+
files: [TYPESCRIPT_FILE],
235+
scheme: "dark",
236+
engine,
237+
firstRowIndex: 0,
238+
lastRowIndex: rows.length - 1,
239+
overscanRows: 0,
240+
signal,
241+
});
242+
243+
const line = (id: number, content: string) =>
244+
makeLine({
245+
id: `line-${id}`,
246+
content,
247+
change: "add",
248+
oldLineNumber: null,
249+
newLineNumber: id,
250+
});
251+
252+
it("still highlights a line at the length limit", async () => {
253+
const content = `// ${"x".repeat(997)}`;
254+
const result = await highlightRows([line(1, content)]);
255+
256+
expect(tokenization.calls).toEqual([content]);
257+
expect(result.tokensByRowId["line-1"]?.some((token) => token.color !== null)).toBe(true);
258+
});
259+
260+
it("keeps long lines and unknown following syntax plain until the next hunk", async () => {
261+
const longLine = `${"x".repeat(1_001)} /*`;
262+
const rows = [
263+
line(1, "export const before = 1;"),
264+
line(2, longLine),
265+
{ kind: "comment", id: "note", commentText: "Check this", fileId: TYPESCRIPT_FILE.id },
266+
line(3, "inside the comment */"),
267+
makeHunk("next-hunk"),
268+
line(100, "export const after = 2;"),
269+
] satisfies ReadonlyArray<NativeReviewDiffRow>;
270+
const result = await highlightRows(rows);
271+
272+
expect(result.engine).toBe(engine);
273+
expect(Object.keys(result.tokensByRowId)).toEqual(["line-1", "line-2", "line-3", "line-100"]);
274+
expect(result.tokensByRowId["line-2"]).toEqual([
275+
{ content: longLine, color: null, fontStyle: null },
276+
]);
277+
expect(result.tokensByRowId["line-3"]).toEqual([
278+
{ content: "inside the comment */", color: null, fontStyle: null },
279+
]);
280+
expect(result.tokensByRowId["line-1"]?.some((token) => token.color !== null)).toBe(true);
281+
expect(result.tokensByRowId["line-100"]?.some((token) => token.color !== null)).toBe(true);
282+
expect(tokenization.calls.some((code) => code.includes(longLine))).toBe(false);
283+
});
284+
285+
it("preserves multiline grammar and row mapping across character-limited batches", async () => {
286+
const opening = line(1, "const message = `open");
287+
const body = Array.from({ length: 40 }, (_, index) => line(index + 2, "inside ".repeat(45)));
288+
const closing = line(42, "closed`; ");
289+
const trailing = line(43, "export const after = 2;");
290+
const result = await highlightRows([opening, ...body, closing, trailing]);
291+
const calls = [...tokenization.calls];
292+
const expected = await highlightRows([
293+
opening,
294+
{ ...closing, newLineNumber: 2 },
295+
{ ...trailing, newLineNumber: 3 },
296+
]);
297+
298+
expect(calls.length).toBeGreaterThan(1);
299+
expect(calls.every((code) => code.length <= 8_000)).toBe(true);
300+
expect(result.rowCount).toBe(43);
301+
for (const row of [opening, ...body, closing, trailing]) {
302+
expect(result.tokensByRowId[row.id]?.map((token) => token.content).join("")).toBe(
303+
row.content,
304+
);
305+
}
306+
expect(result.tokensByRowId[closing.id]).toEqual(expected.tokensByRowId[closing.id]);
307+
expect(result.tokensByRowId[trailing.id]).toEqual(expected.tokensByRowId[trailing.id]);
308+
});
309+
310+
it("stops before later batches and publishes no partial result after cancellation", async () => {
311+
const controller = new AbortController();
312+
tokenization.afterCall = () => controller.abort();
313+
const rows = Array.from({ length: 30 }, (_, index) => line(index + 1, `// ${"x".repeat(400)}`));
314+
315+
const result = await highlightRows(rows, controller.signal);
316+
317+
expect(tokenization.calls).toHaveLength(1);
318+
expect(result.rowCount).toBe(0);
319+
expect(result.tokensByRowId).toEqual({});
320+
});
321+
322+
it("applies the same long-line guard to streamed token chunks", async () => {
323+
const content = "x".repeat(10_000);
324+
const chunks: NativeReviewDiffTokenChunk[] = [];
325+
326+
await streamNativeReviewDiffTokens({
327+
rows: [line(1, content)],
328+
files: [TYPESCRIPT_FILE],
329+
scheme: "dark",
330+
engine,
331+
onChunk: (chunk) => chunks.push(chunk),
332+
});
333+
334+
expect(chunks).toHaveLength(1);
335+
expect(chunks[0]?.tokensByRowId["line-1"]).toEqual([{ content, color: null, fontStyle: null }]);
336+
expect(tokenization.calls).toHaveLength(0);
337+
});
338+
});

‎apps/mobile/src/features/diffs/nativeReviewDiffHighlighter.ts‎

Lines changed: 97 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { createHighlighterCore, type HighlighterCore } from "@shikijs/core";
1+
import { createHighlighterCore, type GrammarState, type HighlighterCore } from "@shikijs/core";
22
import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript";
33
import bashLanguage from "@shikijs/langs/bash";
44
import diffLanguage from "@shikijs/langs/diff";
@@ -46,8 +46,12 @@ export interface NativeReviewDiffHighlighterHandle {
4646
readonly engine: NativeReviewDiffHighlightEngine;
4747
readonly tokenize: (
4848
code: string,
49-
options: { readonly lang: NativeReviewDiffLanguage; readonly theme: string },
50-
) => ReadonlyArray<ReadonlyArray<NativeReviewDiffToken>>;
49+
options: {
50+
readonly lang: NativeReviewDiffLanguage;
51+
readonly theme: string;
52+
readonly signal?: AbortSignal;
53+
},
54+
) => Promise<ReadonlyArray<ReadonlyArray<NativeReviewDiffToken>>>;
5155
}
5256

5357
interface NativeReviewDiffLineRow extends NativeReviewDiffRow {
@@ -97,6 +101,8 @@ export interface HighlightNativeReviewDiffVisibleRowsInput {
97101
const NATIVE_REVIEW_DIFF_HIGHLIGHT_CHUNK_SIZE = 500;
98102
const NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS = 160;
99103
const NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS = 360;
104+
const NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH = 1_000;
105+
const NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS = 8_000;
100106

101107
const NATIVE_REVIEW_DIFF_THEME_NAME_BY_SCHEME = {
102108
dark: "t3-pierre-dark",
@@ -226,6 +232,63 @@ function normalizeTokens(
226232
);
227233
}
228234

235+
function createHighlighterHandle(
236+
highlighter: HighlighterCore,
237+
engine: NativeReviewDiffHighlightEngine,
238+
): NativeReviewDiffHighlighterHandle {
239+
return {
240+
engine,
241+
async tokenize(code, { lang, theme, signal }) {
242+
const lines = code.split("\n");
243+
const highlighted: Array<ReadonlyArray<NativeReviewDiffToken>> = [];
244+
let grammarState: GrammarState | undefined;
245+
let start = 0;
246+
247+
while (start < lines.length) {
248+
if (signal?.aborted) return [];
249+
250+
// Skipping this line leaves its ending grammar state unknown. Keep the
251+
// rest of this contiguous segment plain instead of guessing its syntax.
252+
if (lines[start]!.length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH) {
253+
highlighted.push(
254+
...lines
255+
.slice(start)
256+
.map((content) => [{ content: content || " ", color: null, fontStyle: null }]),
257+
);
258+
break;
259+
}
260+
261+
let end = start;
262+
let characters = 0;
263+
while (end < lines.length) {
264+
const length = lines[end]!.length;
265+
const nextCharacters = characters + length + (end > start ? 1 : 0);
266+
if (
267+
length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH ||
268+
nextCharacters > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS
269+
) {
270+
break;
271+
}
272+
characters = nextCharacters;
273+
end += 1;
274+
}
275+
276+
const tokens = highlighter.codeToTokensBase(lines.slice(start, end).join("\n"), {
277+
lang,
278+
theme,
279+
grammarState,
280+
});
281+
grammarState = highlighter.getLastGrammarState(tokens);
282+
highlighted.push(...normalizeTokens(tokens));
283+
start = end;
284+
if (start < lines.length) await waitForNextFrame();
285+
}
286+
287+
return signal?.aborted ? [] : highlighted;
288+
},
289+
};
290+
}
291+
229292
async function createNativeReviewDiffHighlighter(): Promise<NativeReviewDiffHighlighterHandle> {
230293
const nativeEngineModule = await import("react-native-shiki-engine");
231294
if (!nativeEngineModule.isNativeEngineAvailable()) {
@@ -238,10 +301,7 @@ async function createNativeReviewDiffHighlighter(): Promise<NativeReviewDiffHigh
238301
engine: nativeEngineModule.createNativeEngine(),
239302
});
240303

241-
return {
242-
engine: "native",
243-
tokenize: (code, options) => normalizeTokens(highlighter.codeToTokensBase(code, options)),
244-
};
304+
return createHighlighterHandle(highlighter, "native");
245305
}
246306

247307
async function createJavascriptReviewDiffHighlighter(): Promise<NativeReviewDiffHighlighterHandle> {
@@ -251,10 +311,7 @@ async function createJavascriptReviewDiffHighlighter(): Promise<NativeReviewDiff
251311
engine: createJavaScriptRegexEngine(),
252312
});
253313

254-
return {
255-
engine: "javascript",
256-
tokenize: (code, options) => normalizeTokens(highlighter.codeToTokensBase(code, options)),
257-
};
314+
return createHighlighterHandle(highlighter, "javascript");
258315
}
259316

260317
export async function getNativeReviewDiffHighlighter(
@@ -438,16 +495,30 @@ export async function highlightNativeReviewDiffVisibleRows(
438495
const tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
439496
let segmentRows: IndexedNativeReviewDiffLineRow[] = [];
440497
let segmentFile: NativeReviewDiffFile | undefined;
498+
let charactersSinceYield = 0;
441499

442-
const flushSegment = () => {
500+
const flushSegment = async () => {
443501
if (!segmentFile || segmentRows.length === 0 || input.signal?.aborted) {
444502
segmentRows = [];
445503
segmentFile = undefined;
446504
return;
447505
}
448506

449507
const code = segmentRows.map(({ row }) => row.content).join("\n");
450-
const tokenLines = highlighter.tokenize(code, { lang: segmentFile.language, theme });
508+
if (
509+
charactersSinceYield > 0 &&
510+
charactersSinceYield + code.length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS
511+
) {
512+
await waitForNextFrame();
513+
charactersSinceYield = 0;
514+
if (input.signal?.aborted) return;
515+
}
516+
const tokenLines = await highlighter.tokenize(code, {
517+
lang: segmentFile.language,
518+
theme,
519+
signal: input.signal,
520+
});
521+
charactersSinceYield += code.length;
451522
segmentRows.forEach(({ row }, rowIndex) => {
452523
tokensByRowId[row.id] = tokenLines[rowIndex] ?? makePlainTokenFallback(row);
453524
});
@@ -456,6 +527,7 @@ export async function highlightNativeReviewDiffVisibleRows(
456527
};
457528

458529
for (const selectedRow of selectedRows) {
530+
if (input.signal?.aborted) break;
459531
const { row } = selectedRow;
460532
const file = fileMap.get(row.fileId);
461533
if (!file) {
@@ -469,13 +541,17 @@ export async function highlightNativeReviewDiffVisibleRows(
469541
(previousRow !== undefined &&
470542
!canShareGrammarContext(previousRow, selectedRow, input.rows)))
471543
) {
472-
flushSegment();
544+
await flushSegment();
473545
}
474546

475547
segmentFile = file;
476548
segmentRows.push(selectedRow);
477549
}
478-
flushSegment();
550+
await flushSegment();
551+
552+
if (input.signal?.aborted) {
553+
return { engine: highlighter.engine, tokensByRowId: {}, rowCount: 0, durationMs: 0 };
554+
}
479555

480556
return {
481557
engine: highlighter.engine,
@@ -504,7 +580,12 @@ export async function streamNativeReviewDiffTokens(
504580
const startedAt = performance.now();
505581
const chunkRows = fileRows.slice(startIndex, startIndex + chunkSize);
506582
const code = chunkRows.map((row) => row.content).join("\n");
507-
const tokenLines = highlighter.tokenize(code, { lang: file.language, theme });
583+
const tokenLines = await highlighter.tokenize(code, {
584+
lang: file.language,
585+
theme,
586+
signal: input.signal,
587+
});
588+
if (input.signal?.aborted) return highlighter.engine;
508589
const tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
509590

510591
chunkRows.forEach((row, rowIndex) => {

0 commit comments

Comments
 (0)