1- import { createHighlighterCore , type HighlighterCore } from "@shikijs/core" ;
1+ import { createHighlighterCore , type GrammarState , type HighlighterCore } from "@shikijs/core" ;
22import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript" ;
33import bashLanguage from "@shikijs/langs/bash" ;
44import diffLanguage from "@shikijs/langs/diff" ;
@@ -46,8 +46,12 @@ export interface NativeReviewDiffHighlighterHandle {
4646 readonly engine : NativeReviewDiffHighlightEngine ;
4747 readonly tokenize : (
4848 code : string ,
49- options : { readonly lang : NativeReviewDiffLanguage ; readonly theme : string } ,
50- ) => ReadonlyArray < ReadonlyArray < NativeReviewDiffToken > > ;
49+ options : {
50+ readonly lang : NativeReviewDiffLanguage ;
51+ readonly theme : string ;
52+ readonly signal ?: AbortSignal ;
53+ } ,
54+ ) => Promise < ReadonlyArray < ReadonlyArray < NativeReviewDiffToken > > > ;
5155}
5256
5357interface NativeReviewDiffLineRow extends NativeReviewDiffRow {
@@ -97,6 +101,8 @@ export interface HighlightNativeReviewDiffVisibleRowsInput {
97101const NATIVE_REVIEW_DIFF_HIGHLIGHT_CHUNK_SIZE = 500 ;
98102const NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS = 160 ;
99103const NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS = 360 ;
104+ const NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH = 1_000 ;
105+ const NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS = 8_000 ;
100106
101107const NATIVE_REVIEW_DIFF_THEME_NAME_BY_SCHEME = {
102108 dark : "t3-pierre-dark" ,
@@ -226,6 +232,63 @@ function normalizeTokens(
226232 ) ;
227233}
228234
235+ function createHighlighterHandle (
236+ highlighter : HighlighterCore ,
237+ engine : NativeReviewDiffHighlightEngine ,
238+ ) : NativeReviewDiffHighlighterHandle {
239+ return {
240+ engine,
241+ async tokenize ( code , { lang, theme, signal } ) {
242+ const lines = code . split ( "\n" ) ;
243+ const highlighted : Array < ReadonlyArray < NativeReviewDiffToken > > = [ ] ;
244+ let grammarState : GrammarState | undefined ;
245+ let start = 0 ;
246+
247+ while ( start < lines . length ) {
248+ if ( signal ?. aborted ) return [ ] ;
249+
250+ // Skipping this line leaves its ending grammar state unknown. Keep the
251+ // rest of this contiguous segment plain instead of guessing its syntax.
252+ if ( lines [ start ] ! . length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH ) {
253+ highlighted . push (
254+ ...lines
255+ . slice ( start )
256+ . map ( ( content ) => [ { content : content || " " , color : null , fontStyle : null } ] ) ,
257+ ) ;
258+ break ;
259+ }
260+
261+ let end = start ;
262+ let characters = 0 ;
263+ while ( end < lines . length ) {
264+ const length = lines [ end ] ! . length ;
265+ const nextCharacters = characters + length + ( end > start ? 1 : 0 ) ;
266+ if (
267+ length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_LINE_LENGTH ||
268+ nextCharacters > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS
269+ ) {
270+ break ;
271+ }
272+ characters = nextCharacters ;
273+ end += 1 ;
274+ }
275+
276+ const tokens = highlighter . codeToTokensBase ( lines . slice ( start , end ) . join ( "\n" ) , {
277+ lang,
278+ theme,
279+ grammarState,
280+ } ) ;
281+ grammarState = highlighter . getLastGrammarState ( tokens ) ;
282+ highlighted . push ( ...normalizeTokens ( tokens ) ) ;
283+ start = end ;
284+ if ( start < lines . length ) await waitForNextFrame ( ) ;
285+ }
286+
287+ return signal ?. aborted ? [ ] : highlighted ;
288+ } ,
289+ } ;
290+ }
291+
229292async function createNativeReviewDiffHighlighter ( ) : Promise < NativeReviewDiffHighlighterHandle > {
230293 const nativeEngineModule = await import ( "react-native-shiki-engine" ) ;
231294 if ( ! nativeEngineModule . isNativeEngineAvailable ( ) ) {
@@ -238,10 +301,7 @@ async function createNativeReviewDiffHighlighter(): Promise<NativeReviewDiffHigh
238301 engine : nativeEngineModule . createNativeEngine ( ) ,
239302 } ) ;
240303
241- return {
242- engine : "native" ,
243- tokenize : ( code , options ) => normalizeTokens ( highlighter . codeToTokensBase ( code , options ) ) ,
244- } ;
304+ return createHighlighterHandle ( highlighter , "native" ) ;
245305}
246306
247307async function createJavascriptReviewDiffHighlighter ( ) : Promise < NativeReviewDiffHighlighterHandle > {
@@ -251,10 +311,7 @@ async function createJavascriptReviewDiffHighlighter(): Promise<NativeReviewDiff
251311 engine : createJavaScriptRegexEngine ( ) ,
252312 } ) ;
253313
254- return {
255- engine : "javascript" ,
256- tokenize : ( code , options ) => normalizeTokens ( highlighter . codeToTokensBase ( code , options ) ) ,
257- } ;
314+ return createHighlighterHandle ( highlighter , "javascript" ) ;
258315}
259316
260317export async function getNativeReviewDiffHighlighter (
@@ -438,16 +495,30 @@ export async function highlightNativeReviewDiffVisibleRows(
438495 const tokensByRowId : Record < string , ReadonlyArray < NativeReviewDiffToken > > = { } ;
439496 let segmentRows : IndexedNativeReviewDiffLineRow [ ] = [ ] ;
440497 let segmentFile : NativeReviewDiffFile | undefined ;
498+ let charactersSinceYield = 0 ;
441499
442- const flushSegment = ( ) => {
500+ const flushSegment = async ( ) => {
443501 if ( ! segmentFile || segmentRows . length === 0 || input . signal ?. aborted ) {
444502 segmentRows = [ ] ;
445503 segmentFile = undefined ;
446504 return ;
447505 }
448506
449507 const code = segmentRows . map ( ( { row } ) => row . content ) . join ( "\n" ) ;
450- const tokenLines = highlighter . tokenize ( code , { lang : segmentFile . language , theme } ) ;
508+ if (
509+ charactersSinceYield > 0 &&
510+ charactersSinceYield + code . length > NATIVE_REVIEW_DIFF_TOKENIZE_MAX_CHARACTERS
511+ ) {
512+ await waitForNextFrame ( ) ;
513+ charactersSinceYield = 0 ;
514+ if ( input . signal ?. aborted ) return ;
515+ }
516+ const tokenLines = await highlighter . tokenize ( code , {
517+ lang : segmentFile . language ,
518+ theme,
519+ signal : input . signal ,
520+ } ) ;
521+ charactersSinceYield += code . length ;
451522 segmentRows . forEach ( ( { row } , rowIndex ) => {
452523 tokensByRowId [ row . id ] = tokenLines [ rowIndex ] ?? makePlainTokenFallback ( row ) ;
453524 } ) ;
@@ -456,6 +527,7 @@ export async function highlightNativeReviewDiffVisibleRows(
456527 } ;
457528
458529 for ( const selectedRow of selectedRows ) {
530+ if ( input . signal ?. aborted ) break ;
459531 const { row } = selectedRow ;
460532 const file = fileMap . get ( row . fileId ) ;
461533 if ( ! file ) {
@@ -469,13 +541,17 @@ export async function highlightNativeReviewDiffVisibleRows(
469541 ( previousRow !== undefined &&
470542 ! canShareGrammarContext ( previousRow , selectedRow , input . rows ) ) )
471543 ) {
472- flushSegment ( ) ;
544+ await flushSegment ( ) ;
473545 }
474546
475547 segmentFile = file ;
476548 segmentRows . push ( selectedRow ) ;
477549 }
478- flushSegment ( ) ;
550+ await flushSegment ( ) ;
551+
552+ if ( input . signal ?. aborted ) {
553+ return { engine : highlighter . engine , tokensByRowId : { } , rowCount : 0 , durationMs : 0 } ;
554+ }
479555
480556 return {
481557 engine : highlighter . engine ,
@@ -504,7 +580,12 @@ export async function streamNativeReviewDiffTokens(
504580 const startedAt = performance . now ( ) ;
505581 const chunkRows = fileRows . slice ( startIndex , startIndex + chunkSize ) ;
506582 const code = chunkRows . map ( ( row ) => row . content ) . join ( "\n" ) ;
507- const tokenLines = highlighter . tokenize ( code , { lang : file . language , theme } ) ;
583+ const tokenLines = await highlighter . tokenize ( code , {
584+ lang : file . language ,
585+ theme,
586+ signal : input . signal ,
587+ } ) ;
588+ if ( input . signal ?. aborted ) return highlighter . engine ;
508589 const tokensByRowId : Record < string , ReadonlyArray < NativeReviewDiffToken > > = { } ;
509590
510591 chunkRows . forEach ( ( row , rowIndex ) => {
0 commit comments