@@ -8,6 +8,7 @@ const MAX_BUFFERED_JSONL_BYTES = 128 * 1024 * 1024;
88
99type JSONLMarkerProcessingOptions = {
1010 bufferedEncoding ?: BufferEncoding ;
11+ callbackMode ?: 'async' | 'sync' ;
1112 markerIndex ?: 'byte' | 'decoded' ;
1213 scanMode ?: 'marker' | 'line' ;
1314} ;
@@ -54,6 +55,16 @@ function normalizeMarker(value: string): string | null {
5455 return hasNonWhitespace ( value ) ? value : null ;
5556}
5657
58+ function markerIndexForLine (
59+ line : string ,
60+ markerIndex : number ,
61+ options : JSONLMarkerProcessingOptions ,
62+ ) : number {
63+ return options . markerIndex === 'byte'
64+ ? Buffer . byteLength ( line . slice ( 0 , markerIndex ) , options . bufferedEncoding ?? 'utf8' )
65+ : markerIndex ;
66+ }
67+
5768async function processBufferedJSONLMarkerBytes (
5869 bytes : Uint8Array ,
5970 markers : readonly string [ ] ,
@@ -68,6 +79,7 @@ async function processBufferedJSONLMarkerBytes(
6879 content . toString ( options . bufferedEncoding ?? 'utf8' ) ,
6980 markers ,
7081 processLine ,
82+ options ,
7183 ) ;
7284 return ;
7385 }
@@ -107,6 +119,21 @@ async function processBufferedJSONLMarkerBytes(
107119 }
108120
109121 const lineStarts = Array . from ( candidates . keys ( ) ) . sort ( ( a , b ) => a - b ) ;
122+ if ( options . callbackMode === 'sync' ) {
123+ for ( const lineStart of lineStarts ) {
124+ const candidate = candidates . get ( lineStart ) ! ;
125+ const lineEnd = candidate . lineEnd ;
126+ const decodeEnd = lineEnd > lineStart && content [ lineEnd - 1 ] === 13 ? lineEnd - 1 : lineEnd ;
127+ const line = content . toString ( options . bufferedEncoding ?? 'utf8' , lineStart , decodeEnd ) ;
128+ void processLine (
129+ line ,
130+ options . markerIndex === 'byte' ? candidate . markerIndex : line . indexOf ( candidate . marker ) ,
131+ candidate . marker ,
132+ ) ;
133+ }
134+ return ;
135+ }
136+
110137 for ( const lineStart of lineStarts ) {
111138 const candidate = candidates . get ( lineStart ) ! ;
112139 const lineEnd = candidate . lineEnd ;
@@ -130,6 +157,35 @@ async function processBufferedJSONLSingleMarkerBytes(
130157) : Promise < void > {
131158 let lineStart = 0 ;
132159 let markerIndex = content . indexOf ( markerBuffer , lineStart ) ;
160+ if ( options . callbackMode === 'sync' ) {
161+ while ( markerIndex !== - 1 ) {
162+ while ( true ) {
163+ const nextLineEnd = content . indexOf ( 10 , lineStart ) ;
164+ if ( nextLineEnd === - 1 || nextLineEnd >= markerIndex ) {
165+ break ;
166+ }
167+ lineStart = nextLineEnd + 1 ;
168+ }
169+ let lineEnd = content . indexOf ( 10 , markerIndex ) ;
170+ if ( lineEnd === - 1 ) {
171+ lineEnd = content . length ;
172+ }
173+
174+ const decodeEnd = lineEnd > lineStart && content [ lineEnd - 1 ] === 13 ? lineEnd - 1 : lineEnd ;
175+ const line = content . toString ( options . bufferedEncoding ?? 'utf8' , lineStart , decodeEnd ) ;
176+ const lineMarkerIndex = markerIndex - lineStart ;
177+ void processLine (
178+ line ,
179+ options . markerIndex === 'byte' ? lineMarkerIndex : line . indexOf ( marker ) ,
180+ marker ,
181+ ) ;
182+
183+ lineStart = lineEnd + 1 ;
184+ markerIndex = content . indexOf ( markerBuffer , lineStart ) ;
185+ }
186+ return ;
187+ }
188+
133189 while ( markerIndex !== - 1 ) {
134190 while ( true ) {
135191 const nextLineEnd = content . indexOf ( 10 , lineStart ) ;
@@ -164,6 +220,7 @@ async function processBufferedJSONLMarkerContent(
164220 content : string ,
165221 markers : readonly string [ ] ,
166222 processLine : ( line : string , markerIndex : number , marker : string ) => void | Promise < void > ,
223+ options : JSONLMarkerProcessingOptions = { } ,
167224) : Promise < void > {
168225 let lineStart = 0 ;
169226 while ( lineStart < content . length ) {
@@ -187,7 +244,13 @@ async function processBufferedJSONLMarkerContent(
187244 }
188245 }
189246 if ( firstMarker != null ) {
190- const result = processLine ( line , firstMarkerIndex , firstMarker ) ;
247+ const callbackMarkerIndex = markerIndexForLine ( line , firstMarkerIndex , options ) ;
248+ if ( options . callbackMode === 'sync' ) {
249+ void processLine ( line , callbackMarkerIndex , firstMarker ) ;
250+ lineStart = lineEnd + 1 ;
251+ continue ;
252+ }
253+ const result = processLine ( line , callbackMarkerIndex , firstMarker ) ;
191254 if ( result != null ) {
192255 await result ;
193256 }
@@ -266,7 +329,7 @@ export async function processJSONLFileByMarkers(
266329
267330 const content = await readBufferedJSONLContent ( filePath ) ;
268331 if ( content != null ) {
269- await processBufferedJSONLMarkerContent ( content , normalizedMarkers , processLine ) ;
332+ await processBufferedJSONLMarkerContent ( content , normalizedMarkers , processLine , options ) ;
270333 return ;
271334 }
272335
@@ -289,7 +352,12 @@ export async function processJSONLFileByMarkers(
289352 if ( firstMarker == null ) {
290353 continue ;
291354 }
292- const result = processLine ( line , firstMarkerIndex , firstMarker ) ;
355+ const callbackMarkerIndex = markerIndexForLine ( line , firstMarkerIndex , options ) ;
356+ if ( options . callbackMode === 'sync' ) {
357+ void processLine ( line , callbackMarkerIndex , firstMarker ) ;
358+ continue ;
359+ }
360+ const result = processLine ( line , callbackMarkerIndex , firstMarker ) ;
293361 if ( result != null ) {
294362 await result ;
295363 }
@@ -417,5 +485,25 @@ if (import.meta.vitest != null) {
417485 '{"type":"event_msg","payload":{"type":"token_count"}}' ,
418486 ] ) ;
419487 } ) ;
488+
489+ it ( 'returns byte marker indexes in line-scan mode' , async ( ) => {
490+ const prefix = '{"note":"東京",' ;
491+ const marker = '"type":"token_count"' ;
492+ await using fixture = await createFixture ( {
493+ 'test.jsonl' : `${ prefix } ${ marker } }` ,
494+ } ) ;
495+
496+ const seen : number [ ] = [ ] ;
497+ await processJSONLFileByMarkers (
498+ fixture . getPath ( 'test.jsonl' ) ,
499+ [ marker ] ,
500+ ( _line , markerIndex ) => {
501+ seen . push ( markerIndex ) ;
502+ } ,
503+ { markerIndex : 'byte' , scanMode : 'line' } ,
504+ ) ;
505+
506+ expect ( seen ) . toEqual ( [ Buffer . byteLength ( prefix ) ] ) ;
507+ } ) ;
420508 } ) ;
421509}
0 commit comments