@@ -556,140 +556,6 @@ async function processJSONLFileByLine(
556556 }
557557}
558558
559- if ( import . meta. vitest != null ) {
560- describe ( 'processJSONLFileByLine' , ( ) => {
561- it ( 'should process each non-empty line with correct line numbers' , async ( ) => {
562- await using fixture = await createFixture ( {
563- 'test.jsonl' : '{"line": 1}\n{"line": 2}\n{"line": 3}\n' ,
564- } ) ;
565-
566- const lines : Array < { content : string ; lineNumber : number } > = [ ] ;
567- await processJSONLFileByLine ( path . join ( fixture . path , 'test.jsonl' ) , ( line , lineNumber ) => {
568- lines . push ( { content : line , lineNumber } ) ;
569- } ) ;
570-
571- expect ( lines ) . toHaveLength ( 3 ) ;
572- expect ( lines [ 0 ] ) . toEqual ( { content : '{"line": 1}' , lineNumber : 1 } ) ;
573- expect ( lines [ 1 ] ) . toEqual ( { content : '{"line": 2}' , lineNumber : 2 } ) ;
574- expect ( lines [ 2 ] ) . toEqual ( { content : '{"line": 3}' , lineNumber : 3 } ) ;
575- } ) ;
576-
577- it ( 'should skip empty lines' , async ( ) => {
578- await using fixture = await createFixture ( {
579- 'test.jsonl' : '{"line": 1}\n\n{"line": 2}\n \n{"line": 3}\n' ,
580- } ) ;
581-
582- const lines : string [ ] = [ ] ;
583- await processJSONLFileByLine ( path . join ( fixture . path , 'test.jsonl' ) , ( line ) => {
584- lines . push ( line ) ;
585- } ) ;
586-
587- expect ( lines ) . toHaveLength ( 3 ) ;
588- expect ( lines [ 0 ] ) . toBe ( '{"line": 1}' ) ;
589- expect ( lines [ 1 ] ) . toBe ( '{"line": 2}' ) ;
590- expect ( lines [ 2 ] ) . toBe ( '{"line": 3}' ) ;
591- } ) ;
592-
593- it ( 'should handle async processLine callback' , async ( ) => {
594- await using fixture = await createFixture ( {
595- 'test.jsonl' : '{"line": 1}\n{"line": 2}\n' ,
596- } ) ;
597-
598- const results : string [ ] = [ ] ;
599- await processJSONLFileByLine ( path . join ( fixture . path , 'test.jsonl' ) , async ( line ) => {
600- // Simulate async operation
601- await new Promise ( resolve => setTimeout ( resolve , 1 ) ) ;
602- results . push ( line ) ;
603- } ) ;
604-
605- expect ( results ) . toHaveLength ( 2 ) ;
606- expect ( results [ 0 ] ) . toBe ( '{"line": 1}' ) ;
607- expect ( results [ 1 ] ) . toBe ( '{"line": 2}' ) ;
608- } ) ;
609-
610- it ( 'should throw error when file does not exist' , async ( ) => {
611- await expect (
612- processJSONLFileByLine ( '/nonexistent/file.jsonl' , ( ) => { } ) ,
613- ) . rejects . toThrow ( ) ;
614- } ) ;
615-
616- it ( 'should handle empty file' , async ( ) => {
617- await using fixture = await createFixture ( {
618- 'empty.jsonl' : '' ,
619- } ) ;
620-
621- const lines : string [ ] = [ ] ;
622- await processJSONLFileByLine ( path . join ( fixture . path , 'empty.jsonl' ) , ( line ) => {
623- lines . push ( line ) ;
624- } ) ;
625-
626- expect ( lines ) . toHaveLength ( 0 ) ;
627- } ) ;
628-
629- it ( 'should handle file with only empty lines' , async ( ) => {
630- await using fixture = await createFixture ( {
631- 'only-empty.jsonl' : '\n\n \n\t\n' ,
632- } ) ;
633-
634- const lines : string [ ] = [ ] ;
635- await processJSONLFileByLine ( path . join ( fixture . path , 'only-empty.jsonl' ) , ( line ) => {
636- lines . push ( line ) ;
637- } ) ;
638-
639- expect ( lines ) . toHaveLength ( 0 ) ;
640- } ) ;
641-
642- it ( 'should process large files (600MB+) without RangeError' , async ( ) => {
643- // Create a realistic JSONL entry similar to actual Claude data (~283 bytes per line)
644- const sampleEntry = JSON . stringify ( {
645- timestamp : '2025-01-10T10:00:00Z' ,
646- message : {
647- id : 'msg_01234567890123456789' ,
648- usage : { input_tokens : 1000 , output_tokens : 500 } ,
649- model : 'claude-sonnet-4-20250514' ,
650- } ,
651- requestId : 'req_01234567890123456789' ,
652- costUSD : 0.01 ,
653- } ) + '\n' ;
654-
655- // Target 600MB file (this would cause RangeError with readFile in Node.js)
656- const targetMB = 600 ;
657- const lineSize = Buffer . byteLength ( sampleEntry , 'utf-8' ) ;
658- const lineCount = Math . ceil ( ( targetMB * 1024 * 1024 ) / lineSize ) ;
659-
660- // Create fixture directory first
661- await using fixture = await createFixture ( { } ) ;
662- const filePath = path . join ( fixture . path , 'large.jsonl' ) ;
663-
664- // Write file using streaming to avoid Node.js string length limit (~512MB)
665- // Creating a 600MB string directly would cause "RangeError: Invalid string length"
666- const writeStream = createWriteStream ( filePath ) ;
667-
668- // Write lines and handle backpressure
669- for ( let i = 0 ; i < lineCount ; i ++ ) {
670- const canContinue = writeStream . write ( sampleEntry ) ;
671- // Respect backpressure by waiting for drain event
672- if ( ! canContinue ) {
673- await new Promise < void > ( resolve => writeStream . once ( 'drain' , ( ) => resolve ( ) ) ) ;
674- }
675- }
676-
677- // Ensure all data is flushed
678- await new Promise < void > ( ( resolve , reject ) => {
679- writeStream . end ( ( err ?: Error | null ) => err ? reject ( err ) : resolve ( ) ) ;
680- } ) ;
681-
682- // Test streaming processing
683- let processedCount = 0 ;
684- await processJSONLFileByLine ( filePath , ( ) => {
685- processedCount ++ ;
686- } ) ;
687-
688- expect ( processedCount ) . toBe ( lineCount ) ;
689- } ) ;
690- } ) ;
691- }
692-
693559/**
694560 * Extract the earliest timestamp from a JSONL file
695561 * Scans through the file until it finds a valid timestamp
@@ -4215,6 +4081,138 @@ invalid json line
42154081 expect ( result ) . toHaveLength ( 1 ) ;
42164082 expect ( result [ 0 ] ?. entries ) . toHaveLength ( 1 ) ;
42174083 } ) ;
4084+
4085+ describe ( 'processJSONLFileByLine' , ( ) => {
4086+ it ( 'should process each non-empty line with correct line numbers' , async ( ) => {
4087+ await using fixture = await createFixture ( {
4088+ 'test.jsonl' : '{"line": 1}\n{"line": 2}\n{"line": 3}\n' ,
4089+ } ) ;
4090+
4091+ const lines : Array < { content : string ; lineNumber : number } > = [ ] ;
4092+ await processJSONLFileByLine ( path . join ( fixture . path , 'test.jsonl' ) , ( line , lineNumber ) => {
4093+ lines . push ( { content : line , lineNumber } ) ;
4094+ } ) ;
4095+
4096+ expect ( lines ) . toHaveLength ( 3 ) ;
4097+ expect ( lines [ 0 ] ) . toEqual ( { content : '{"line": 1}' , lineNumber : 1 } ) ;
4098+ expect ( lines [ 1 ] ) . toEqual ( { content : '{"line": 2}' , lineNumber : 2 } ) ;
4099+ expect ( lines [ 2 ] ) . toEqual ( { content : '{"line": 3}' , lineNumber : 3 } ) ;
4100+ } ) ;
4101+
4102+ it ( 'should skip empty lines' , async ( ) => {
4103+ await using fixture = await createFixture ( {
4104+ 'test.jsonl' : '{"line": 1}\n\n{"line": 2}\n \n{"line": 3}\n' ,
4105+ } ) ;
4106+
4107+ const lines : string [ ] = [ ] ;
4108+ await processJSONLFileByLine ( path . join ( fixture . path , 'test.jsonl' ) , ( line ) => {
4109+ lines . push ( line ) ;
4110+ } ) ;
4111+
4112+ expect ( lines ) . toHaveLength ( 3 ) ;
4113+ expect ( lines [ 0 ] ) . toBe ( '{"line": 1}' ) ;
4114+ expect ( lines [ 1 ] ) . toBe ( '{"line": 2}' ) ;
4115+ expect ( lines [ 2 ] ) . toBe ( '{"line": 3}' ) ;
4116+ } ) ;
4117+
4118+ it ( 'should handle async processLine callback' , async ( ) => {
4119+ await using fixture = await createFixture ( {
4120+ 'test.jsonl' : '{"line": 1}\n{"line": 2}\n' ,
4121+ } ) ;
4122+
4123+ const results : string [ ] = [ ] ;
4124+ await processJSONLFileByLine ( path . join ( fixture . path , 'test.jsonl' ) , async ( line ) => {
4125+ // Simulate async operation
4126+ await new Promise ( resolve => setTimeout ( resolve , 1 ) ) ;
4127+ results . push ( line ) ;
4128+ } ) ;
4129+
4130+ expect ( results ) . toHaveLength ( 2 ) ;
4131+ expect ( results [ 0 ] ) . toBe ( '{"line": 1}' ) ;
4132+ expect ( results [ 1 ] ) . toBe ( '{"line": 2}' ) ;
4133+ } ) ;
4134+
4135+ it ( 'should throw error when file does not exist' , async ( ) => {
4136+ await expect (
4137+ processJSONLFileByLine ( '/nonexistent/file.jsonl' , ( ) => { } ) ,
4138+ ) . rejects . toThrow ( ) ;
4139+ } ) ;
4140+
4141+ it ( 'should handle empty file' , async ( ) => {
4142+ await using fixture = await createFixture ( {
4143+ 'empty.jsonl' : '' ,
4144+ } ) ;
4145+
4146+ const lines : string [ ] = [ ] ;
4147+ await processJSONLFileByLine ( path . join ( fixture . path , 'empty.jsonl' ) , ( line ) => {
4148+ lines . push ( line ) ;
4149+ } ) ;
4150+
4151+ expect ( lines ) . toHaveLength ( 0 ) ;
4152+ } ) ;
4153+
4154+ it ( 'should handle file with only empty lines' , async ( ) => {
4155+ await using fixture = await createFixture ( {
4156+ 'only-empty.jsonl' : '\n\n \n\t\n' ,
4157+ } ) ;
4158+
4159+ const lines : string [ ] = [ ] ;
4160+ await processJSONLFileByLine ( path . join ( fixture . path , 'only-empty.jsonl' ) , ( line ) => {
4161+ lines . push ( line ) ;
4162+ } ) ;
4163+
4164+ expect ( lines ) . toHaveLength ( 0 ) ;
4165+ } ) ;
4166+
4167+ it ( 'should process large files (600MB+) without RangeError' , async ( ) => {
4168+ // Create a realistic JSONL entry similar to actual Claude data (~283 bytes per line)
4169+ const sampleEntry = JSON . stringify ( {
4170+ timestamp : '2025-01-10T10:00:00Z' ,
4171+ message : {
4172+ id : 'msg_01234567890123456789' ,
4173+ usage : { input_tokens : 1000 , output_tokens : 500 } ,
4174+ model : 'claude-sonnet-4-20250514' ,
4175+ } ,
4176+ requestId : 'req_01234567890123456789' ,
4177+ costUSD : 0.01 ,
4178+ } ) + '\n' ;
4179+
4180+ // Target 600MB file (this would cause RangeError with readFile in Node.js)
4181+ const targetMB = 600 ;
4182+ const lineSize = Buffer . byteLength ( sampleEntry , 'utf-8' ) ;
4183+ const lineCount = Math . ceil ( ( targetMB * 1024 * 1024 ) / lineSize ) ;
4184+
4185+ // Create fixture directory first
4186+ await using fixture = await createFixture ( { } ) ;
4187+ const filePath = path . join ( fixture . path , 'large.jsonl' ) ;
4188+
4189+ // Write file using streaming to avoid Node.js string length limit (~512MB)
4190+ // Creating a 600MB string directly would cause "RangeError: Invalid string length"
4191+ const writeStream = createWriteStream ( filePath ) ;
4192+
4193+ // Write lines and handle backpressure
4194+ for ( let i = 0 ; i < lineCount ; i ++ ) {
4195+ const canContinue = writeStream . write ( sampleEntry ) ;
4196+ // Respect backpressure by waiting for drain event
4197+ if ( ! canContinue ) {
4198+ await new Promise < void > ( resolve => writeStream . once ( 'drain' , ( ) => resolve ( ) ) ) ;
4199+ }
4200+ }
4201+
4202+ // Ensure all data is flushed
4203+ await new Promise < void > ( ( resolve , reject ) => {
4204+ writeStream . end ( ( err ?: Error | null ) => err ? reject ( err ) : resolve ( ) ) ;
4205+ } ) ;
4206+
4207+ // Test streaming processing
4208+ let processedCount = 0 ;
4209+ await processJSONLFileByLine ( filePath , ( ) => {
4210+ processedCount ++ ;
4211+ } ) ;
4212+
4213+ expect ( processedCount ) . toBe ( lineCount ) ;
4214+ } ) ;
4215+ } ) ;
42184216 } ) ;
42194217}
42204218
0 commit comments