Skip to content

Commit 778ca08

Browse files
committed
refactor(test): move processJSONLFileByLine tests to existing test block
Move processJSONLFileByLine tests from standalone block into the main test block with other data-loader tests, as requested in code review.
1 parent 48842e3 commit 778ca08

1 file changed

Lines changed: 132 additions & 134 deletions

File tree

‎apps/ccusage/src/data-loader.ts‎

Lines changed: 132 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -556,140 +556,6 @@ async function processJSONLFileByLine(
556556
}
557557
}
558558

559-
if (import.meta.vitest != null) {
560-
describe('processJSONLFileByLine', () => {
561-
it('should process each non-empty line with correct line numbers', async () => {
562-
await using fixture = await createFixture({
563-
'test.jsonl': '{"line": 1}\n{"line": 2}\n{"line": 3}\n',
564-
});
565-
566-
const lines: Array<{ content: string; lineNumber: number }> = [];
567-
await processJSONLFileByLine(path.join(fixture.path, 'test.jsonl'), (line, lineNumber) => {
568-
lines.push({ content: line, lineNumber });
569-
});
570-
571-
expect(lines).toHaveLength(3);
572-
expect(lines[0]).toEqual({ content: '{"line": 1}', lineNumber: 1 });
573-
expect(lines[1]).toEqual({ content: '{"line": 2}', lineNumber: 2 });
574-
expect(lines[2]).toEqual({ content: '{"line": 3}', lineNumber: 3 });
575-
});
576-
577-
it('should skip empty lines', async () => {
578-
await using fixture = await createFixture({
579-
'test.jsonl': '{"line": 1}\n\n{"line": 2}\n \n{"line": 3}\n',
580-
});
581-
582-
const lines: string[] = [];
583-
await processJSONLFileByLine(path.join(fixture.path, 'test.jsonl'), (line) => {
584-
lines.push(line);
585-
});
586-
587-
expect(lines).toHaveLength(3);
588-
expect(lines[0]).toBe('{"line": 1}');
589-
expect(lines[1]).toBe('{"line": 2}');
590-
expect(lines[2]).toBe('{"line": 3}');
591-
});
592-
593-
it('should handle async processLine callback', async () => {
594-
await using fixture = await createFixture({
595-
'test.jsonl': '{"line": 1}\n{"line": 2}\n',
596-
});
597-
598-
const results: string[] = [];
599-
await processJSONLFileByLine(path.join(fixture.path, 'test.jsonl'), async (line) => {
600-
// Simulate async operation
601-
await new Promise(resolve => setTimeout(resolve, 1));
602-
results.push(line);
603-
});
604-
605-
expect(results).toHaveLength(2);
606-
expect(results[0]).toBe('{"line": 1}');
607-
expect(results[1]).toBe('{"line": 2}');
608-
});
609-
610-
it('should throw error when file does not exist', async () => {
611-
await expect(
612-
processJSONLFileByLine('/nonexistent/file.jsonl', () => {}),
613-
).rejects.toThrow();
614-
});
615-
616-
it('should handle empty file', async () => {
617-
await using fixture = await createFixture({
618-
'empty.jsonl': '',
619-
});
620-
621-
const lines: string[] = [];
622-
await processJSONLFileByLine(path.join(fixture.path, 'empty.jsonl'), (line) => {
623-
lines.push(line);
624-
});
625-
626-
expect(lines).toHaveLength(0);
627-
});
628-
629-
it('should handle file with only empty lines', async () => {
630-
await using fixture = await createFixture({
631-
'only-empty.jsonl': '\n\n \n\t\n',
632-
});
633-
634-
const lines: string[] = [];
635-
await processJSONLFileByLine(path.join(fixture.path, 'only-empty.jsonl'), (line) => {
636-
lines.push(line);
637-
});
638-
639-
expect(lines).toHaveLength(0);
640-
});
641-
642-
it('should process large files (600MB+) without RangeError', async () => {
643-
// Create a realistic JSONL entry similar to actual Claude data (~283 bytes per line)
644-
const sampleEntry = JSON.stringify({
645-
timestamp: '2025-01-10T10:00:00Z',
646-
message: {
647-
id: 'msg_01234567890123456789',
648-
usage: { input_tokens: 1000, output_tokens: 500 },
649-
model: 'claude-sonnet-4-20250514',
650-
},
651-
requestId: 'req_01234567890123456789',
652-
costUSD: 0.01,
653-
}) + '\n';
654-
655-
// Target 600MB file (this would cause RangeError with readFile in Node.js)
656-
const targetMB = 600;
657-
const lineSize = Buffer.byteLength(sampleEntry, 'utf-8');
658-
const lineCount = Math.ceil((targetMB * 1024 * 1024) / lineSize);
659-
660-
// Create fixture directory first
661-
await using fixture = await createFixture({});
662-
const filePath = path.join(fixture.path, 'large.jsonl');
663-
664-
// Write file using streaming to avoid Node.js string length limit (~512MB)
665-
// Creating a 600MB string directly would cause "RangeError: Invalid string length"
666-
const writeStream = createWriteStream(filePath);
667-
668-
// Write lines and handle backpressure
669-
for (let i = 0; i < lineCount; i++) {
670-
const canContinue = writeStream.write(sampleEntry);
671-
// Respect backpressure by waiting for drain event
672-
if (!canContinue) {
673-
await new Promise<void>(resolve => writeStream.once('drain', () => resolve()));
674-
}
675-
}
676-
677-
// Ensure all data is flushed
678-
await new Promise<void>((resolve, reject) => {
679-
writeStream.end((err?: Error | null) => err ? reject(err) : resolve());
680-
});
681-
682-
// Test streaming processing
683-
let processedCount = 0;
684-
await processJSONLFileByLine(filePath, () => {
685-
processedCount++;
686-
});
687-
688-
expect(processedCount).toBe(lineCount);
689-
});
690-
});
691-
}
692-
693559
/**
694560
* Extract the earliest timestamp from a JSONL file
695561
* Scans through the file until it finds a valid timestamp
@@ -4215,6 +4081,138 @@ invalid json line
42154081
expect(result).toHaveLength(1);
42164082
expect(result[0]?.entries).toHaveLength(1);
42174083
});
4084+
4085+
describe('processJSONLFileByLine', () => {
4086+
it('should process each non-empty line with correct line numbers', async () => {
4087+
await using fixture = await createFixture({
4088+
'test.jsonl': '{"line": 1}\n{"line": 2}\n{"line": 3}\n',
4089+
});
4090+
4091+
const lines: Array<{ content: string; lineNumber: number }> = [];
4092+
await processJSONLFileByLine(path.join(fixture.path, 'test.jsonl'), (line, lineNumber) => {
4093+
lines.push({ content: line, lineNumber });
4094+
});
4095+
4096+
expect(lines).toHaveLength(3);
4097+
expect(lines[0]).toEqual({ content: '{"line": 1}', lineNumber: 1 });
4098+
expect(lines[1]).toEqual({ content: '{"line": 2}', lineNumber: 2 });
4099+
expect(lines[2]).toEqual({ content: '{"line": 3}', lineNumber: 3 });
4100+
});
4101+
4102+
it('should skip empty lines', async () => {
4103+
await using fixture = await createFixture({
4104+
'test.jsonl': '{"line": 1}\n\n{"line": 2}\n \n{"line": 3}\n',
4105+
});
4106+
4107+
const lines: string[] = [];
4108+
await processJSONLFileByLine(path.join(fixture.path, 'test.jsonl'), (line) => {
4109+
lines.push(line);
4110+
});
4111+
4112+
expect(lines).toHaveLength(3);
4113+
expect(lines[0]).toBe('{"line": 1}');
4114+
expect(lines[1]).toBe('{"line": 2}');
4115+
expect(lines[2]).toBe('{"line": 3}');
4116+
});
4117+
4118+
it('should handle async processLine callback', async () => {
4119+
await using fixture = await createFixture({
4120+
'test.jsonl': '{"line": 1}\n{"line": 2}\n',
4121+
});
4122+
4123+
const results: string[] = [];
4124+
await processJSONLFileByLine(path.join(fixture.path, 'test.jsonl'), async (line) => {
4125+
// Simulate async operation
4126+
await new Promise(resolve => setTimeout(resolve, 1));
4127+
results.push(line);
4128+
});
4129+
4130+
expect(results).toHaveLength(2);
4131+
expect(results[0]).toBe('{"line": 1}');
4132+
expect(results[1]).toBe('{"line": 2}');
4133+
});
4134+
4135+
it('should throw error when file does not exist', async () => {
4136+
await expect(
4137+
processJSONLFileByLine('/nonexistent/file.jsonl', () => {}),
4138+
).rejects.toThrow();
4139+
});
4140+
4141+
it('should handle empty file', async () => {
4142+
await using fixture = await createFixture({
4143+
'empty.jsonl': '',
4144+
});
4145+
4146+
const lines: string[] = [];
4147+
await processJSONLFileByLine(path.join(fixture.path, 'empty.jsonl'), (line) => {
4148+
lines.push(line);
4149+
});
4150+
4151+
expect(lines).toHaveLength(0);
4152+
});
4153+
4154+
it('should handle file with only empty lines', async () => {
4155+
await using fixture = await createFixture({
4156+
'only-empty.jsonl': '\n\n \n\t\n',
4157+
});
4158+
4159+
const lines: string[] = [];
4160+
await processJSONLFileByLine(path.join(fixture.path, 'only-empty.jsonl'), (line) => {
4161+
lines.push(line);
4162+
});
4163+
4164+
expect(lines).toHaveLength(0);
4165+
});
4166+
4167+
it('should process large files (600MB+) without RangeError', async () => {
4168+
// Create a realistic JSONL entry similar to actual Claude data (~283 bytes per line)
4169+
const sampleEntry = JSON.stringify({
4170+
timestamp: '2025-01-10T10:00:00Z',
4171+
message: {
4172+
id: 'msg_01234567890123456789',
4173+
usage: { input_tokens: 1000, output_tokens: 500 },
4174+
model: 'claude-sonnet-4-20250514',
4175+
},
4176+
requestId: 'req_01234567890123456789',
4177+
costUSD: 0.01,
4178+
}) + '\n';
4179+
4180+
// Target 600MB file (this would cause RangeError with readFile in Node.js)
4181+
const targetMB = 600;
4182+
const lineSize = Buffer.byteLength(sampleEntry, 'utf-8');
4183+
const lineCount = Math.ceil((targetMB * 1024 * 1024) / lineSize);
4184+
4185+
// Create fixture directory first
4186+
await using fixture = await createFixture({});
4187+
const filePath = path.join(fixture.path, 'large.jsonl');
4188+
4189+
// Write file using streaming to avoid Node.js string length limit (~512MB)
4190+
// Creating a 600MB string directly would cause "RangeError: Invalid string length"
4191+
const writeStream = createWriteStream(filePath);
4192+
4193+
// Write lines and handle backpressure
4194+
for (let i = 0; i < lineCount; i++) {
4195+
const canContinue = writeStream.write(sampleEntry);
4196+
// Respect backpressure by waiting for drain event
4197+
if (!canContinue) {
4198+
await new Promise<void>(resolve => writeStream.once('drain', () => resolve()));
4199+
}
4200+
}
4201+
4202+
// Ensure all data is flushed
4203+
await new Promise<void>((resolve, reject) => {
4204+
writeStream.end((err?: Error | null) => err ? reject(err) : resolve());
4205+
});
4206+
4207+
// Test streaming processing
4208+
let processedCount = 0;
4209+
await processJSONLFileByLine(filePath, () => {
4210+
processedCount++;
4211+
});
4212+
4213+
expect(processedCount).toBe(lineCount);
4214+
});
4215+
});
42184216
});
42194217
}
42204218

0 commit comments

Comments
 (0)