Skip to content

Commit a3c9046

Browse files
committed
fix(ccusage): use streaming to handle large JSONL files
Fixes #460 by implementing line-by-line streaming for JSONL file processing instead of loading entire files into memory. This prevents RangeError: Invalid string length errors when processing large session files (>500MB). Changes: Add processJSONLFileByLine() helper using Node.js readline, Update getEarliestTimestamp() to use streaming, Update loadDailyUsageData() to use streaming, Update loadSessionData() to use streaming, Update loadSessionUsageById() to use streaming, Update loadSessionBlockData() to use streaming All existing tests pass with the new streaming implementation.
1 parent b40070b commit a3c9046

1 file changed

Lines changed: 47 additions & 47 deletions

File tree

‎apps/ccusage/src/data-loader.ts‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@ import type {
1818
SortOrder,
1919
Version,
2020
} from './_types.ts';
21+
import { createReadStream } from 'node:fs';
2122
import { readFile } from 'node:fs/promises';
2223
import path from 'node:path';
2324
import process from 'node:process';
25+
import { createInterface } from 'node:readline';
2426
import { toArray } from '@antfu/utils';
2527
import { Result } from '@praha/byethrow';
2628
import { groupBy, uniq } from 'es-toolkit'; // TODO: after node20 is deprecated, switch to native Object.groupBy
@@ -529,22 +531,41 @@ export function createUniqueHash(data: UsageData): string | null {
529531
return `${messageId}:${requestId}`;
530532
}
531533

534+
/**
535+
* Process a JSONL file line by line using streams to avoid memory issues with large files
536+
* @param filePath - Path to the JSONL file
537+
* @param processLine - Callback function to process each line
538+
*/
539+
async function processJSONLFileByLine(
540+
filePath: string,
541+
processLine: (line: string, lineNumber: number) => void | Promise<void>,
542+
): Promise<void> {
543+
const fileStream = createReadStream(filePath, { encoding: 'utf-8' });
544+
const rl = createInterface({
545+
input: fileStream,
546+
crlfDelay: Number.POSITIVE_INFINITY,
547+
});
548+
549+
let lineNumber = 0;
550+
for await (const line of rl) {
551+
lineNumber++;
552+
if (line.trim().length === 0) {
553+
continue;
554+
}
555+
await processLine(line, lineNumber);
556+
}
557+
}
558+
532559
/**
533560
* Extract the earliest timestamp from a JSONL file
534561
* Scans through the file until it finds a valid timestamp
562+
* Uses streaming to handle large files without loading entire content into memory
535563
*/
536564
export async function getEarliestTimestamp(filePath: string): Promise<Date | null> {
537565
try {
538-
const content = await readFile(filePath, 'utf-8');
539-
const lines = content.trim().split('\n');
540-
541566
let earliestDate: Date | null = null;
542567

543-
for (const line of lines) {
544-
if (line.trim() === '') {
545-
continue;
546-
}
547-
568+
await processJSONLFileByLine(filePath, (line) => {
548569
try {
549570
const json = JSON.parse(line) as Record<string, unknown>;
550571
if (json.timestamp != null && typeof json.timestamp === 'string') {
@@ -558,9 +579,8 @@ export async function getEarliestTimestamp(filePath: string): Promise<Date | nul
558579
}
559580
catch {
560581
// Skip invalid JSON lines
561-
continue;
562582
}
563-
}
583+
});
564584

565585
return earliestDate;
566586
}
@@ -759,26 +779,23 @@ export async function loadDailyUsageData(
759779
const allEntries: { data: UsageData; date: string; cost: number; model: string | undefined; project: string }[] = [];
760780

761781
for (const file of sortedFiles) {
762-
const content = await readFile(file, 'utf-8');
763-
const lines = content
764-
.trim()
765-
.split('\n')
766-
.filter(line => line.length > 0);
782+
// Extract project name from file path once per file
783+
const project = extractProjectFromPath(file);
767784

768-
for (const line of lines) {
785+
await processJSONLFileByLine(file, async (line) => {
769786
try {
770787
const parsed = JSON.parse(line) as unknown;
771788
const result = v.safeParse(usageDataSchema, parsed);
772789
if (!result.success) {
773-
continue;
790+
return;
774791
}
775792
const data = result.output;
776793

777794
// Check for duplicate message + request ID combination
778795
const uniqueHash = createUniqueHash(data);
779796
if (isDuplicateEntry(uniqueHash, processedHashes)) {
780797
// Skip duplicate message
781-
continue;
798+
return;
782799
}
783800

784801
// Mark this combination as processed
@@ -792,15 +809,12 @@ export async function loadDailyUsageData(
792809
? await calculateCostForEntry(data, mode, fetcher)
793810
: data.costUSD ?? 0;
794811

795-
// Extract project name from file path
796-
const project = extractProjectFromPath(file);
797-
798812
allEntries.push({ data, date, cost, model: data.message.model, project });
799813
}
800814
catch {
801815
// Skip invalid JSON lines
802816
}
803-
}
817+
});
804818
}
805819

806820
// Group by date, optionally including project
@@ -933,26 +947,20 @@ export async function loadSessionData(
933947
const joinedPath = parts.slice(0, -2).join(path.sep);
934948
const projectPath = joinedPath.length > 0 ? joinedPath : 'Unknown Project';
935949

936-
const content = await readFile(file, 'utf-8');
937-
const lines = content
938-
.trim()
939-
.split('\n')
940-
.filter(line => line.length > 0);
941-
942-
for (const line of lines) {
950+
await processJSONLFileByLine(file, async (line) => {
943951
try {
944952
const parsed = JSON.parse(line) as unknown;
945953
const result = v.safeParse(usageDataSchema, parsed);
946954
if (!result.success) {
947-
continue;
955+
return;
948956
}
949957
const data = result.output;
950958

951959
// Check for duplicate message + request ID combination
952960
const uniqueHash = createUniqueHash(data);
953961
if (isDuplicateEntry(uniqueHash, processedHashes)) {
954962
// Skip duplicate message
955-
continue;
963+
return;
956964
}
957965

958966
// Mark this combination as processed
@@ -976,7 +984,7 @@ export async function loadSessionData(
976984
catch {
977985
// Skip invalid JSON lines
978986
}
979-
}
987+
});
980988
}
981989

982990
// Group by session using Object.groupBy
@@ -1103,21 +1111,19 @@ export async function loadSessionUsageById(
11031111
if (file == null) {
11041112
return null;
11051113
}
1106-
const content = await readFile(file, 'utf-8');
1107-
const lines = content.trim().split('\n').filter(line => line.length > 0);
11081114

11091115
const mode = options?.mode ?? 'auto';
11101116
using fetcher = mode === 'display' ? null : new PricingFetcher(options?.offline);
11111117

11121118
const entries: UsageData[] = [];
11131119
let totalCost = 0;
11141120

1115-
for (const line of lines) {
1121+
await processJSONLFileByLine(file, async (line) => {
11161122
try {
11171123
const parsed = JSON.parse(line) as unknown;
11181124
const result = v.safeParse(usageDataSchema, parsed);
11191125
if (!result.success) {
1120-
continue;
1126+
return;
11211127
}
11221128
const data = result.output;
11231129

@@ -1131,7 +1137,7 @@ export async function loadSessionUsageById(
11311137
catch {
11321138
// Skip invalid JSON lines
11331139
}
1134-
}
1140+
});
11351141

11361142
return { totalCost, entries };
11371143
}
@@ -1353,26 +1359,20 @@ export async function loadSessionBlockData(
13531359
const allEntries: LoadedUsageEntry[] = [];
13541360

13551361
for (const file of sortedFiles) {
1356-
const content = await readFile(file, 'utf-8');
1357-
const lines = content
1358-
.trim()
1359-
.split('\n')
1360-
.filter(line => line.length > 0);
1361-
1362-
for (const line of lines) {
1362+
await processJSONLFileByLine(file, async (line) => {
13631363
try {
13641364
const parsed = JSON.parse(line) as unknown;
13651365
const result = v.safeParse(usageDataSchema, parsed);
13661366
if (!result.success) {
1367-
continue;
1367+
return;
13681368
}
13691369
const data = result.output;
13701370

13711371
// Check for duplicate message + request ID combination
13721372
const uniqueHash = createUniqueHash(data);
13731373
if (isDuplicateEntry(uniqueHash, processedHashes)) {
13741374
// Skip duplicate message
1375-
continue;
1375+
return;
13761376
}
13771377

13781378
// Mark this combination as processed
@@ -1403,7 +1403,7 @@ export async function loadSessionBlockData(
14031403
// Skip invalid JSON lines but log for debugging purposes
14041404
logger.debug(`Skipping invalid JSON line in 5-hour blocks: ${error instanceof Error ? error.message : String(error)}`);
14051405
}
1406-
}
1406+
});
14071407
}
14081408

14091409
// Identify session blocks

0 commit comments

Comments
 (0)