Skip to content

Commit 2878e9c

Browse files
committed
perf(internal): preserve synchronous JSONL marker scans
Add an explicit synchronous callback mode to the shared JSONL marker helper so agent loaders that do purely synchronous parsing do not pay per-row Promise handling overhead. Keep markerIndex byte semantics consistent for line-scan and text fallback paths by converting decoded-string indexes back to byte offsets with Buffer.byteLength. This addresses the latest CodeRabbit JSONL marker-index comment without failing large-file fallback paths. Claude and Codex opt into the synchronous mode for their hot JSONL parser callbacks. Validation: pnpm run format; pnpm typecheck; pnpm --filter @ccusage/internal test src/jsonl.ts --run; pnpm --filter ccusage test src/adapter/codex/parser.ts --run; pnpm --filter ccusage build; JSON parity against main through 2026-05-14 for claude/codex with TZ=UTC.
1 parent 6b2e2c9 commit 2878e9c

3 files changed

Lines changed: 93 additions & 3 deletions

File tree

‎apps/ccusage/src/adapter/claude/data-loader.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1200,6 +1200,7 @@ async function processJSONLUsageFileByLine(
12001200
},
12011201
{
12021202
bufferedEncoding: 'latin1',
1203+
callbackMode: 'sync',
12031204
markerIndex: 'byte',
12041205
},
12051206
);

‎apps/ccusage/src/adapter/codex/parser.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,7 @@ async function parseCodexSessionFile(
280280
},
281281
{
282282
bufferedEncoding: 'latin1',
283+
callbackMode: 'sync',
283284
scanMode: 'line',
284285
},
285286
);

‎packages/internal/src/jsonl.ts‎

Lines changed: 91 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ const MAX_BUFFERED_JSONL_BYTES = 128 * 1024 * 1024;
88

99
type JSONLMarkerProcessingOptions = {
1010
bufferedEncoding?: BufferEncoding;
11+
callbackMode?: 'async' | 'sync';
1112
markerIndex?: 'byte' | 'decoded';
1213
scanMode?: 'marker' | 'line';
1314
};
@@ -54,6 +55,16 @@ function normalizeMarker(value: string): string | null {
5455
return hasNonWhitespace(value) ? value : null;
5556
}
5657

58+
function markerIndexForLine(
59+
line: string,
60+
markerIndex: number,
61+
options: JSONLMarkerProcessingOptions,
62+
): number {
63+
return options.markerIndex === 'byte'
64+
? Buffer.byteLength(line.slice(0, markerIndex), options.bufferedEncoding ?? 'utf8')
65+
: markerIndex;
66+
}
67+
5768
async function processBufferedJSONLMarkerBytes(
5869
bytes: Uint8Array,
5970
markers: readonly string[],
@@ -68,6 +79,7 @@ async function processBufferedJSONLMarkerBytes(
6879
content.toString(options.bufferedEncoding ?? 'utf8'),
6980
markers,
7081
processLine,
82+
options,
7183
);
7284
return;
7385
}
@@ -107,6 +119,21 @@ async function processBufferedJSONLMarkerBytes(
107119
}
108120

109121
const lineStarts = Array.from(candidates.keys()).sort((a, b) => a - b);
122+
if (options.callbackMode === 'sync') {
123+
for (const lineStart of lineStarts) {
124+
const candidate = candidates.get(lineStart)!;
125+
const lineEnd = candidate.lineEnd;
126+
const decodeEnd = lineEnd > lineStart && content[lineEnd - 1] === 13 ? lineEnd - 1 : lineEnd;
127+
const line = content.toString(options.bufferedEncoding ?? 'utf8', lineStart, decodeEnd);
128+
void processLine(
129+
line,
130+
options.markerIndex === 'byte' ? candidate.markerIndex : line.indexOf(candidate.marker),
131+
candidate.marker,
132+
);
133+
}
134+
return;
135+
}
136+
110137
for (const lineStart of lineStarts) {
111138
const candidate = candidates.get(lineStart)!;
112139
const lineEnd = candidate.lineEnd;
@@ -130,6 +157,35 @@ async function processBufferedJSONLSingleMarkerBytes(
130157
): Promise<void> {
131158
let lineStart = 0;
132159
let markerIndex = content.indexOf(markerBuffer, lineStart);
160+
if (options.callbackMode === 'sync') {
161+
while (markerIndex !== -1) {
162+
while (true) {
163+
const nextLineEnd = content.indexOf(10, lineStart);
164+
if (nextLineEnd === -1 || nextLineEnd >= markerIndex) {
165+
break;
166+
}
167+
lineStart = nextLineEnd + 1;
168+
}
169+
let lineEnd = content.indexOf(10, markerIndex);
170+
if (lineEnd === -1) {
171+
lineEnd = content.length;
172+
}
173+
174+
const decodeEnd = lineEnd > lineStart && content[lineEnd - 1] === 13 ? lineEnd - 1 : lineEnd;
175+
const line = content.toString(options.bufferedEncoding ?? 'utf8', lineStart, decodeEnd);
176+
const lineMarkerIndex = markerIndex - lineStart;
177+
void processLine(
178+
line,
179+
options.markerIndex === 'byte' ? lineMarkerIndex : line.indexOf(marker),
180+
marker,
181+
);
182+
183+
lineStart = lineEnd + 1;
184+
markerIndex = content.indexOf(markerBuffer, lineStart);
185+
}
186+
return;
187+
}
188+
133189
while (markerIndex !== -1) {
134190
while (true) {
135191
const nextLineEnd = content.indexOf(10, lineStart);
@@ -164,6 +220,7 @@ async function processBufferedJSONLMarkerContent(
164220
content: string,
165221
markers: readonly string[],
166222
processLine: (line: string, markerIndex: number, marker: string) => void | Promise<void>,
223+
options: JSONLMarkerProcessingOptions = {},
167224
): Promise<void> {
168225
let lineStart = 0;
169226
while (lineStart < content.length) {
@@ -187,7 +244,13 @@ async function processBufferedJSONLMarkerContent(
187244
}
188245
}
189246
if (firstMarker != null) {
190-
const result = processLine(line, firstMarkerIndex, firstMarker);
247+
const callbackMarkerIndex = markerIndexForLine(line, firstMarkerIndex, options);
248+
if (options.callbackMode === 'sync') {
249+
void processLine(line, callbackMarkerIndex, firstMarker);
250+
lineStart = lineEnd + 1;
251+
continue;
252+
}
253+
const result = processLine(line, callbackMarkerIndex, firstMarker);
191254
if (result != null) {
192255
await result;
193256
}
@@ -266,7 +329,7 @@ export async function processJSONLFileByMarkers(
266329

267330
const content = await readBufferedJSONLContent(filePath);
268331
if (content != null) {
269-
await processBufferedJSONLMarkerContent(content, normalizedMarkers, processLine);
332+
await processBufferedJSONLMarkerContent(content, normalizedMarkers, processLine, options);
270333
return;
271334
}
272335

@@ -289,7 +352,12 @@ export async function processJSONLFileByMarkers(
289352
if (firstMarker == null) {
290353
continue;
291354
}
292-
const result = processLine(line, firstMarkerIndex, firstMarker);
355+
const callbackMarkerIndex = markerIndexForLine(line, firstMarkerIndex, options);
356+
if (options.callbackMode === 'sync') {
357+
void processLine(line, callbackMarkerIndex, firstMarker);
358+
continue;
359+
}
360+
const result = processLine(line, callbackMarkerIndex, firstMarker);
293361
if (result != null) {
294362
await result;
295363
}
@@ -417,5 +485,25 @@ if (import.meta.vitest != null) {
417485
'{"type":"event_msg","payload":{"type":"token_count"}}',
418486
]);
419487
});
488+
489+
it('returns byte marker indexes in line-scan mode', async () => {
490+
const prefix = '{"note":"東京",';
491+
const marker = '"type":"token_count"';
492+
await using fixture = await createFixture({
493+
'test.jsonl': `${prefix}${marker}}`,
494+
});
495+
496+
const seen: number[] = [];
497+
await processJSONLFileByMarkers(
498+
fixture.getPath('test.jsonl'),
499+
[marker],
500+
(_line, markerIndex) => {
501+
seen.push(markerIndex);
502+
},
503+
{ markerIndex: 'byte', scanMode: 'line' },
504+
);
505+
506+
expect(seen).toEqual([Buffer.byteLength(prefix)]);
507+
});
420508
});
421509
}

0 commit comments

Comments
 (0)