Skip to content

Commit 3f8c461

Browse files
committed
fix(codex): stream large session jsonl files
1 parent 9f4129c commit 3f8c461

1 file changed

Lines changed: 166 additions & 108 deletions

File tree

‎apps/codex/src/data-loader.ts‎

Lines changed: 166 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import type { TokenUsageDelta, TokenUsageEvent } from './_types.ts';
2-
import { readFile, stat } from 'node:fs/promises';
2+
import { createReadStream } from 'node:fs';
3+
import { stat } from 'node:fs/promises';
34
import path from 'node:path';
45
import process from 'node:process';
6+
import { createInterface } from 'node:readline';
57
import { Result } from '@praha/byethrow';
68
import { createFixture } from 'fs-fixture';
79
import { glob } from 'tinyglobby';
@@ -225,136 +227,135 @@ export async function loadTokenUsageEvents(options: LoadOptions = {}): Promise<L
225227
const relativeSessionPath = path.relative(directoryPath, file);
226228
const normalizedSessionPath = relativeSessionPath.split(path.sep).join('/');
227229
const sessionId = normalizedSessionPath.replace(/\.jsonl$/i, '');
228-
const fileContentResult = await Result.try({
229-
try: readFile(file, 'utf8'),
230-
catch: (error) => error,
231-
});
232-
233-
if (Result.isFailure(fileContentResult)) {
234-
logger.debug('Failed to read Codex session file', fileContentResult.error);
235-
continue;
236-
}
237-
238230
let previousTotals: RawUsage | null = null;
239231
let currentModel: string | undefined;
240232
let currentModelIsFallback = false;
241233
let legacyFallbackUsed = false;
242-
const lines = fileContentResult.value.split(/\r?\n/);
243-
for (const line of lines) {
244-
const trimmed = line.trim();
245-
if (trimmed === '') {
246-
continue;
247-
}
234+
const lineReader = createInterface({
235+
input: createReadStream(file, { encoding: 'utf8' }),
236+
crlfDelay: Number.POSITIVE_INFINITY,
237+
});
248238

249-
const parseLine = Result.try({
250-
try: () => JSON.parse(trimmed) as unknown,
251-
catch: (error) => error,
252-
});
253-
const parsedResult = parseLine();
239+
try {
240+
for await (const line of lineReader) {
241+
const trimmed = line.trim();
242+
if (trimmed === '') {
243+
continue;
244+
}
254245

255-
if (Result.isFailure(parsedResult)) {
256-
continue;
257-
}
246+
const parseLine = Result.try({
247+
try: () => JSON.parse(trimmed) as unknown,
248+
catch: (error) => error,
249+
});
250+
const parsedResult = parseLine();
258251

259-
const entryParse = v.safeParse(entrySchema, parsedResult.value);
260-
if (!entryParse.success) {
261-
continue;
262-
}
252+
if (Result.isFailure(parsedResult)) {
253+
continue;
254+
}
255+
256+
const entryParse = v.safeParse(entrySchema, parsedResult.value);
257+
if (!entryParse.success) {
258+
continue;
259+
}
263260

264-
const { type: entryType, payload, timestamp } = entryParse.output;
261+
const { type: entryType, payload, timestamp } = entryParse.output;
265262

266-
if (entryType === 'turn_context') {
267-
const contextPayload = v.safeParse(recordSchema, payload ?? null);
268-
if (contextPayload.success) {
269-
const contextModel = extractModel(contextPayload.output);
270-
if (contextModel != null) {
271-
currentModel = contextModel;
272-
currentModelIsFallback = false;
263+
if (entryType === 'turn_context') {
264+
const contextPayload = v.safeParse(recordSchema, payload ?? null);
265+
if (contextPayload.success) {
266+
const contextModel = extractModel(contextPayload.output);
267+
if (contextModel != null) {
268+
currentModel = contextModel;
269+
currentModelIsFallback = false;
270+
}
273271
}
272+
continue;
274273
}
275-
continue;
276-
}
277274

278-
if (entryType !== 'event_msg') {
279-
continue;
280-
}
275+
if (entryType !== 'event_msg') {
276+
continue;
277+
}
281278

282-
const tokenPayloadResult = v.safeParse(tokenCountPayloadSchema, payload ?? undefined);
283-
if (!tokenPayloadResult.success) {
284-
continue;
285-
}
279+
const tokenPayloadResult = v.safeParse(tokenCountPayloadSchema, payload ?? undefined);
280+
if (!tokenPayloadResult.success) {
281+
continue;
282+
}
286283

287-
if (timestamp == null) {
288-
continue;
289-
}
284+
if (timestamp == null) {
285+
continue;
286+
}
290287

291-
const info = tokenPayloadResult.output.info;
292-
const lastUsage = normalizeRawUsage(info?.last_token_usage);
293-
const totalUsage = normalizeRawUsage(info?.total_token_usage);
288+
const info = tokenPayloadResult.output.info;
289+
const lastUsage = normalizeRawUsage(info?.last_token_usage);
290+
const totalUsage = normalizeRawUsage(info?.total_token_usage);
294291

295-
let raw = lastUsage;
296-
if (raw == null && totalUsage != null) {
297-
raw = subtractRawUsage(totalUsage, previousTotals);
298-
}
292+
let raw = lastUsage;
293+
if (raw == null && totalUsage != null) {
294+
raw = subtractRawUsage(totalUsage, previousTotals);
295+
}
299296

300-
if (totalUsage != null) {
301-
previousTotals = totalUsage;
302-
}
297+
if (totalUsage != null) {
298+
previousTotals = totalUsage;
299+
}
303300

304-
if (raw == null) {
305-
continue;
306-
}
301+
if (raw == null) {
302+
continue;
303+
}
307304

308-
const delta = convertToDelta(raw);
309-
if (
310-
delta.inputTokens === 0 &&
311-
delta.cachedInputTokens === 0 &&
312-
delta.outputTokens === 0 &&
313-
delta.reasoningOutputTokens === 0
314-
) {
315-
continue;
316-
}
305+
const delta = convertToDelta(raw);
306+
if (
307+
delta.inputTokens === 0 &&
308+
delta.cachedInputTokens === 0 &&
309+
delta.outputTokens === 0 &&
310+
delta.reasoningOutputTokens === 0
311+
) {
312+
continue;
313+
}
317314

318-
const payloadRecordResult = v.safeParse(recordSchema, payload ?? undefined);
319-
const extractionSource = payloadRecordResult.success
320-
? Object.assign({}, payloadRecordResult.output, { info })
321-
: { info };
322-
const extractedModel = extractModel(extractionSource);
323-
let isFallbackModel = false;
324-
if (extractedModel != null) {
325-
currentModel = extractedModel;
326-
currentModelIsFallback = false;
327-
}
315+
const payloadRecordResult = v.safeParse(recordSchema, payload ?? undefined);
316+
const extractionSource = payloadRecordResult.success
317+
? Object.assign({}, payloadRecordResult.output, { info })
318+
: { info };
319+
const extractedModel = extractModel(extractionSource);
320+
let isFallbackModel = false;
321+
if (extractedModel != null) {
322+
currentModel = extractedModel;
323+
currentModelIsFallback = false;
324+
}
328325

329-
let model = extractedModel ?? currentModel;
330-
if (model == null) {
331-
model = LEGACY_FALLBACK_MODEL;
332-
isFallbackModel = true;
333-
legacyFallbackUsed = true;
334-
currentModel = model;
335-
currentModelIsFallback = true;
336-
} else if (extractedModel == null && currentModelIsFallback) {
337-
isFallbackModel = true;
338-
}
326+
let model = extractedModel ?? currentModel;
327+
if (model == null) {
328+
model = LEGACY_FALLBACK_MODEL;
329+
isFallbackModel = true;
330+
legacyFallbackUsed = true;
331+
currentModel = model;
332+
currentModelIsFallback = true;
333+
} else if (extractedModel == null && currentModelIsFallback) {
334+
isFallbackModel = true;
335+
}
339336

340-
const event: TokenUsageEvent = {
341-
sessionId,
342-
timestamp,
343-
model,
344-
inputTokens: delta.inputTokens,
345-
cachedInputTokens: delta.cachedInputTokens,
346-
outputTokens: delta.outputTokens,
347-
reasoningOutputTokens: delta.reasoningOutputTokens,
348-
totalTokens: delta.totalTokens,
349-
};
350-
351-
if (isFallbackModel) {
352-
// Surface the fallback so both table + JSON outputs can annotate pricing that was
353-
// inferred rather than sourced from the log metadata.
354-
event.isFallbackModel = true;
355-
}
337+
const event: TokenUsageEvent = {
338+
sessionId,
339+
timestamp,
340+
model,
341+
inputTokens: delta.inputTokens,
342+
cachedInputTokens: delta.cachedInputTokens,
343+
outputTokens: delta.outputTokens,
344+
reasoningOutputTokens: delta.reasoningOutputTokens,
345+
totalTokens: delta.totalTokens,
346+
};
347+
348+
if (isFallbackModel) {
349+
// Surface the fallback so both table + JSON outputs can annotate pricing that was
350+
// inferred rather than sourced from the log metadata.
351+
event.isFallbackModel = true;
352+
}
356353

357-
events.push(event);
354+
events.push(event);
355+
}
356+
} catch (error) {
357+
logger.debug('Failed to read Codex session file', error);
358+
continue;
358359
}
359360

360361
if (legacyFallbackUsed) {
@@ -484,5 +485,62 @@ if (import.meta.vitest != null) {
484485
expect(events[0]!.model).toBe('gpt-5');
485486
expect(events[0]!.isFallbackModel).toBe(true);
486487
});
488+
489+
it('parses compacted-heavy sessions', async () => {
490+
const hugeCompactedPayload = 'x'.repeat(2_000_000);
491+
await using fixture = await createFixture({
492+
sessions: {
493+
'compacted.jsonl': [
494+
JSON.stringify({
495+
timestamp: '2025-09-15T13:00:00.000Z',
496+
type: 'compacted',
497+
payload: {
498+
replacement_history: hugeCompactedPayload,
499+
},
500+
}),
501+
JSON.stringify({
502+
timestamp: '2025-09-15T13:00:10.000Z',
503+
type: 'turn_context',
504+
payload: {
505+
model: 'gpt-5.4',
506+
},
507+
}),
508+
JSON.stringify({
509+
timestamp: '2025-09-15T13:00:11.000Z',
510+
type: 'event_msg',
511+
payload: {
512+
type: 'token_count',
513+
info: {
514+
last_token_usage: {
515+
input_tokens: 42,
516+
cached_input_tokens: 2,
517+
output_tokens: 8,
518+
reasoning_output_tokens: 1,
519+
total_tokens: 50,
520+
},
521+
},
522+
},
523+
}),
524+
].join('\n'),
525+
},
526+
});
527+
528+
const { events } = await loadTokenUsageEvents({
529+
sessionDirs: [fixture.getPath('sessions')],
530+
});
531+
532+
expect(events).toEqual([
533+
expect.objectContaining({
534+
sessionId: 'compacted',
535+
timestamp: '2025-09-15T13:00:11.000Z',
536+
model: 'gpt-5.4',
537+
inputTokens: 42,
538+
cachedInputTokens: 2,
539+
outputTokens: 8,
540+
reasoningOutputTokens: 1,
541+
totalTokens: 50,
542+
}),
543+
]);
544+
});
487545
});
488546
}

0 commit comments

Comments
 (0)