|
1 | 1 | import type { TokenUsageDelta, TokenUsageEvent } from './_types.ts'; |
2 | | -import { readFile, stat } from 'node:fs/promises'; |
| 2 | +import { createReadStream } from 'node:fs'; |
| 3 | +import { stat } from 'node:fs/promises'; |
3 | 4 | import path from 'node:path'; |
4 | 5 | import process from 'node:process'; |
| 6 | +import { createInterface } from 'node:readline'; |
5 | 7 | import { Result } from '@praha/byethrow'; |
6 | 8 | import { createFixture } from 'fs-fixture'; |
7 | 9 | import { glob } from 'tinyglobby'; |
@@ -225,136 +227,135 @@ export async function loadTokenUsageEvents(options: LoadOptions = {}): Promise<L |
225 | 227 | const relativeSessionPath = path.relative(directoryPath, file); |
226 | 228 | const normalizedSessionPath = relativeSessionPath.split(path.sep).join('/'); |
227 | 229 | const sessionId = normalizedSessionPath.replace(/\.jsonl$/i, ''); |
228 | | - const fileContentResult = await Result.try({ |
229 | | - try: readFile(file, 'utf8'), |
230 | | - catch: (error) => error, |
231 | | - }); |
232 | | - |
233 | | - if (Result.isFailure(fileContentResult)) { |
234 | | - logger.debug('Failed to read Codex session file', fileContentResult.error); |
235 | | - continue; |
236 | | - } |
237 | | - |
238 | 230 | let previousTotals: RawUsage | null = null; |
239 | 231 | let currentModel: string | undefined; |
240 | 232 | let currentModelIsFallback = false; |
241 | 233 | let legacyFallbackUsed = false; |
242 | | - const lines = fileContentResult.value.split(/\r?\n/); |
243 | | - for (const line of lines) { |
244 | | - const trimmed = line.trim(); |
245 | | - if (trimmed === '') { |
246 | | - continue; |
247 | | - } |
| 234 | + const lineReader = createInterface({ |
| 235 | + input: createReadStream(file, { encoding: 'utf8' }), |
| 236 | + crlfDelay: Number.POSITIVE_INFINITY, |
| 237 | + }); |
248 | 238 |
|
249 | | - const parseLine = Result.try({ |
250 | | - try: () => JSON.parse(trimmed) as unknown, |
251 | | - catch: (error) => error, |
252 | | - }); |
253 | | - const parsedResult = parseLine(); |
| 239 | + try { |
| 240 | + for await (const line of lineReader) { |
| 241 | + const trimmed = line.trim(); |
| 242 | + if (trimmed === '') { |
| 243 | + continue; |
| 244 | + } |
254 | 245 |
|
255 | | - if (Result.isFailure(parsedResult)) { |
256 | | - continue; |
257 | | - } |
| 246 | + const parseLine = Result.try({ |
| 247 | + try: () => JSON.parse(trimmed) as unknown, |
| 248 | + catch: (error) => error, |
| 249 | + }); |
| 250 | + const parsedResult = parseLine(); |
258 | 251 |
|
259 | | - const entryParse = v.safeParse(entrySchema, parsedResult.value); |
260 | | - if (!entryParse.success) { |
261 | | - continue; |
262 | | - } |
| 252 | + if (Result.isFailure(parsedResult)) { |
| 253 | + continue; |
| 254 | + } |
| 255 | + |
| 256 | + const entryParse = v.safeParse(entrySchema, parsedResult.value); |
| 257 | + if (!entryParse.success) { |
| 258 | + continue; |
| 259 | + } |
263 | 260 |
|
264 | | - const { type: entryType, payload, timestamp } = entryParse.output; |
| 261 | + const { type: entryType, payload, timestamp } = entryParse.output; |
265 | 262 |
|
266 | | - if (entryType === 'turn_context') { |
267 | | - const contextPayload = v.safeParse(recordSchema, payload ?? null); |
268 | | - if (contextPayload.success) { |
269 | | - const contextModel = extractModel(contextPayload.output); |
270 | | - if (contextModel != null) { |
271 | | - currentModel = contextModel; |
272 | | - currentModelIsFallback = false; |
| 263 | + if (entryType === 'turn_context') { |
| 264 | + const contextPayload = v.safeParse(recordSchema, payload ?? null); |
| 265 | + if (contextPayload.success) { |
| 266 | + const contextModel = extractModel(contextPayload.output); |
| 267 | + if (contextModel != null) { |
| 268 | + currentModel = contextModel; |
| 269 | + currentModelIsFallback = false; |
| 270 | + } |
273 | 271 | } |
| 272 | + continue; |
274 | 273 | } |
275 | | - continue; |
276 | | - } |
277 | 274 |
|
278 | | - if (entryType !== 'event_msg') { |
279 | | - continue; |
280 | | - } |
| 275 | + if (entryType !== 'event_msg') { |
| 276 | + continue; |
| 277 | + } |
281 | 278 |
|
282 | | - const tokenPayloadResult = v.safeParse(tokenCountPayloadSchema, payload ?? undefined); |
283 | | - if (!tokenPayloadResult.success) { |
284 | | - continue; |
285 | | - } |
| 279 | + const tokenPayloadResult = v.safeParse(tokenCountPayloadSchema, payload ?? undefined); |
| 280 | + if (!tokenPayloadResult.success) { |
| 281 | + continue; |
| 282 | + } |
286 | 283 |
|
287 | | - if (timestamp == null) { |
288 | | - continue; |
289 | | - } |
| 284 | + if (timestamp == null) { |
| 285 | + continue; |
| 286 | + } |
290 | 287 |
|
291 | | - const info = tokenPayloadResult.output.info; |
292 | | - const lastUsage = normalizeRawUsage(info?.last_token_usage); |
293 | | - const totalUsage = normalizeRawUsage(info?.total_token_usage); |
| 288 | + const info = tokenPayloadResult.output.info; |
| 289 | + const lastUsage = normalizeRawUsage(info?.last_token_usage); |
| 290 | + const totalUsage = normalizeRawUsage(info?.total_token_usage); |
294 | 291 |
|
295 | | - let raw = lastUsage; |
296 | | - if (raw == null && totalUsage != null) { |
297 | | - raw = subtractRawUsage(totalUsage, previousTotals); |
298 | | - } |
| 292 | + let raw = lastUsage; |
| 293 | + if (raw == null && totalUsage != null) { |
| 294 | + raw = subtractRawUsage(totalUsage, previousTotals); |
| 295 | + } |
299 | 296 |
|
300 | | - if (totalUsage != null) { |
301 | | - previousTotals = totalUsage; |
302 | | - } |
| 297 | + if (totalUsage != null) { |
| 298 | + previousTotals = totalUsage; |
| 299 | + } |
303 | 300 |
|
304 | | - if (raw == null) { |
305 | | - continue; |
306 | | - } |
| 301 | + if (raw == null) { |
| 302 | + continue; |
| 303 | + } |
307 | 304 |
|
308 | | - const delta = convertToDelta(raw); |
309 | | - if ( |
310 | | - delta.inputTokens === 0 && |
311 | | - delta.cachedInputTokens === 0 && |
312 | | - delta.outputTokens === 0 && |
313 | | - delta.reasoningOutputTokens === 0 |
314 | | - ) { |
315 | | - continue; |
316 | | - } |
| 305 | + const delta = convertToDelta(raw); |
| 306 | + if ( |
| 307 | + delta.inputTokens === 0 && |
| 308 | + delta.cachedInputTokens === 0 && |
| 309 | + delta.outputTokens === 0 && |
| 310 | + delta.reasoningOutputTokens === 0 |
| 311 | + ) { |
| 312 | + continue; |
| 313 | + } |
317 | 314 |
|
318 | | - const payloadRecordResult = v.safeParse(recordSchema, payload ?? undefined); |
319 | | - const extractionSource = payloadRecordResult.success |
320 | | - ? Object.assign({}, payloadRecordResult.output, { info }) |
321 | | - : { info }; |
322 | | - const extractedModel = extractModel(extractionSource); |
323 | | - let isFallbackModel = false; |
324 | | - if (extractedModel != null) { |
325 | | - currentModel = extractedModel; |
326 | | - currentModelIsFallback = false; |
327 | | - } |
| 315 | + const payloadRecordResult = v.safeParse(recordSchema, payload ?? undefined); |
| 316 | + const extractionSource = payloadRecordResult.success |
| 317 | + ? Object.assign({}, payloadRecordResult.output, { info }) |
| 318 | + : { info }; |
| 319 | + const extractedModel = extractModel(extractionSource); |
| 320 | + let isFallbackModel = false; |
| 321 | + if (extractedModel != null) { |
| 322 | + currentModel = extractedModel; |
| 323 | + currentModelIsFallback = false; |
| 324 | + } |
328 | 325 |
|
329 | | - let model = extractedModel ?? currentModel; |
330 | | - if (model == null) { |
331 | | - model = LEGACY_FALLBACK_MODEL; |
332 | | - isFallbackModel = true; |
333 | | - legacyFallbackUsed = true; |
334 | | - currentModel = model; |
335 | | - currentModelIsFallback = true; |
336 | | - } else if (extractedModel == null && currentModelIsFallback) { |
337 | | - isFallbackModel = true; |
338 | | - } |
| 326 | + let model = extractedModel ?? currentModel; |
| 327 | + if (model == null) { |
| 328 | + model = LEGACY_FALLBACK_MODEL; |
| 329 | + isFallbackModel = true; |
| 330 | + legacyFallbackUsed = true; |
| 331 | + currentModel = model; |
| 332 | + currentModelIsFallback = true; |
| 333 | + } else if (extractedModel == null && currentModelIsFallback) { |
| 334 | + isFallbackModel = true; |
| 335 | + } |
339 | 336 |
|
340 | | - const event: TokenUsageEvent = { |
341 | | - sessionId, |
342 | | - timestamp, |
343 | | - model, |
344 | | - inputTokens: delta.inputTokens, |
345 | | - cachedInputTokens: delta.cachedInputTokens, |
346 | | - outputTokens: delta.outputTokens, |
347 | | - reasoningOutputTokens: delta.reasoningOutputTokens, |
348 | | - totalTokens: delta.totalTokens, |
349 | | - }; |
350 | | - |
351 | | - if (isFallbackModel) { |
352 | | - // Surface the fallback so both table + JSON outputs can annotate pricing that was |
353 | | - // inferred rather than sourced from the log metadata. |
354 | | - event.isFallbackModel = true; |
355 | | - } |
| 337 | + const event: TokenUsageEvent = { |
| 338 | + sessionId, |
| 339 | + timestamp, |
| 340 | + model, |
| 341 | + inputTokens: delta.inputTokens, |
| 342 | + cachedInputTokens: delta.cachedInputTokens, |
| 343 | + outputTokens: delta.outputTokens, |
| 344 | + reasoningOutputTokens: delta.reasoningOutputTokens, |
| 345 | + totalTokens: delta.totalTokens, |
| 346 | + }; |
| 347 | + |
| 348 | + if (isFallbackModel) { |
| 349 | + // Surface the fallback so both table + JSON outputs can annotate pricing that was |
| 350 | + // inferred rather than sourced from the log metadata. |
| 351 | + event.isFallbackModel = true; |
| 352 | + } |
356 | 353 |
|
357 | | - events.push(event); |
| 354 | + events.push(event); |
| 355 | + } |
| 356 | + } catch (error) { |
| 357 | + logger.debug('Failed to read Codex session file', error); |
| 358 | + continue; |
358 | 359 | } |
359 | 360 |
|
360 | 361 | if (legacyFallbackUsed) { |
@@ -484,5 +485,62 @@ if (import.meta.vitest != null) { |
484 | 485 | expect(events[0]!.model).toBe('gpt-5'); |
485 | 486 | expect(events[0]!.isFallbackModel).toBe(true); |
486 | 487 | }); |
| 488 | + |
| 489 | + it('parses compacted-heavy sessions', async () => { |
| 490 | + const hugeCompactedPayload = 'x'.repeat(2_000_000); |
| 491 | + await using fixture = await createFixture({ |
| 492 | + sessions: { |
| 493 | + 'compacted.jsonl': [ |
| 494 | + JSON.stringify({ |
| 495 | + timestamp: '2025-09-15T13:00:00.000Z', |
| 496 | + type: 'compacted', |
| 497 | + payload: { |
| 498 | + replacement_history: hugeCompactedPayload, |
| 499 | + }, |
| 500 | + }), |
| 501 | + JSON.stringify({ |
| 502 | + timestamp: '2025-09-15T13:00:10.000Z', |
| 503 | + type: 'turn_context', |
| 504 | + payload: { |
| 505 | + model: 'gpt-5.4', |
| 506 | + }, |
| 507 | + }), |
| 508 | + JSON.stringify({ |
| 509 | + timestamp: '2025-09-15T13:00:11.000Z', |
| 510 | + type: 'event_msg', |
| 511 | + payload: { |
| 512 | + type: 'token_count', |
| 513 | + info: { |
| 514 | + last_token_usage: { |
| 515 | + input_tokens: 42, |
| 516 | + cached_input_tokens: 2, |
| 517 | + output_tokens: 8, |
| 518 | + reasoning_output_tokens: 1, |
| 519 | + total_tokens: 50, |
| 520 | + }, |
| 521 | + }, |
| 522 | + }, |
| 523 | + }), |
| 524 | + ].join('\n'), |
| 525 | + }, |
| 526 | + }); |
| 527 | + |
| 528 | + const { events } = await loadTokenUsageEvents({ |
| 529 | + sessionDirs: [fixture.getPath('sessions')], |
| 530 | + }); |
| 531 | + |
| 532 | + expect(events).toEqual([ |
| 533 | + expect.objectContaining({ |
| 534 | + sessionId: 'compacted', |
| 535 | + timestamp: '2025-09-15T13:00:11.000Z', |
| 536 | + model: 'gpt-5.4', |
| 537 | + inputTokens: 42, |
| 538 | + cachedInputTokens: 2, |
| 539 | + outputTokens: 8, |
| 540 | + reasoningOutputTokens: 1, |
| 541 | + totalTokens: 50, |
| 542 | + }), |
| 543 | + ]); |
| 544 | + }); |
487 | 545 | }); |
488 | 546 | } |
0 commit comments