Skip to content

Commit 2289524

Browse files
committed
perf(server): the first usage scan skips decoding transcript lines that carry no usage
The cold usage scan decoded every transcript line to a string before the substring gate threw most of them away. The gate now runs on the raw line bytes, so only lines that reach a parser are decoded. The Codex reducer markers move into the same gate and are matched in one pass instead of four. On a 30-day window (warm file cache): Claude 288 MB ~555ms -> ~335ms, Codex 824 MB ~1,780ms -> ~980ms. Parse results are identical for every file.
1 parent a6ec88f commit 2289524

4 files changed

Lines changed: 109 additions & 28 deletions

File tree

‎apps/server/src/usage/usageTranscriptReader.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,29 @@ describe("readTranscriptRecords resume", () => {
244244
);
245245
});
246246

247+
it("keeps a CRLF Codex reducer line whose marker straddles a stream chunk", async () => {
248+
const path = NodePath.join(dir, "rollout.jsonl");
249+
const crlf = (line: string) => line.replace("\n", "\r\n");
250+
const modelLine = crlf(codexModelLine("gpt-5.2-codex"));
251+
// Multi-byte tool output that fails the gate, sized so the 256 KiB chunk
252+
// boundary falls inside the model line's `"turn_context"` marker.
253+
const head = crlf(codexMetaLine());
254+
const fillerBytes =
255+
256 * 1024 - Buffer.byteLength(head) - modelLine.indexOf("turn_context") - 4;
256+
const fillerEnvelope = crlf(`${JSON.stringify({ type: "response_item", output: "" })}\n`);
257+
const padding = "é".repeat((fillerBytes - Buffer.byteLength(fillerEnvelope)) / 2);
258+
const filler = crlf(`${JSON.stringify({ type: "response_item", output: padding })}\n`);
259+
const content = head + filler + modelLine + crlf(codexUsageLine(40, 2));
260+
await NodeFSP.writeFile(path, content);
261+
262+
const result = await readTranscriptRecords(path, "codex");
263+
assert.isNotNull(result);
264+
assert.strictEqual(result.records.length, 1);
265+
assert.strictEqual(result.records[0]?.model, "gpt-5.2-codex");
266+
assert.strictEqual(result.records[0]?.sessionId, "codex-session-1");
267+
assert.strictEqual(result.position.resumeOffset, Buffer.byteLength(content));
268+
});
269+
247270
it("returns null for an unreadable file", async () => {
248271
assert.isNull(await readTranscriptRecords(NodePath.join(dir, "missing.jsonl"), "claude"));
249272
});

‎apps/server/src/usage/usageTranscriptReader.ts‎

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -291,21 +291,24 @@ export async function readTranscriptRecords(
291291
resumed = true;
292292
}
293293

294-
const parseLine = (line: string, state: CodexScanState, out: UsageRecord[]): void => {
294+
const toLineString = (lineBuffer: Buffer): string => {
295+
const content =
296+
lineBuffer.length > 0 && lineBuffer[lineBuffer.length - 1] === CARRIAGE_RETURN
297+
? lineBuffer.subarray(0, -1)
298+
: lineBuffer;
299+
return content.toString("utf8");
300+
};
301+
302+
// Gate on the raw bytes so the lines that fail it, which is most of them,
303+
// are never decoded.
304+
const parseLine = (lineBuffer: Buffer, state: CodexScanState, out: UsageRecord[]): void => {
305+
if (!mightCarryUsage(lineBuffer, provider)) return;
306+
const line = toLineString(lineBuffer);
295307
if (provider === "codex") {
296-
if (
297-
!mightCarryUsage(line, provider) &&
298-
!line.includes('"turn_context"') &&
299-
!line.includes('"thread_settings_applied"') &&
300-
!line.includes('"session_meta"')
301-
) {
302-
return;
303-
}
304308
const record = parseCodexLine(line, state);
305309
if (record !== null) out.push(record);
306310
return;
307311
}
308-
if (!mightCarryUsage(line, provider)) return;
309312
if (provider === "grok") {
310313
for (const grokRecord of parseGrokLine(line)) out.push(grokRecord);
311314
return;
@@ -314,14 +317,6 @@ export async function readTranscriptRecords(
314317
if (record !== null) out.push(record);
315318
};
316319

317-
const toLineString = (lineBuffer: Buffer): string => {
318-
const content =
319-
lineBuffer.length > 0 && lineBuffer[lineBuffer.length - 1] === CARRIAGE_RETURN
320-
? lineBuffer.subarray(0, -1)
321-
: lineBuffer;
322-
return content.toString("utf8");
323-
};
324-
325320
const records: UsageRecord[] = [];
326321
// Byte offsets remain independent of UTF-8 decoding. Only complete lines
327322
// commit the resume point; an unfinished tail is replayed on the next scan.
@@ -368,7 +363,7 @@ export async function readTranscriptRecords(
368363
pendingChunks.length === 1
369364
? pendingChunks[0]!
370365
: Buffer.concat(pendingChunks, pendingBytes);
371-
parseLine(toLineString(line), state, out);
366+
parseLine(line, state, out);
372367
}
373368
pendingChunks = [];
374369
pendingBytes = 0;
@@ -391,7 +386,7 @@ export async function readTranscriptRecords(
391386
// Most lines fit in the current chunk. Avoid buffering/streaming
392387
// machinery on this hot path.
393388
if (!streaming && pendingBytes === 0) {
394-
parseLine(toLineString(chunk.subarray(lineStart, newlineIndex)), codexState, records);
389+
parseLine(chunk.subarray(lineStart, newlineIndex), codexState, records);
395390
} else {
396391
append(chunk.subarray(lineStart, newlineIndex));
397392
finish(codexState, records);

‎apps/server/src/usage/usageTranscripts.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest";
33
import {
44
GROK_COST_USD_TICKS_PER_DOLLAR,
55
initialCodexScanState,
6+
mightCarryUsage,
67
parseClaudeLine,
78
parseCodexLine,
89
parseGrokLine,
@@ -38,6 +39,38 @@ function claudeLine(overrides: {
3839
});
3940
}
4041

42+
describe("mightCarryUsage", () => {
43+
const passes = (line: string | Buffer, provider: "claude" | "codex" | "grok") =>
44+
mightCarryUsage(Buffer.from(line), provider);
45+
46+
it("passes only lines carrying a provider's own markers", () => {
47+
expect(passes('{"message":{"usage":{}}}', "claude")).toBe(true);
48+
expect(passes('{"type":"token_count"}', "claude")).toBe(false);
49+
expect(passes('{"update":{"sessionUpdate":"turn_completed"}}', "grok")).toBe(true);
50+
expect(passes('{"message":{"usage":{}}}', "grok")).toBe(false);
51+
expect(passes('{"usage":{},"sessionUpdate":"turn_completed"}', "codex")).toBe(false);
52+
});
53+
54+
it("passes the Codex lines that feed the reducer without carrying usage", () => {
55+
for (const type of ["token_count", "turn_context", "thread_settings_applied", "session_meta"]) {
56+
expect(passes(`{"payload":{"type":"${type}"}}`, "codex")).toBe(true);
57+
}
58+
expect(passes('{"payload":{"type":"token_counts"}}', "codex")).toBe(false);
59+
});
60+
61+
it("matches markers beside multi-byte and malformed UTF-8", () => {
62+
const invalid = Buffer.from([0xff, 0xc3]);
63+
for (const provider of ["claude", "codex", "grok"] as const) {
64+
const marker = { claude: '"usage"', codex: '"session_meta"', grok: '"turn_completed"' }[
65+
provider
66+
];
67+
expect(passes(`{"text":"日本語 é",${marker}:1}`, provider)).toBe(true);
68+
expect(passes(Buffer.concat([invalid, Buffer.from(marker), invalid]), provider)).toBe(true);
69+
expect(passes(`{"text":"日本語 é ${marker.slice(1, -1)}"}`, provider)).toBe(false);
70+
}
71+
});
72+
});
73+
4174
describe("parseClaudeLine", () => {
4275
it("extracts token totals and a dedupe key", () => {
4376
const record = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" }));

‎apps/server/src/usage/usageTranscripts.ts‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,46 @@ export function totalTokens(totals: UsageTokenTotals): number {
6565
}
6666

6767
/**
68-
* Cheap substring gate applied before `JSON.parse`.
68+
* Builds a byte-level substring test for a set of ASCII needles.
6969
*
70-
* Transcripts are mostly tool output; only a minority of lines carry usage. On
71-
* a 30-day window this skips roughly half the lines outright and is worth about
72-
* an order of magnitude.
70+
* One needle is a single `Buffer` search. Several would be one pass over the
71+
* line each, so they are matched together as one alternation over the line's
72+
* latin1 view instead: latin1 maps every byte to one character without
73+
* validating it, so an ASCII needle matches exactly where its bytes do.
7374
*/
74-
export function mightCarryUsage(line: string, provider: UsageProviderKind): boolean {
75-
if (provider === "claude") return line.includes('"usage"');
76-
if (provider === "grok") return line.includes('"turn_completed"');
77-
return line.includes('"token_count"');
75+
function lineGate(...needles: readonly string[]): (line: Buffer) => boolean {
76+
if (needles.length === 1) {
77+
const needle = Buffer.from(needles[0]!);
78+
return (line) => line.includes(needle);
79+
}
80+
const pattern = new RegExp(
81+
needles.map((needle) => needle.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|"),
82+
);
83+
return (line) => pattern.test(line.toString("latin1"));
84+
}
85+
86+
const CLAUDE_GATE = lineGate('"usage"');
87+
const GROK_GATE = lineGate('"turn_completed"');
88+
// `token_count` carries the usage. The other three hold none of their own but
89+
// feed the Codex reducer: active model, service tier, and session identity.
90+
const CODEX_GATE = lineGate(
91+
'"token_count"',
92+
'"turn_context"',
93+
'"thread_settings_applied"',
94+
'"session_meta"',
95+
);
96+
97+
/**
98+
* Cheap substring gate applied to a line's raw bytes before it is decoded and
99+
* handed to `JSON.parse`. Only lines that pass need to reach a parser.
100+
*
101+
* Transcripts are mostly tool output; only a minority of lines carry usage, and
102+
* decoding the rest as UTF-8 just to discard them dominates a cold scan.
103+
*/
104+
export function mightCarryUsage(line: Buffer, provider: UsageProviderKind): boolean {
105+
if (provider === "claude") return CLAUDE_GATE(line);
106+
if (provider === "grok") return GROK_GATE(line);
107+
return CODEX_GATE(line);
78108
}
79109

80110
/**

0 commit comments

Comments
 (0)