Skip to content

Commit fa9b4e2

Browse files
axisrowclaude
andcommitted
fix(jsonl): dedup cumulative token totals by message.id (#8, sirmalloc#549)
Claude Code writes one JSONL entry per content block of a single API call, all sharing one message.id. The cumulative token path summed every entry, inflating Tokens Input/Output/Total by up to ~1.84x. Mirror the grouping already used for Tokens Last Turn: consecutive counted entries sharing an id contribute one usage (prompt side once, output as the running max). - transcripts without duplicate-id entries are unchanged (existing tests) - streaming partials resolve to the finalized value, counted once mid-stream - sidechain/API-error rows keep their current treatment in sums and recency Added vitest cases on a synthetic sirmalloc#549-shaped transcript asserting exact cumulative totals (12 in / 292 out / 16660 total vs inflated 16/393/...), streaming partials with ids, and a sidechain per-content-block group. Co-Authored-By: Claude Code <[email protected]>
1 parent 51a8819 commit fa9b4e2

2 files changed

Lines changed: 215 additions & 10 deletions

File tree

‎src/utils/__tests__/jsonl-metrics.test.ts‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -756,6 +756,180 @@ describe('jsonl transcript metrics', () => {
756756
});
757757
});
758758

759+
it('deduplicates cumulative token totals across one API call’s per-content-block entries', async () => {
760+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-metrics-'));
761+
tempRoots.push(root);
762+
const transcriptPath = path.join(root, 'cumulative-dedup.jsonl');
763+
764+
// The same #549-shaped transcript as the last-turn dedup case above:
765+
// msg_01 writes three per-content-block entries with identical
766+
// prompt-side usage and growing output_tokens, so the cumulative
767+
// widgets must count the call once, not once per block.
768+
fs.writeFileSync(transcriptPath, [
769+
makeUsageLine({
770+
timestamp: '2026-01-01T10:00:00.000Z',
771+
input: 10,
772+
output: 5,
773+
cacheRead: 100,
774+
cacheCreate: 0,
775+
stopReason: 'end_turn',
776+
messageId: 'msg_00'
777+
}),
778+
makeUsageLine({
779+
timestamp: '2026-01-01T10:01:00.000Z',
780+
input: 2,
781+
output: 1,
782+
cacheRead: 3978,
783+
cacheCreate: 12278,
784+
stopReason: 'tool_use',
785+
messageId: 'msg_01'
786+
}),
787+
makeUsageLine({
788+
timestamp: '2026-01-01T10:01:00.000Z',
789+
input: 2,
790+
output: 100,
791+
cacheRead: 3978,
792+
cacheCreate: 12278,
793+
stopReason: 'tool_use',
794+
messageId: 'msg_01'
795+
}),
796+
makeUsageLine({
797+
timestamp: '2026-01-01T10:01:01.000Z',
798+
input: 2,
799+
output: 287,
800+
cacheRead: 3978,
801+
cacheCreate: 12278,
802+
stopReason: 'tool_use',
803+
messageId: 'msg_01'
804+
})
805+
].join('\n'));
806+
807+
const metrics = await getTokenMetrics(transcriptPath);
808+
809+
expect(metrics).toEqual({
810+
inputTokens: 12, // 10 + 2 (msg_01 counted once)
811+
outputTokens: 292, // 5 + 287 (finalized output wins)
812+
cachedTokens: 16356, // (100 + 0) + (3978 + 12278)
813+
cacheReadTokens: 4078, // 100 + 3978
814+
cacheCreationTokens: 12278, // 0 + 12278
815+
totalTokens: 16660, // 12 + 292 + 16356
816+
contextLength: 16258 // 2 + 3978 + 12278
817+
});
818+
});
819+
820+
it('resolves streaming partials to their finalized values in cumulative totals', async () => {
821+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-metrics-'));
822+
tempRoots.push(root);
823+
const transcriptPath = path.join(root, 'cumulative-streaming-dedup.jsonl');
824+
825+
// Same shape as the live-update case, but every entry carries a
826+
// message.id: msg_a's partials must collapse into its finalized entry,
827+
// and msg_b's in-flight partials must count once, not per partial.
828+
fs.writeFileSync(transcriptPath, [
829+
makeUsageLine({
830+
timestamp: '2026-01-01T10:00:00.000Z',
831+
input: 2,
832+
output: 25,
833+
cacheRead: 100,
834+
cacheCreate: 50,
835+
stopReason: null,
836+
messageId: 'msg_a'
837+
}),
838+
makeUsageLine({
839+
timestamp: '2026-01-01T10:00:01.000Z',
840+
input: 2,
841+
output: 80,
842+
cacheRead: 100,
843+
cacheCreate: 50,
844+
stopReason: 'end_turn',
845+
messageId: 'msg_a'
846+
}),
847+
makeUsageLine({
848+
timestamp: '2026-01-01T10:00:02.000Z',
849+
input: 3,
850+
output: 30,
851+
cacheRead: 200,
852+
cacheCreate: 25,
853+
stopReason: null,
854+
messageId: 'msg_b'
855+
}),
856+
makeUsageLine({
857+
timestamp: '2026-01-01T10:00:03.000Z',
858+
input: 3,
859+
output: 120,
860+
cacheRead: 200,
861+
cacheCreate: 25,
862+
stopReason: null,
863+
messageId: 'msg_b'
864+
})
865+
].join('\n'));
866+
867+
const metrics = await getTokenMetrics(transcriptPath);
868+
869+
expect(metrics).toEqual({
870+
inputTokens: 5, // 2 + 3, one call each
871+
outputTokens: 200, // 80 + 120, not 25+80 or 30+120
872+
cachedTokens: 375,
873+
cacheReadTokens: 300,
874+
cacheCreationTokens: 75,
875+
totalTokens: 580,
876+
contextLength: 228
877+
});
878+
});
879+
880+
it('counts a sidechain call’s per-content-block entries once in cumulative totals', async () => {
881+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-metrics-'));
882+
tempRoots.push(root);
883+
const transcriptPath = path.join(root, 'cumulative-sidechain-dedup.jsonl');
884+
885+
// Sidechain rows keep today's treatment in the cumulative sums (still
886+
// included, still excluded from context length); grouping by their own
887+
// message.id must not count one subagent call per content block.
888+
fs.writeFileSync(transcriptPath, [
889+
makeUsageLine({
890+
timestamp: '2026-01-01T10:00:00.000Z',
891+
input: 10,
892+
output: 50,
893+
cacheRead: 20,
894+
cacheCreate: 10,
895+
stopReason: 'end_turn',
896+
messageId: 'msg_main'
897+
}),
898+
makeUsageLine({
899+
timestamp: '2026-01-01T10:01:00.000Z',
900+
input: 900,
901+
output: 400,
902+
cacheRead: 5000,
903+
cacheCreate: 100,
904+
stopReason: 'tool_use',
905+
messageId: 'msg_sub',
906+
isSidechain: true
907+
}),
908+
makeUsageLine({
909+
timestamp: '2026-01-01T10:01:01.000Z',
910+
input: 900,
911+
output: 650,
912+
cacheRead: 5000,
913+
cacheCreate: 100,
914+
stopReason: 'tool_use',
915+
messageId: 'msg_sub',
916+
isSidechain: true
917+
})
918+
].join('\n'));
919+
920+
const metrics = await getTokenMetrics(transcriptPath);
921+
922+
expect(metrics).toEqual({
923+
inputTokens: 910, // 10 + 900 (sidechain call counted once)
924+
outputTokens: 700, // 50 + 650
925+
cachedTokens: 5130, // (20 + 10) + (5000 + 100)
926+
cacheReadTokens: 5020, // 20 + 5000
927+
cacheCreationTokens: 110, // 10 + 100
928+
totalTokens: 6740, // 910 + 700 + 5130
929+
contextLength: 40 // main-chain msg_main only
930+
});
931+
});
932+
759933
it('does not report sidechain or API-error usage as the last turn', async () => {
760934
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ccstatusline-jsonl-metrics-'));
761935
tempRoots.push(root);

‎src/utils/jsonl-metrics.ts‎

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ interface TokenMetricEntry {
8888
stopReason: string | null | undefined;
8989
timestampMs: number | null;
9090
isMainChain: boolean;
91+
messageId: string | null;
9192
}
9293

9394
interface TokenMetricAccumulator {
@@ -104,6 +105,8 @@ interface TokenMetricAccumulator {
104105
interface TokenMetricState {
105106
metrics: TokenMetricAccumulator;
106107
hasStopReasonField: boolean;
108+
countedMessageId: string | null;
109+
countedOutputTokens: number;
107110
lastUsageEntry: TokenMetricEntry | null;
108111
sawCompactBoundary: boolean;
109112
boundaryAfterLastUsage: boolean;
@@ -173,6 +176,8 @@ function createTokenMetricState(): TokenMetricState {
173176
return {
174177
metrics: createTokenMetricAccumulator(),
175178
hasStopReasonField: false,
179+
countedMessageId: null,
180+
countedOutputTokens: 0,
176181
lastUsageEntry: null,
177182
sawCompactBoundary: false,
178183
boundaryAfterLastUsage: false,
@@ -195,20 +200,42 @@ function createLastTurnTokens(usage: UsageTokens): LastTurnTokens {
195200
// Claude Code writes one JSONL entry per content block (thinking / text / each
196201
// tool_use) of a single API call, all sharing one message.id and repeating
197202
// identical prompt-side usage; only output_tokens grows entry to entry.
198-
// Grouping by id keeps one usage per call instead of one per content block
199-
// (see sirmalloc/ccstatusline#549). Entries without an id each start a new group.
200-
function trackLastTurnTokens(state: TokenMetricState, entry: TokenMetricEntry, messageId: string | undefined): void {
203+
// Grouping consecutive counted entries by id keeps one usage per call instead
204+
// of one per content block (see sirmalloc/ccstatusline#549). Entries without
205+
// an id each start a new group.
206+
function accumulateCumulativeTokenEntry(
207+
state: TokenMetricState,
208+
entry: TokenMetricEntry,
209+
includePostCompactionUsage: boolean
210+
): void {
211+
if (state.countedMessageId !== null
212+
&& entry.messageId !== null
213+
&& entry.messageId === state.countedMessageId) {
214+
const outputDelta = entry.usage.output - state.countedOutputTokens;
215+
if (outputDelta > 0) {
216+
state.countedOutputTokens = entry.usage.output;
217+
state.metrics.outputTokens += outputDelta;
218+
}
219+
return;
220+
}
221+
222+
state.countedMessageId = entry.messageId;
223+
state.countedOutputTokens = entry.usage.output;
224+
accumulateTokenMetricEntry(state.metrics, entry, includePostCompactionUsage);
225+
}
226+
227+
function trackLastTurnTokens(state: TokenMetricState, entry: TokenMetricEntry): void {
228+
const { messageId } = entry;
201229
if (state.lastTurnTokens !== null
202-
&& typeof messageId === 'string'
203-
&& messageId.length > 0
230+
&& messageId !== null
204231
&& messageId === state.lastTurnMessageId) {
205232
const lastTurn = state.lastTurnTokens;
206233
lastTurn.outputTokens = Math.max(lastTurn.outputTokens, entry.usage.output);
207234
lastTurn.totalTokens = lastTurn.inputTokens + lastTurn.outputTokens + lastTurn.cachedTokens;
208235
return;
209236
}
210237

211-
state.lastTurnMessageId = typeof messageId === 'string' && messageId.length > 0 ? messageId : null;
238+
state.lastTurnMessageId = messageId;
212239
state.lastTurnTokens = createLastTurnTokens(entry.usage);
213240
}
214241

@@ -224,25 +251,29 @@ function collectTokenMetricRecord(state: TokenMetricState, data: TranscriptLine
224251
const message = data?.message;
225252
const usage = message?.usage;
226253
if (usage) {
254+
const rawMessageId = message.id;
227255
const entry: TokenMetricEntry = {
228256
usage: parseUsageTokens(usage),
229257
stopReason: message.stop_reason,
230258
timestampMs,
231-
isMainChain: data?.isSidechain !== true && !data?.isApiErrorMessage
259+
isMainChain: data?.isSidechain !== true && !data?.isApiErrorMessage,
260+
messageId: typeof rawMessageId === 'string' && rawMessageId.length > 0 ? rawMessageId : null
232261
};
233262

234263
const hasStopReason = Object.prototype.hasOwnProperty.call(message, 'stop_reason');
235264
if (hasStopReason && !state.hasStopReasonField) {
236265
state.hasStopReasonField = true;
237266
state.metrics = createTokenMetricAccumulator();
267+
state.countedMessageId = null;
268+
state.countedOutputTokens = 0;
238269
}
239270
if (!state.hasStopReasonField || entry.stopReason) {
240-
accumulateTokenMetricEntry(state.metrics, entry, !compactBoundary);
271+
accumulateCumulativeTokenEntry(state, entry, !compactBoundary);
241272
}
242273
// Sidechain (subagent) and API-error rows never form the user's turn,
243274
// matching the recency rule used for context length above.
244275
if (entry.isMainChain) {
245-
trackLastTurnTokens(state, entry, message.id);
276+
trackLastTurnTokens(state, entry);
246277
}
247278
state.lastUsageEntry = entry;
248279
state.boundaryAfterLastUsage = compactBoundary;
@@ -251,7 +282,7 @@ function collectTokenMetricRecord(state: TokenMetricState, data: TranscriptLine
251282

252283
function finishTokenMetrics(state: TokenMetricState, includeLastTurnTokens: boolean): TokenMetrics {
253284
if (state.hasStopReasonField && state.lastUsageEntry?.stopReason === null) {
254-
accumulateTokenMetricEntry(state.metrics, state.lastUsageEntry, !state.boundaryAfterLastUsage);
285+
accumulateCumulativeTokenEntry(state, state.lastUsageEntry, !state.boundaryAfterLastUsage);
255286
}
256287

257288
const contextLengthFromUsage = (usage: UsageTokens | null): number | null => usage

0 commit comments

Comments
 (0)