Skip to content

Commit 729f675

Browse files
rfrost-xyzjuliusmarmingeclaude
authored
fix(usage): Claude subagent output is no longer undercounted (pingdotgg#20276)
Co-authored-by: Julius Marminge <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent b5256ad commit 729f675

7 files changed

Lines changed: 251 additions & 50 deletions

File tree

‎apps/server/src/provider/Drivers/claudeUsage.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ import * as HostProcess from "@t3tools/shared/HostProcess";
1616
export const CLAUDE_SYNTHETIC_MODEL = "<synthetic>";
1717

1818
/**
19-
* T3 Code writes one record per assistant *content block*, and every one of those records repeats
20-
* the same complete `usage` object for the parent message. Summing them overcounts by roughly
21-
* 2.4x on a real workload, so the caller must drop repeats by `dedupeKey` and keep the first.
19+
* Claude Code repeats a message's `dedupeKey` for each assistant content block.
20+
* Subagent blocks can progress from stream-start to final usage, so callers
21+
* must count each key once using the maximum of each token channel.
2222
*/
2323
export function parseClaudeLine(line: string): readonly UsageRecord[] {
2424
let parsed: unknown;

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,42 @@ function claudeLine(
9393
})}\n`;
9494
}
9595

96+
/** One content-block line of message `msg_1`; Claude Code writes several per message. */
97+
function claudeCopy(usage: Readonly<Record<string, number>>): string {
98+
return `${JSON.stringify({
99+
type: "assistant",
100+
timestamp: "2026-08-01T10:00:00Z",
101+
requestId: "req_1",
102+
sessionId: "session-1",
103+
message: { id: "msg_1", model: "claude-fable-5", usage },
104+
})}\n`;
105+
}
106+
107+
// A subagent message: its stream-start copy, then its final copy.
108+
const PARTIAL_USAGE = {
109+
input_tokens: 2,
110+
cache_creation_input_tokens: 1097,
111+
cache_read_input_tokens: 67934,
112+
output_tokens: 3,
113+
};
114+
const COMPLETE_USAGE = { ...PARTIAL_USAGE, output_tokens: 264 };
115+
const COPY_RATES = {
116+
"claude-fable-5": {
117+
input_cost_per_token: 1e-5,
118+
output_cost_per_token: 5e-5,
119+
cache_read_input_token_cost: 1e-6,
120+
cache_creation_input_token_cost: 1.25e-5,
121+
},
122+
};
123+
const COMPLETE_COST_USD = 2 * 1e-5 + 67934 * 1e-6 + 1097 * 1.25e-5 + 264 * 5e-5;
124+
const COMPLETE_TOTALS = {
125+
uncachedInputTokens: 2,
126+
cachedInputTokens: 67934,
127+
cacheCreationTokens: 1097,
128+
outputTokens: 264,
129+
reasoningTokens: 0,
130+
};
131+
96132
const WINDOW: UsageSummaryInput = {
97133
timeZone: "UTC",
98134
sinceDay: UsageDay.make("2026-07-31"),
@@ -2531,6 +2567,65 @@ describe("UsageService", () => {
25312567
}).pipe(Effect.scoped),
25322568
);
25332569

2570+
it.live(
2571+
"merges partial Claude copies through tail replacement, newline completion and restart",
2572+
() =>
2573+
Effect.gen(function* () {
2574+
const { transcript, settings, home } = yield* setup;
2575+
const partial = claudeCopy(PARTIAL_USAGE);
2576+
const complete = claudeCopy(COMPLETE_USAGE);
2577+
yield* Effect.promise(() => NodeFSP.writeFile(transcript, partial));
2578+
yield* Effect.gen(function* () {
2579+
const first = yield* UsageService.make;
2580+
assert.strictEqual(totalOutputTokens(yield* first.readSummary(WINDOW)), 3);
2581+
yield* first.awaitPersisted;
2582+
2583+
yield* Effect.promise(() => NodeFSP.appendFile(transcript, complete.trimEnd()));
2584+
const resumed = yield* UsageService.make;
2585+
const tail = yield* resumed.readSummary(WINDOW);
2586+
assert.deepStrictEqual(tail.buckets[0]?.totals, COMPLETE_TOTALS);
2587+
assert.strictEqual(tail.buckets[0]?.records, 1);
2588+
assert.closeTo(tail.buckets[0]?.costUsd ?? 0, COMPLETE_COST_USD, 1e-12);
2589+
yield* resumed.awaitPersisted;
2590+
const restarted = yield* UsageService.make;
2591+
assert.deepStrictEqual((yield* restarted.readSummary(WINDOW)).buckets, tail.buckets);
2592+
2593+
// Grow the file without changing its settled prefix: the old tail must be discarded.
2594+
yield* Effect.promise(() =>
2595+
NodeFSP.writeFile(transcript, partial + claudeLine(2, 7) + claudeLine(3, 5)),
2596+
);
2597+
const replaced = yield* restarted.readSummary(WINDOW);
2598+
assert.strictEqual(totalOutputTokens(replaced), 3 + 7 + 5);
2599+
assert.strictEqual(replaced.buckets[0]?.records, 3);
2600+
2601+
yield* Effect.promise(() => NodeFSP.appendFile(transcript, complete.trimEnd()));
2602+
const grown = yield* restarted.readSummary(WINDOW);
2603+
assert.deepStrictEqual(grown.buckets[0]?.totals, {
2604+
...COMPLETE_TOTALS,
2605+
uncachedInputTokens: 22,
2606+
outputTokens: 264 + 7 + 5,
2607+
});
2608+
assert.strictEqual(grown.buckets[0]?.records, 3);
2609+
yield* Effect.promise(() => NodeFSP.appendFile(transcript, "\n"));
2610+
assert.deepStrictEqual((yield* restarted.readSummary(WINDOW)).buckets, grown.buckets);
2611+
yield* restarted.awaitPersisted;
2612+
assert.deepStrictEqual(
2613+
(yield* (yield* UsageService.make).readSummary(WINDOW)).buckets,
2614+
grown.buckets,
2615+
);
2616+
}).pipe(
2617+
Effect.provide(
2618+
layerService({
2619+
prefix: "usage-service-copy-lifecycle-test",
2620+
home,
2621+
settings,
2622+
ratesDocument: COPY_RATES,
2623+
}),
2624+
),
2625+
);
2626+
}).pipe(Effect.scoped),
2627+
);
2628+
25342629
it.live(
25352630
"keeps large-record totals and costs exact through append, dedupe, restart and cleanup",
25362631
() =>
@@ -2687,7 +2782,7 @@ describe("UsageService", () => {
26872782
60 + 20 * (version === 4 ? 1 : 2),
26882783
);
26892784
yield* upgraded.awaitPersisted;
2690-
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v11.json");
2785+
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v12.json");
26912786
assert.deepStrictEqual(
26922787
(yield* Effect.promise(() => NodeFSP.readdir(stateDir)))
26932788
.filter((name) => name.startsWith("usage-scan-cache"))

‎apps/server/src/usage/UsageService.ts‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ export const make = Effect.gen(function* () {
473473
/**
474474
* Reads run concurrently, and the caller stores updates in walk order rather
475475
* than completion order: saved records of deleted transcripts aggregate in
476-
* cache order, where the first copy of a duplicate wins. Untraced: a scan
476+
* cache order, where the first copy of a duplicate decides its bucket. Untraced: a scan
477477
* reads thousands of files, so scanTranscriptDir records the counts instead.
478478
*/
479479
const readFileRecords = Effect.fnUntraced(function* (
@@ -551,9 +551,8 @@ export const make = Effect.gen(function* () {
551551
}
552552
}
553553
const base = parsed.resumed && cached !== undefined ? cached.records : [];
554-
const seen = new Set<string>();
555-
const records = dedupeWithinFile([...base, ...parsed.records], seen);
556-
const tailRecords = dedupeWithinFile(parsed.tailRecords, seen);
554+
const records = dedupeWithinFile([...base, ...parsed.records]);
555+
const tailRecords = parsed.tailRecords;
557556

558557
return {
559558
records: tailRecords.length === 0 ? records : [...records, ...tailRecords],
@@ -686,7 +685,7 @@ export const make = Effect.gen(function* () {
686685
{ concurrency: "unbounded" },
687686
);
688687

689-
// Scan independent sources together, directories sequentially to bound open files; preserve order for first-copy dedupe.
688+
// Scan independent sources together, directories sequentially to bound open files; keep order, since the first copy decides a duplicate's bucket.
690689
const [transcripts, scanDirs] = yield* Effect.all(
691690
[Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs)), scans],
692691
{ concurrency: "unbounded" },

‎apps/server/src/usage/usageAggregation.test.ts‎

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,17 +85,68 @@ describe("UsageAggregator", () => {
8585
expect(bucket).not.toHaveProperty("ultrafastCostUsd");
8686
});
8787

88-
it("keeps only the first record for a repeated dedupe key", () => {
88+
it("merges partial copies of a key by channel maximum in the first copy's bucket", () => {
89+
const zeroTotals = {
90+
uncachedInputTokens: 0,
91+
cachedInputTokens: 0,
92+
cacheCreationTokens: 0,
93+
outputTokens: 0,
94+
reasoningTokens: 0,
95+
};
96+
const partial = record({
97+
dedupeKey: "msg_1:",
98+
speed: "fast",
99+
reportedCostUsd: 1,
100+
totals: { ...record().totals, cachedInputTokens: 1200, outputTokens: 3, reasoningTokens: 5 },
101+
});
102+
const complete = record({
103+
dedupeKey: "msg_1:",
104+
model: "claude-fable-5[1m]",
105+
timestampMs: Date.parse("2026-08-08T12:00:00Z"),
106+
sessionId: "session-fork",
107+
reportedCostUsd: 2,
108+
totals: {
109+
uncachedInputTokens: 2,
110+
cachedInputTokens: 1000,
111+
cacheCreationTokens: 40,
112+
outputTokens: 264,
113+
reasoningTokens: 135,
114+
},
115+
});
116+
const zero = { ...partial, totals: zeroTotals, reportedCostUsd: null };
117+
const merged = {
118+
reportedCostUsd: 2,
119+
totals: { ...complete.totals, uncachedInputTokens: 100, cachedInputTokens: 1200 },
120+
};
121+
122+
for (const copies of [
123+
[zero, partial, complete, complete],
124+
[complete, partial, zero, zero],
125+
]) {
126+
const aggregator = new UsageAggregator({
127+
timeZone: "UTC",
128+
sinceDay: "2026-08-01",
129+
untilDay: "2026-08-31",
130+
rates,
131+
});
132+
copies.forEach((copy, index) => aggregator.add(copy, `${index}.jsonl`));
133+
const result = aggregator.finish();
134+
const expected = aggregate([{ ...copies[0]!, ...merged }]).buckets[0];
135+
136+
expect(result.duplicatesDropped).toBe(3);
137+
expect(result.buckets).toEqual([{ ...expected, sourcePath: "0.jsonl" }]);
138+
}
139+
});
140+
141+
it("does not count a key whose first copy fell outside the window", () => {
89142
const result = aggregate([
90-
record({ dedupeKey: "msg_1:" }),
91-
record({ dedupeKey: "msg_1:" }),
92-
record({ dedupeKey: "msg_1:" }),
143+
record({ dedupeKey: "msg_1:", timestampMs: Date.parse("2026-07-01T12:00:00Z") }),
144+
record({ dedupeKey: "msg_1:", totals: { ...record().totals, outputTokens: 500 } }),
93145
]);
94146

95-
expect(result.duplicatesDropped).toBe(2);
96-
expect(result.buckets).toHaveLength(1);
97-
expect(result.buckets[0]?.records).toBe(1);
98-
expect(result.buckets[0]?.totals.outputTokens).toBe(50);
147+
expect(result.duplicatesDropped).toBe(1);
148+
expect(result.outOfWindow).toBe(1);
149+
expect(result.buckets).toHaveLength(0);
99150
});
100151

101152
it("buckets by the day in the requested time zone", () => {

‎apps/server/src/usage/usageAggregation.ts‎

Lines changed: 66 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -129,10 +129,50 @@ export interface AggregateResult {
129129
readonly outOfWindow: number;
130130
}
131131

132+
/**
133+
* Partial copies share a billed message's key: keep each token channel's maximum and the larger
134+
* reported cost, and the first copy's time, model, session and speed so it stays in its bucket.
135+
*/
136+
export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRecord {
137+
const a = held.totals;
138+
const b = copy.totals;
139+
const reportedCostUsd =
140+
held.reportedCostUsd === null || copy.reportedCostUsd === null
141+
? (held.reportedCostUsd ?? copy.reportedCostUsd)
142+
: Math.max(held.reportedCostUsd, copy.reportedCostUsd);
143+
if (
144+
b.uncachedInputTokens <= a.uncachedInputTokens &&
145+
b.cachedInputTokens <= a.cachedInputTokens &&
146+
b.cacheCreationTokens <= a.cacheCreationTokens &&
147+
b.outputTokens <= a.outputTokens &&
148+
b.reasoningTokens <= a.reasoningTokens &&
149+
reportedCostUsd === held.reportedCostUsd
150+
) {
151+
return held;
152+
}
153+
return {
154+
...held,
155+
totals: {
156+
uncachedInputTokens: Math.max(a.uncachedInputTokens, b.uncachedInputTokens),
157+
cachedInputTokens: Math.max(a.cachedInputTokens, b.cachedInputTokens),
158+
cacheCreationTokens: Math.max(a.cacheCreationTokens, b.cacheCreationTokens),
159+
outputTokens: Math.max(a.outputTokens, b.outputTokens),
160+
reasoningTokens: Math.max(a.reasoningTokens, b.reasoningTokens),
161+
},
162+
reportedCostUsd,
163+
};
164+
}
165+
166+
interface Contribution {
167+
record: UsageRecord;
168+
/** Null when the first copy fell outside the window. */
169+
readonly bucket: MutableBucket | null;
170+
}
171+
132172
/** Deduplicates across the whole scan; Claude copies records into resumed/forked transcripts. */
133173
export class UsageAggregator {
134174
readonly #buckets = new Map<string, MutableBucket>();
135-
readonly #seen = new Set<string>();
175+
readonly #seen = new Map<string, Contribution>();
136176
readonly #toDay: (timestampMs: number) => string;
137177
readonly #hourlyWindow: { readonly sinceTimeMs: number; readonly untilTimeMs: number } | null;
138178
readonly #options: AggregateOptions;
@@ -167,37 +207,51 @@ export class UsageAggregator {
167207
if (!keepsUsageRecord(this.#options.modelFilter, input)) return false;
168208
const record = this.#mapModel(input);
169209
if (record.dedupeKey !== null) {
170-
if (this.#seen.has(record.dedupeKey)) {
210+
const held = this.#seen.get(record.dedupeKey);
211+
if (held !== undefined) {
171212
this.#duplicatesDropped += 1;
213+
if (held.bucket !== null) held.record = mergeUsageCopies(held.record, record);
172214
return false;
173215
}
174-
this.#seen.add(record.dedupeKey);
175216
}
176217

218+
const bucket = this.#placeInWindow(record, sourcePath ?? "");
219+
if (record.dedupeKey !== null) this.#seen.set(record.dedupeKey, { record, bucket });
220+
if (bucket === null) {
221+
this.#outOfWindow += 1;
222+
return false;
223+
}
224+
// Keyed usage is counted in finish(), once all its copies are merged.
225+
if (record.dedupeKey === null) this.#accumulate(bucket, record);
226+
if (record.sessionId.length > 0) bucket.sessions.add(record.sessionId);
227+
return true;
228+
}
229+
230+
#placeInWindow(record: UsageRecord, source: string): MutableBucket | null {
177231
if (
178232
this.#hourlyWindow !== null &&
179233
(record.timestampMs < this.#hourlyWindow.sinceTimeMs ||
180234
record.timestampMs >= this.#hourlyWindow.untilTimeMs)
181235
) {
182-
this.#outOfWindow += 1;
183-
return false;
236+
return null;
184237
}
185238

186239
const day = this.#toDay(record.timestampMs);
187240
if (
188241
this.#hourlyWindow === null &&
189242
(day < this.#options.sinceDay || day > this.#options.untilDay)
190243
) {
191-
this.#outOfWindow += 1;
192-
return false;
244+
return null;
193245
}
194246

195247
const hourIndex =
196248
this.#hourlyWindow === null
197249
? -1
198250
: Math.floor((record.timestampMs - this.#hourlyWindow.sinceTimeMs) / HOUR_MS);
199-
const bucket = this.#bucketFor(day, hourIndex, record.provider, record.model, sourcePath ?? "");
251+
return this.#bucketFor(day, hourIndex, record.provider, record.model, source);
252+
}
200253

254+
#accumulate(bucket: MutableBucket, record: UsageRecord): void {
201255
const priced = priceUsage(this.#options.rates, record, this.#options.priceOverrides);
202256

203257
const totals = bucket.totals;
@@ -231,8 +285,6 @@ export class UsageAggregator {
231285
bucket.records += 1;
232286
if (priced.costSource === "unpriced") bucket.unpricedRecords += 1;
233287
if (priced.costSource === "providerReported") bucket.providerReportedRecords += 1;
234-
if (record.sessionId.length > 0) bucket.sessions.add(record.sessionId);
235-
return true;
236288
}
237289

238290
// Aliases use the target's rate, replacing provider-specific rateModel.
@@ -288,6 +340,10 @@ export class UsageAggregator {
288340
}
289341

290342
finish(): AggregateResult {
343+
for (const { record, bucket } of this.#seen.values()) {
344+
if (bucket !== null) this.#accumulate(bucket, record);
345+
}
346+
this.#seen.clear();
291347
const buckets: UsageBucket[] = [];
292348
for (const [key, bucket] of this.#buckets) {
293349
const [day = "", hourStart = "", provider = "", model = "", sourcePath = ""] =

0 commit comments

Comments
 (0)