Skip to content

Commit adba9a3

Browse files
authored
fix(usage): count Claude advisor calls under the advisor model (pingdotgg#24139)
1 parent 2c119bd commit adba9a3

5 files changed

Lines changed: 205 additions & 26 deletions

File tree

‎apps/server/src/provider/Drivers/claudeUsage.test.ts‎

Lines changed: 134 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ function claudeLine(overrides: {
3232

3333
describe("parseClaudeLine", () => {
3434
it("extracts token totals and a dedupe key", () => {
35-
const record = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" }));
35+
const [record] = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" }));
3636

37-
expect(record).not.toBeNull();
37+
expect(record).toBeDefined();
3838
expect(record?.provider).toBe("claude");
3939
expect(record?.model).toBe("claude-fable-5");
4040
expect(record?.totals).toEqual({
@@ -50,8 +50,8 @@ describe("parseClaudeLine", () => {
5050

5151
it("gives every content block of one message the same dedupe key", () => {
5252
// Content-block records repeat full parent usage; summing them overcounts.
53-
const text = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" }));
54-
const toolUse = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" }));
53+
const [text] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" }));
54+
const [toolUse] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" }));
5555

5656
expect(text?.dedupeKey).toBe(toolUse?.dedupeKey);
5757
expect(text?.totals).toEqual(toolUse?.totals);
@@ -86,3 +86,133 @@ describe("parseClaudeLine", () => {
8686
expect(claudeUsageFormat.parseProjected(synthetic, undefined)).toEqual([]);
8787
});
8888
});
89+
90+
describe("Claude advisor iterations", () => {
91+
const advisorTurn = {
92+
type: "assistant",
93+
timestamp: "2026-10-22T01:31:11.984Z",
94+
sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0",
95+
requestId: "req_1",
96+
message: {
97+
id: "msg_1",
98+
model: "claude-opus-5-5",
99+
usage: {
100+
input_tokens: 4,
101+
cache_read_input_tokens: 118320,
102+
cache_creation_input_tokens: 2676,
103+
output_tokens: 4506,
104+
speed: "standard",
105+
iterations: [
106+
{
107+
type: "message",
108+
input_tokens: 2,
109+
cache_read_input_tokens: 58352,
110+
cache_creation_input_tokens: 1616,
111+
output_tokens: 141,
112+
},
113+
{
114+
type: "advisor_message",
115+
model: "claude-fable-5-1",
116+
input_tokens: 54204,
117+
cache_read_input_tokens: 0,
118+
cache_creation_input_tokens: 0,
119+
output_tokens: 4229,
120+
},
121+
{
122+
type: "message",
123+
input_tokens: 2,
124+
cache_read_input_tokens: 59968,
125+
cache_creation_input_tokens: 1060,
126+
output_tokens: 4365,
127+
},
128+
],
129+
},
130+
},
131+
};
132+
const withIterations = (iterations: readonly unknown[]) => ({
133+
...advisorTurn,
134+
message: { ...advisorTurn.message, usage: { ...advisorTurn.message.usage, iterations } },
135+
});
136+
const parse = (turn: unknown) => claudeUsageFormat.parseLine(JSON.stringify(turn), undefined);
137+
138+
it("reports the advisor model's tokens as their own record beside the unchanged executor record", () => {
139+
const executor = {
140+
provider: "claude",
141+
timestampMs: Date.parse("2026-10-22T01:31:11.984Z"),
142+
model: "claude-opus-5-5",
143+
sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0",
144+
totals: {
145+
uncachedInputTokens: 4,
146+
cachedInputTokens: 118320,
147+
cacheCreationTokens: 2676,
148+
outputTokens: 4506,
149+
reasoningTokens: 0,
150+
},
151+
reportedCostUsd: null,
152+
speed: "standard",
153+
dedupeKey: "msg_1:req_1",
154+
};
155+
const expected = [
156+
executor,
157+
{
158+
...executor,
159+
model: "claude-fable-5-1",
160+
totals: {
161+
uncachedInputTokens: 54204,
162+
cachedInputTokens: 0,
163+
cacheCreationTokens: 0,
164+
outputTokens: 4229,
165+
reasoningTokens: 0,
166+
},
167+
dedupeKey: expect.any(String),
168+
},
169+
];
170+
171+
expect(parse(advisorTurn)).toEqual(expected);
172+
expect(claudeUsageFormat.parseProjected(advisorTurn, undefined)).toEqual(expected);
173+
});
174+
175+
it("keys the advisor record apart from the executor and the same across a message's content blocks", () => {
176+
const block = (type: string) =>
177+
parse({ ...advisorTurn, message: { ...advisorTurn.message, content: [{ type }] } });
178+
const [executorKey, advisorKey] = block("text").map((record) => record.dedupeKey);
179+
180+
expect(advisorKey).toEqual(expect.any(String));
181+
expect(advisorKey).not.toBe(executorKey);
182+
expect(block("tool_use").map((record) => record.dedupeKey)).toEqual([executorKey, advisorKey]);
183+
184+
const unkeyed = parse({
185+
...advisorTurn,
186+
requestId: undefined,
187+
message: { ...advisorTurn.message, id: undefined },
188+
});
189+
expect(unkeyed.map((record) => record.dedupeKey)).toEqual([null, null]);
190+
});
191+
192+
it("emits only the executor record without an advisor iteration", () => {
193+
const executorIterations = advisorTurn.message.usage.iterations.filter(
194+
(iteration) => iteration.type === "message",
195+
);
196+
197+
expect(parse(withIterations(executorIterations))).toHaveLength(1);
198+
expect(claudeUsageFormat.parseProjected(withIterations([]), undefined)).toHaveLength(1);
199+
expect(
200+
claudeUsageFormat.parseLine(
201+
claudeLine({ messageId: "msg_3", contentType: "text" }),
202+
undefined,
203+
),
204+
).toHaveLength(1);
205+
});
206+
207+
it("ignores advisor iterations without a usable model and malformed entries", () => {
208+
const unusable = withIterations([
209+
null,
210+
"advisor_message",
211+
{ type: "advisor_message", input_tokens: 5 },
212+
{ type: "advisor_message", model: "", input_tokens: 5 },
213+
{ type: "advisor_message", model: "<synthetic>", input_tokens: 5 },
214+
]);
215+
216+
expect(parse(unusable).map((record) => record.model)).toEqual(["claude-opus-5-5"]);
217+
});
218+
});

‎apps/server/src/provider/Drivers/claudeUsage.ts‎

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,35 +20,35 @@ export const CLAUDE_SYNTHETIC_MODEL = "<synthetic>";
2020
* the same complete `usage` object for the parent message. Summing them overcounts by roughly
2121
* 2.4x on a real workload, so the caller must drop repeats by `dedupeKey` and keep the first.
2222
*/
23-
export function parseClaudeLine(line: string): UsageRecord | null {
23+
export function parseClaudeLine(line: string): readonly UsageRecord[] {
2424
let parsed: unknown;
2525
try {
2626
parsed = JSON.parse(line);
2727
} catch {
28-
return null;
28+
return [];
2929
}
3030
return parseClaudeRecord(parsed);
3131
}
3232

33-
function parseClaudeRecord(parsed: unknown): UsageRecord | null {
34-
if (typeof parsed !== "object" || parsed === null) return null;
33+
function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] {
34+
if (typeof parsed !== "object" || parsed === null) return [];
3535

3636
const record = parsed as Record<string, unknown>;
37-
if (record["type"] !== "assistant") return null;
37+
if (record["type"] !== "assistant") return [];
3838

3939
const message = record["message"];
40-
if (typeof message !== "object" || message === null) return null;
40+
if (typeof message !== "object" || message === null) return [];
4141
const messageRecord = message as Record<string, unknown>;
4242

4343
const usage = messageRecord["usage"];
44-
if (typeof usage !== "object" || usage === null) return null;
44+
if (typeof usage !== "object" || usage === null) return [];
4545
const usageRecord = usage as Record<string, unknown>;
4646

4747
const timestampMs = parseTimestampMs(record["timestamp"]);
48-
if (timestampMs === null) return null;
48+
if (timestampMs === null) return [];
4949

5050
const model = typeof messageRecord["model"] === "string" ? messageRecord["model"] : "";
51-
if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return null;
51+
if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return [];
5252

5353
const messageId = typeof messageRecord["id"] === "string" ? messageRecord["id"] : null;
5454
const requestId = typeof record["requestId"] === "string" ? record["requestId"] : null;
@@ -57,7 +57,7 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null {
5757

5858
const cost = record["costUSD"];
5959

60-
return {
60+
const executor: UsageRecord = {
6161
provider: "claude",
6262
timestampMs,
6363
model,
@@ -74,10 +74,40 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null {
7474
speed: usageRecord["speed"] === "fast" ? "fast" : "standard",
7575
dedupeKey,
7676
};
77-
}
7877

79-
const orEmpty = (record: UsageRecord | null): readonly UsageRecord[] =>
80-
record === null ? [] : [record];
78+
// Top-level `usage` is the executor's alone. Advisor calls run on another model at its own rates
79+
// and appear only in `iterations`, so each one is its own record under that model.
80+
const records = [executor];
81+
const iterations: readonly unknown[] = Array.isArray(usageRecord["iterations"])
82+
? usageRecord["iterations"]
83+
: [];
84+
for (const [index, iteration] of iterations.entries()) {
85+
if (typeof iteration !== "object" || iteration === null) continue;
86+
const advisor = iteration as Record<string, unknown>;
87+
if (advisor["type"] !== "advisor_message") continue;
88+
89+
const advisorModel = typeof advisor["model"] === "string" ? advisor["model"] : "";
90+
if (advisorModel.length === 0 || advisorModel === CLAUDE_SYNTHETIC_MODEL) continue;
91+
92+
records.push({
93+
provider: "claude",
94+
timestampMs,
95+
model: advisorModel,
96+
sessionId: executor.sessionId,
97+
totals: {
98+
uncachedInputTokens: tokenCount(advisor["input_tokens"]),
99+
cachedInputTokens: tokenCount(advisor["cache_read_input_tokens"]),
100+
cacheCreationTokens: tokenCount(advisor["cache_creation_input_tokens"]),
101+
outputTokens: tokenCount(advisor["output_tokens"]),
102+
reasoningTokens: 0,
103+
},
104+
reportedCostUsd: null,
105+
speed: executor.speed,
106+
dedupeKey: dedupeKey === null ? null : `${dedupeKey}:advisor:${index}`,
107+
});
108+
}
109+
return records;
110+
}
81111

82112
export const claudeUsageFormat: TranscriptUsageFormat<void> = {
83113
selectFields: {
@@ -89,8 +119,8 @@ export const claudeUsageFormat: TranscriptUsageFormat<void> = {
89119
message: { id: true, model: true, usage: true },
90120
},
91121
usageMarkers: ['"usage"'],
92-
parseLine: (line) => orEmpty(parseClaudeLine(line)),
93-
parseProjected: (projected) => orEmpty(parseClaudeRecord(projected)),
122+
parseLine: (line) => parseClaudeLine(line),
123+
parseProjected: (projected) => parseClaudeRecord(projected),
94124
};
95125

96126
export const claudeUsageReader: ProviderUsageReader<ClaudeSettings, Path.Path> = {

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2299,7 +2299,7 @@ describe("UsageService", () => {
22992299
60 + 20 * (version === 4 ? 1 : 2),
23002300
);
23012301
yield* upgraded.awaitPersisted;
2302-
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v9.json");
2302+
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v10.json");
23032303
assert.strictEqual(
23042304
yield* Effect.promise(() => NodeFSP.readFile(legacyPath, "utf8")),
23052305
legacy,

‎apps/server/src/usage/usageScanCache.test.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,11 @@ describe("scan cache round trip", () => {
218218
};
219219

220220
const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP);
221-
expect(restored.get("/gone.jsonl")).toEqual(original.get("/gone.jsonl"));
221+
expect(restored.get("/gone.jsonl")).toEqual({
222+
...original.get("/gone.jsonl"),
223+
size: -1,
224+
position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
225+
});
222226
expect(restored.get("/gone-codex.jsonl")).toEqual({
223227
...original.get("/gone-codex.jsonl"),
224228
size: -1,
@@ -245,6 +249,18 @@ describe("scan cache round trip", () => {
245249
},
246250
);
247251

252+
it("re-parses a Claude transcript cached before advisor usage was counted", () => {
253+
const original = cacheWith([["/claude.jsonl", 100, [record()]]]);
254+
const v9 = { ...encodeScanCache(original), version: 9 };
255+
256+
const restored = decodeScanCache(JSON.parse(JSON.stringify(v9)), TEST_FORMAT_MAP);
257+
expect(restored.get("/claude.jsonl")).toEqual({
258+
...original.get("/claude.jsonl"),
259+
size: -1,
260+
position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
261+
});
262+
});
263+
248264
it("rewrites only changed entries and still restores the whole cache", () => {
249265
const write = makeScanCacheWriter();
250266
const cache = cacheWith([

‎apps/server/src/usage/usageScanCache.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,17 @@ import * as Schema from "effect/Schema";
99
import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts";
1010
import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts";
1111

12-
const USAGE_SCAN_CACHE_VERSION = 9 as const;
12+
const USAGE_SCAN_CACHE_VERSION = 10 as const;
1313
const SPEED_COMPATIBLE_SINCE_VERSION = 4;
1414

1515
/**
1616
* An older server sharing that directory cannot read a newer cache and would
1717
* replace it, dropping saved usage for deleted transcripts. Separate files keep
1818
* both.
1919
*/
20-
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v9.json";
20+
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v10.json";
2121
export const LEGACY_SCAN_CACHE_FILE_NAMES = [
22+
"usage-scan-cache-v9.json",
2223
"usage-scan-cache-v8.json",
2324
"usage-scan-cache-v7.json",
2425
"usage-scan-cache-v6.json",
@@ -291,10 +292,12 @@ export function decodeScanCache(
291292
}
292293
// Keep history for deleted rollouts, but re-parse incompatible reducer state.
293294
// v9 repairs skipped initial Codex requests; clean v8 entries can still resume.
295+
// v10 adds Claude advisor calls, which no older Claude entry holds.
294296
const legacy =
295-
format.state !== undefined &&
296-
(version < 6 ||
297-
(entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0))));
297+
(entry.p === "claude" && version < 10) ||
298+
(format.state !== undefined &&
299+
(version < 6 ||
300+
(entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0)))));
298301
// A corrupt state disqualifies the entry: resuming with it would attach
299302
// appended usage to the wrong model or replay fork-copied history.
300303
if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue;

0 commit comments

Comments
 (0)