Skip to content

Commit f6e6a01

Browse files
fix(script): replay tokens exactly and compact the way the product does
Review of 2d87d50198 (token-usage / token-report): - "actual (no lever)" was not actual: the replay folded cache writes into cached reads (priced at the read price) and dropped reasoning, which totals() charges. It now tracks cache writes and reasoning separately; with no lever it reproduces input, cached read, cache write, output and reasoning exactly, tested with nonzero writes and reasoning, and the report prices projections with priced() like the actual requests. - The compaction replay diverged from the product. It now calls the product's overThreshold before each request on the last finished request's tokens (prompt + output, as count() does), skips the check after a summary, and resets the re-compaction floor after every summary, real or simulated, as floorAfterCompaction does. - The two settings are modelled separately: --compact-at is compaction.threshold (every turn); --autonomous-compact-at is experimental.accuracy.autonomous_compact_at (turns sent with autonomous: true, read from the parent user message). On the Muse data the autonomous default alone changes nothing: their turns are not flagged autonomous. - The database reading moved into readSteps() with a test against an in-memory SQLite database (tool rows assigned to their step, UTF-8 byte counts, cut flags from outputPath/archive, summary and autonomous flags). Also tested: the histogram, and an exact compaction count for the guard (7) instead of a range. - The report prints "-" instead of InfinityB for an empty measure. Ref LDE-40, anomalyco#47 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01GHX6CpBR14wnFxJLDahwTr
1 parent 6e37289 commit f6e6a01

3 files changed

Lines changed: 403 additions & 165 deletions

File tree

‎packages/opencode/script/token-report.ts‎

Lines changed: 37 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
// [--after-compaction 75000] [--json]
88
//
99
// Levers are projected one at a time, then combined ("all of the above" takes
10-
// the first value of each). --cut is a fixed number of tokens taken off every
10+
// the first value of each). --compact-at models compaction.threshold (every
11+
// turn); --autonomous-compact-at models experimental.accuracy.autonomous_compact_at
12+
// (turns sent with autonomous: true only). --cut is a fixed number of tokens taken off every
1113
// request, e.g. the tool schemas or instructions a config change removes
1214
// (measure those with script/prompt-probe.ts).
1315
//
@@ -30,6 +32,7 @@ const { values: args } = parseArgs({
3032
provider: { type: "string" },
3133
model: { type: "string" },
3234
"compact-at": { type: "string" },
35+
"autonomous-compact-at": { type: "string" },
3336
budget: { type: "string" },
3437
cut: { type: "string" },
3538
"after-compaction": { type: "string" },
@@ -44,87 +47,7 @@ const until = args.until ? new Date(`${args.until}T00:00:00`).getTime() : Number
4447

4548
// --- read ---------------------------------------------------------------
4649

47-
type Row = {
48-
id: string
49-
message_id: string
50-
session_id: string
51-
type: string
52-
data: string | null
53-
bytes: number | null
54-
tool: string | null
55-
call: string | null
56-
cut: number | null
57-
}
58-
const messages = new Map<string, { time: number; agent: string; provider: string; model: string; summary: boolean }>()
59-
for (const row of db
60-
.query(
61-
`select id, time_created t, json_extract(data,'$.agent') agent, json_extract(data,'$.providerID') provider,
62-
json_extract(data,'$.modelID') model, json_extract(data,'$.summary') summary
63-
from message where json_extract(data,'$.role') = 'assistant'`,
64-
)
65-
.all() as any[])
66-
messages.set(row.id, {
67-
time: row.t,
68-
agent: row.agent ?? "?",
69-
provider: row.provider ?? "?",
70-
model: row.model ?? "?",
71-
summary: row.summary === 1 || row.summary === true || row.summary === "true",
72-
})
73-
74-
const rows = db
75-
.query(
76-
`select id, message_id, session_id, json_extract(data,'$.type') type,
77-
case when json_extract(data,'$.type') = 'step-finish' then data end data,
78-
case when json_extract(data,'$.type') = 'tool'
79-
then length(cast(coalesce(json_extract(data,'$.state.output'),'') as blob)) end bytes,
80-
case when json_extract(data,'$.type') = 'tool' then json_extract(data,'$.tool') end tool,
81-
case when json_extract(data,'$.type') = 'tool' then json_extract(data,'$.callID') end call,
82-
case when json_extract(data,'$.type') = 'tool' then
83-
(json_extract(data,'$.state.metadata.outputPath') is not null
84-
or json_extract(data,'$.state.metadata.archive') is not null) end cut
85-
from part
86-
where json_extract(data,'$.type') in ('step-finish','tool')
87-
order by message_id, id`,
88-
)
89-
.all() as Row[]
90-
91-
const steps: TokenUsage.Step[] = []
92-
let pending: TokenUsage.ToolOutput[] = []
93-
let current = ""
94-
for (const row of rows) {
95-
if (row.message_id !== current) {
96-
current = row.message_id
97-
pending = []
98-
}
99-
if (row.type === "tool") {
100-
pending.push({ tool: row.tool ?? "?", callID: row.call ?? row.id, bytes: row.bytes ?? 0, cut: row.cut === 1 })
101-
continue
102-
}
103-
const message = messages.get(row.message_id)
104-
if (!message) continue
105-
const data = JSON.parse(row.data!) as {
106-
tokens?: { input?: number; output?: number; reasoning?: number; cache?: { read?: number; write?: number } }
107-
cost?: number
108-
}
109-
steps.push({
110-
session: row.session_id,
111-
message: row.message_id,
112-
agent: message.agent,
113-
provider: message.provider,
114-
model: message.model,
115-
// steps of one message share its time; the part id keeps their order
116-
time: message.time + steps.length * 1e-6,
117-
input: data.tokens?.input ?? 0,
118-
output: data.tokens?.output ?? 0,
119-
reasoning: data.tokens?.reasoning ?? 0,
120-
cacheRead: data.tokens?.cache?.read ?? 0,
121-
cacheWrite: data.tokens?.cache?.write ?? 0,
122-
cost: data.cost ?? 0,
123-
summary: message.summary,
124-
tools: pending,
125-
})
126-
pending = []
127-
}
50+
const steps = TokenUsage.readSteps(db)
12851

12952
const selected = steps.filter(
13053
(step) =>
@@ -316,15 +239,15 @@ table(
316239
[
317240
[
318241
"first request of a session",
319-
k(Math.min(...firsts)),
320-
k(TokenUsage.quantile(firsts, 0.5)),
321-
k(Math.max(...firsts)),
242+
firsts.length ? k(Math.min(...firsts)) : "-",
243+
firsts.length ? k(TokenUsage.quantile(firsts, 0.5)) : "-",
244+
firsts.length ? k(Math.max(...firsts)) : "-",
322245
firsts.length,
323246
],
324247
[
325248
"first request after a compaction",
326249
afterCompaction.length ? k(Math.min(...afterCompaction)) : "-",
327-
k(TokenUsage.quantile(afterCompaction, 0.5)),
250+
afterCompaction.length ? k(TokenUsage.quantile(afterCompaction, 0.5)) : "-",
328251
afterCompaction.length ? k(Math.max(...afterCompaction)) : "-",
329252
afterCompaction.length,
330253
],
@@ -359,18 +282,15 @@ out.push(
359282
`${Math.round((100 * toolBytes * calibration.tokensPerByte) / Math.max(1, growth))}% of all prompt growth (${k(growth)} tokens).`,
360283
"",
361284
)
362-
const byTool = [
363-
...TokenUsage.group(
364-
allTools.map((t) => ({ ...t, session: "", time: 0 }) as any),
365-
(t: any) => t.tool as string,
366-
).entries(),
367-
]
285+
const toolGroups = new Map<string, TokenUsage.ToolOutput[]>()
286+
for (const output of allTools) toolGroups.set(output.tool, [...(toolGroups.get(output.tool) ?? []), output])
287+
const byTool = [...toolGroups.entries()]
368288
.map(([tool, list]) => ({
369289
tool,
370290
count: list.length,
371-
bytes: (list as any[]).reduce((total, t) => total + t.bytes, 0),
372-
max: Math.max(...(list as any[]).map((t) => t.bytes)),
373-
cut: (list as any[]).filter((t) => t.cut).length,
291+
bytes: list.reduce((total, t) => total + t.bytes, 0),
292+
max: Math.max(...list.map((t) => t.bytes)),
293+
cut: list.filter((t) => t.cut).length,
374294
}))
375295
.sort((a, b) => b.bytes - a.bytes)
376296
table(
@@ -419,7 +339,12 @@ out.push(
419339
)
420340
const levers: { name: string; lever: TokenUsage.Lever }[] = [{ name: "actual (no lever)", lever: {} }]
421341
for (const value of (args["compact-at"] ?? "").split(",").filter(Boolean))
422-
levers.push({ name: `compact at ${k(Number(value))}`, lever: { compactAt: Number(value) } })
342+
levers.push({ name: `compaction.threshold ${k(Number(value))} (all turns)`, lever: { compactAt: Number(value) } })
343+
for (const value of (args["autonomous-compact-at"] ?? "").split(",").filter(Boolean))
344+
levers.push({
345+
name: `autonomous_compact_at ${k(Number(value))} (autonomous turns only)`,
346+
lever: { autonomousCompactAt: Number(value) },
347+
})
423348
for (const value of (args.budget ?? "").split(",").filter(Boolean)) {
424349
const [stepBytes, floorBytes] = value.split(":").map(Number)
425350
levers.push({
@@ -442,18 +367,32 @@ if (levers.length > 2)
442367
})
443368

444369
const projections = levers.map(({ name, lever }) => {
445-
let total = { requests: 0, compactions: 0, prompt: 0, promptMax: 0, input: 0, cacheRead: 0, output: 0, cost: 0 }
370+
let total = {
371+
requests: 0,
372+
compactions: 0,
373+
prompt: 0,
374+
promptMax: 0,
375+
input: 0,
376+
cacheRead: 0,
377+
cacheWrite: 0,
378+
output: 0,
379+
reasoning: 0,
380+
cost: 0,
381+
}
446382
for (const list of ordered.values()) {
447383
const p = TokenUsage.project(list, lever, options)
448-
const cost = TokenUsage.priced({ input: p.input, cacheRead: p.cacheRead, output: p.output }, dominant(list)).total
384+
// priced exactly as totals() prices the actual requests: writes at their price, reasoning as output
385+
const cost = TokenUsage.priced(p, dominant(list)).total
449386
total = {
450387
requests: total.requests + p.requests,
451388
compactions: total.compactions + p.compactions,
452389
prompt: total.prompt + p.prompt,
453390
promptMax: Math.max(total.promptMax, p.promptMax),
454391
input: total.input + p.input,
455392
cacheRead: total.cacheRead + p.cacheRead,
393+
cacheWrite: total.cacheWrite + p.cacheWrite,
456394
output: total.output + p.output,
395+
reasoning: total.reasoning + p.reasoning,
457396
cost: total.cost + cost,
458397
}
459398
}

0 commit comments

Comments
 (0)