Skip to content

Commit 6e37289

Browse files
feat(script): measure tokens per request and project each token lever (anomalyco#47)
Two read-only tools to measure every token lever before and after: script/token-report.ts --db <copy of the session DB> --models <models.json> [--provider/--model/--since/--until] [--compact-at N,...] [--budget step:floor,...] [--cut N,...] [--after-compaction N] - requests by provider/model, session x agent and day: prompt tokens per request (mean/p50/p90/max), uncached per request, cache misses, cost priced from models.json; - prompt-size histogram, the prompt floor (first request of a session, first after a compaction), every compaction with the prompt before and after; - tool output: tokens per byte calibrated from step pairs, share of prompt growth, per tool, steps over 16-128 KB; - lever projections: the same requests replayed with a compaction threshold (with the product's re-compaction guard), the step budget (OutputBudget.plan on the real sizes), or a fixed per-request cut. Opens the DB read-only; run it on a VACUUM INTO copy. The analysis is pure (script/token-usage.ts, 15 tests). script/prompt-probe.ts --dir <project> [--config '<json>'] [--env K=V] - captures one real request (real home and config, temporary data dirs, a local fake endpoint) and breaks it down: system prompt sections (instruction files, the skill list, MCP instructions), tool definitions grouped by MCP server, messages. --config / --env measure a change. Stops the whole process tree (MCP servers are children) on Windows. Ref LDE-40, anomalyco#47 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01GHX6CpBR14wnFxJLDahwTr
1 parent 208bb21 commit 6e37289

4 files changed

Lines changed: 1188 additions & 0 deletions

File tree

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
#!/usr/bin/env bun
2+
// What does one request carry before any conversation? (#47, Muse token efficiency)
3+
//
4+
// bun script/prompt-probe.ts --dir <project dir> [--dir ...] [--agent build] [--save <file>]
5+
// [--tokens-per-byte 0.3] [--env NAME=VALUE ...] [--config '<extra opencode.json JSON>']
6+
//
7+
// Starts a local OpenAI-compatible endpoint that records the request and
8+
// answers "ok", then runs `opencode run` once in each directory against it.
9+
// The real home and config are used (so instructions, skills and MCP servers
10+
// load as they do for the Muses); the data, state and cache directories are
11+
// temporary, so no session is written to the real database. The request body
12+
// is broken down into system prompt sections, tool definitions (grouped by MCP
13+
// server) and messages. --save writes the raw body; it can contain whatever the
14+
// instruction files contain, so keep it out of the repository.
15+
16+
import { spawn } from "node:child_process"
17+
import { mkdtempSync, rmSync } from "node:fs"
18+
import { tmpdir } from "node:os"
19+
import path from "node:path"
20+
import { parseArgs } from "node:util"
21+
22+
const { values: args } = parseArgs({
23+
options: {
24+
dir: { type: "string", multiple: true },
25+
agent: { type: "string", default: "build" },
26+
save: { type: "string" },
27+
"tokens-per-byte": { type: "string", default: "0.3" },
28+
env: { type: "string", multiple: true },
29+
config: { type: "string" },
30+
timeout: { type: "string", default: "240000" },
31+
},
32+
})
33+
const dirs = args.dir ?? [process.cwd()]
34+
const ratio = Number(args["tokens-per-byte"])
35+
const extraEnv = Object.fromEntries((args.env ?? []).map((pair) => pair.split(/=(.*)/s).slice(0, 2)))
36+
const opencode = path.resolve(import.meta.dir, "..")
37+
38+
const bytes = (value: unknown) => Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf-8")
39+
const k = (n: number) => (n >= 1e6 ? `${(n / 1e6).toFixed(2)}M` : n >= 1e3 ? `${(n / 1e3).toFixed(1)}k` : `${n}`)
40+
const tok = (n: number) => k(Math.round(n * ratio))
41+
42+
async function capture(dir: string) {
43+
let body: any
44+
const got = Promise.withResolvers<void>()
45+
const server = Bun.serve({
46+
port: 0,
47+
hostname: "127.0.0.1",
48+
async fetch(request) {
49+
const url = new URL(request.url)
50+
if (!url.pathname.endsWith("/chat/completions")) return new Response("{}", { status: 404 })
51+
const json = await request.json()
52+
// the first request of the session; title generation uses the small model and has no tools
53+
if (!body && Array.isArray(json.tools) && json.tools.length > 0) {
54+
body = json
55+
got.resolve()
56+
}
57+
const chunk = (delta: object, finish: string | null) =>
58+
`data: ${JSON.stringify({ id: "probe", object: "chat.completion.chunk", created: 0, model: "probe", choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`
59+
return new Response(
60+
chunk({ role: "assistant", content: "ok" }, null) +
61+
chunk({}, "stop") +
62+
`data: ${JSON.stringify({ id: "probe", object: "chat.completion.chunk", created: 0, model: "probe", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n` +
63+
"data: [DONE]\n\n",
64+
{ headers: { "content-type": "text/event-stream" } },
65+
)
66+
},
67+
})
68+
const temp = mkdtempSync(path.join(tmpdir(), "prompt-probe-"))
69+
const config = {
70+
provider: {
71+
capture: {
72+
npm: "@ai-sdk/openai-compatible",
73+
name: "Capture",
74+
options: { baseURL: `http://127.0.0.1:${server.port}/v1`, apiKey: "probe" },
75+
models: { probe: { name: "Probe", tool_call: true, limit: { context: 1_048_576, output: 131_072 } } },
76+
},
77+
},
78+
model: "capture/probe",
79+
small_model: "capture/probe",
80+
// --config: extra opencode.json content to measure a change, e.g.
81+
// '{"permission":{"linear_*":"deny"}}' or a skill allowlist
82+
...(args.config ? JSON.parse(args.config) : {}),
83+
}
84+
const child = spawn(
85+
process.execPath,
86+
["run", path.join(opencode, "src", "index.ts"), "run", "--agent", args.agent!, "Reply with the single word ok."],
87+
{
88+
cwd: dir,
89+
env: {
90+
...process.env,
91+
// `run` takes its directory from PWD when set; the inherited one is this script's
92+
PWD: dir,
93+
XDG_DATA_HOME: path.join(temp, "share"),
94+
XDG_STATE_HOME: path.join(temp, "state"),
95+
XDG_CACHE_HOME: path.join(temp, "cache"),
96+
OPENCODE_MODELS_PATH: path.join(opencode, "test", "tool", "fixtures", "models-api.json"),
97+
OPENCODE_DISABLE_MODELS_FETCH: "true",
98+
OPENCODE_DISABLE_AUTOUPDATE: "true",
99+
OPENCODE_CONFIG_CONTENT: JSON.stringify(config),
100+
...extraEnv,
101+
},
102+
stdio: ["ignore", "ignore", "pipe"],
103+
},
104+
)
105+
let stderr = ""
106+
child.stderr?.on("data", (data) => (stderr += data.toString()))
107+
const exited = new Promise<void>((done) => child.once("exit", () => done()))
108+
const timer = setTimeout(
109+
() => got.reject(new Error(`no request within ${args.timeout} ms\n${stderr.slice(-2000)}`)),
110+
Number(args.timeout),
111+
)
112+
try {
113+
await Promise.race([got.promise, exited.then(() => got.promise)])
114+
} finally {
115+
clearTimeout(timer)
116+
// the run starts MCP servers as its own children: stop the whole tree
117+
if (process.platform === "win32" && child.pid)
118+
await new Promise((done) =>
119+
spawn("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" }).once("exit", done),
120+
)
121+
else child.kill()
122+
await Promise.race([exited, new Promise((done) => setTimeout(done, 5000))])
123+
server.stop(true)
124+
try {
125+
rmSync(temp, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 })
126+
} catch {
127+
console.error(`note: could not remove ${temp} (still locked); remove it later`)
128+
}
129+
}
130+
return { body, stderr }
131+
}
132+
133+
function breakdown(body: any) {
134+
const messages: any[] = body.messages ?? []
135+
const text = (content: unknown) =>
136+
typeof content === "string"
137+
? content
138+
: Array.isArray(content)
139+
? content.map((part: any) => (typeof part?.text === "string" ? part.text : "")).join("\n")
140+
: ""
141+
const system = messages
142+
.filter((m) => m.role === "system")
143+
.map((m) => text(m.content))
144+
.join("\n")
145+
const rest = messages.filter((m) => m.role !== "system")
146+
147+
// sections of the system prompt
148+
const sections: { name: string; bytes: number }[] = []
149+
const skills = system.match(/Skills provide specialized[\s\S]*?<\/available_skills>/)?.[0] ?? ""
150+
if (skills)
151+
sections.push({ name: `skills list (${(skills.match(/<skill>/g) ?? []).length} skills)`, bytes: bytes(skills) })
152+
const instructions = [...system.matchAll(/Instructions from: (.+)\n/g)]
153+
for (let i = 0; i < instructions.length; i++) {
154+
const start = instructions[i].index!
155+
const end = i + 1 < instructions.length ? instructions[i + 1].index! : system.length
156+
// an instruction file ends where the next known block starts
157+
const block = system
158+
.slice(start, end)
159+
.split(/\n(?=Skills provide specialized|<mcp_instructions>|<available_references>)/)[0]
160+
sections.push({ name: `instructions: ${instructions[i][1]}`, bytes: bytes(block) })
161+
}
162+
const mcp = system.match(/<mcp_instructions>[\s\S]*?<\/mcp_instructions>/)?.[0]
163+
if (mcp) sections.push({ name: "mcp instructions", bytes: bytes(mcp) })
164+
const known = sections.reduce((total, s) => total + s.bytes, 0)
165+
sections.push({ name: "everything else (base prompt, environment, agent)", bytes: bytes(system) - known })
166+
167+
// tools, grouped by the MCP server prefix where there is one
168+
const tools: any[] = body.tools ?? []
169+
const groups = new Map<string, { count: number; bytes: number }>()
170+
for (const tool of tools) {
171+
const name: string = tool.function?.name ?? tool.name ?? "?"
172+
const server =
173+
name.includes("_") && !["todowrite", "webfetch", "websearch"].includes(name) ? name.split("_")[0] : name
174+
const group = groups.get(server) ?? { count: 0, bytes: 0 }
175+
group.count++
176+
group.bytes += bytes(tool)
177+
groups.set(server, group)
178+
}
179+
return {
180+
system: bytes(system),
181+
sections,
182+
tools: bytes(tools),
183+
toolCount: tools.length,
184+
groups,
185+
messages: bytes(rest),
186+
total: bytes(body),
187+
}
188+
}
189+
190+
for (const dir of dirs) {
191+
const { body } = await capture(dir)
192+
if (args.save) await Bun.write(args.save.replace("{n}", String(dirs.indexOf(dir))), JSON.stringify(body, null, 2))
193+
const b = breakdown(body)
194+
console.log(`\n## ${dir} (agent ${args.agent})\n`)
195+
console.log(`Request body ${k(b.total)} bytes, ~${tok(b.total)} tokens at ${ratio} tokens/byte.\n`)
196+
console.log("| part | bytes | ~tokens |\n|---|---|---|")
197+
console.log(`| system prompt | ${k(b.system)} | ${tok(b.system)} |`)
198+
for (const s of b.sections.sort((a, z) => z.bytes - a.bytes))
199+
console.log(`| &nbsp;&nbsp;${s.name} | ${k(s.bytes)} | ${tok(s.bytes)} |`)
200+
console.log(`| tool definitions (${b.toolCount}) | ${k(b.tools)} | ${tok(b.tools)} |`)
201+
for (const [name, g] of [...b.groups.entries()].sort((a, z) => z[1].bytes - a[1].bytes).slice(0, 12))
202+
console.log(`| &nbsp;&nbsp;${name}${g.count > 1 ? ` (${g.count})` : ""} | ${k(g.bytes)} | ${tok(g.bytes)} |`)
203+
console.log(`| messages | ${k(b.messages)} | ${tok(b.messages)} |`)
204+
}

0 commit comments

Comments
 (0)