|
| 1 | +#!/usr/bin/env bun |
| 2 | +// What does one request carry before any conversation? (#47, Muse token efficiency) |
| 3 | +// |
| 4 | +// bun script/prompt-probe.ts --dir <project dir> [--dir ...] [--agent build] [--save <file>] |
| 5 | +// [--tokens-per-byte 0.3] [--env NAME=VALUE ...] [--config '<extra opencode.json JSON>'] |
| 6 | +// |
| 7 | +// Starts a local OpenAI-compatible endpoint that records the request and |
| 8 | +// answers "ok", then runs `opencode run` once in each directory against it. |
| 9 | +// The real home and config are used (so instructions, skills and MCP servers |
| 10 | +// load as they do for the Muses); the data, state and cache directories are |
| 11 | +// temporary, so no session is written to the real database. The request body |
| 12 | +// is broken down into system prompt sections, tool definitions (grouped by MCP |
| 13 | +// server) and messages. --save writes the raw body; it can contain whatever the |
| 14 | +// instruction files contain, so keep it out of the repository. |
| 15 | + |
| 16 | +import { spawn } from "node:child_process" |
| 17 | +import { mkdtempSync, rmSync } from "node:fs" |
| 18 | +import { tmpdir } from "node:os" |
| 19 | +import path from "node:path" |
| 20 | +import { parseArgs } from "node:util" |
| 21 | + |
| 22 | +const { values: args } = parseArgs({ |
| 23 | + options: { |
| 24 | + dir: { type: "string", multiple: true }, |
| 25 | + agent: { type: "string", default: "build" }, |
| 26 | + save: { type: "string" }, |
| 27 | + "tokens-per-byte": { type: "string", default: "0.3" }, |
| 28 | + env: { type: "string", multiple: true }, |
| 29 | + config: { type: "string" }, |
| 30 | + timeout: { type: "string", default: "240000" }, |
| 31 | + }, |
| 32 | +}) |
| 33 | +const dirs = args.dir ?? [process.cwd()] |
| 34 | +const ratio = Number(args["tokens-per-byte"]) |
| 35 | +const extraEnv = Object.fromEntries((args.env ?? []).map((pair) => pair.split(/=(.*)/s).slice(0, 2))) |
| 36 | +const opencode = path.resolve(import.meta.dir, "..") |
| 37 | + |
| 38 | +const bytes = (value: unknown) => Buffer.byteLength(typeof value === "string" ? value : JSON.stringify(value), "utf-8") |
| 39 | +const k = (n: number) => (n >= 1e6 ? `${(n / 1e6).toFixed(2)}M` : n >= 1e3 ? `${(n / 1e3).toFixed(1)}k` : `${n}`) |
| 40 | +const tok = (n: number) => k(Math.round(n * ratio)) |
| 41 | + |
| 42 | +async function capture(dir: string) { |
| 43 | + let body: any |
| 44 | + const got = Promise.withResolvers<void>() |
| 45 | + const server = Bun.serve({ |
| 46 | + port: 0, |
| 47 | + hostname: "127.0.0.1", |
| 48 | + async fetch(request) { |
| 49 | + const url = new URL(request.url) |
| 50 | + if (!url.pathname.endsWith("/chat/completions")) return new Response("{}", { status: 404 }) |
| 51 | + const json = await request.json() |
| 52 | + // the first request of the session; title generation uses the small model and has no tools |
| 53 | + if (!body && Array.isArray(json.tools) && json.tools.length > 0) { |
| 54 | + body = json |
| 55 | + got.resolve() |
| 56 | + } |
| 57 | + const chunk = (delta: object, finish: string | null) => |
| 58 | + `data: ${JSON.stringify({ id: "probe", object: "chat.completion.chunk", created: 0, model: "probe", choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n` |
| 59 | + return new Response( |
| 60 | + chunk({ role: "assistant", content: "ok" }, null) + |
| 61 | + chunk({}, "stop") + |
| 62 | + `data: ${JSON.stringify({ id: "probe", object: "chat.completion.chunk", created: 0, model: "probe", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n` + |
| 63 | + "data: [DONE]\n\n", |
| 64 | + { headers: { "content-type": "text/event-stream" } }, |
| 65 | + ) |
| 66 | + }, |
| 67 | + }) |
| 68 | + const temp = mkdtempSync(path.join(tmpdir(), "prompt-probe-")) |
| 69 | + const config = { |
| 70 | + provider: { |
| 71 | + capture: { |
| 72 | + npm: "@ai-sdk/openai-compatible", |
| 73 | + name: "Capture", |
| 74 | + options: { baseURL: `http://127.0.0.1:${server.port}/v1`, apiKey: "probe" }, |
| 75 | + models: { probe: { name: "Probe", tool_call: true, limit: { context: 1_048_576, output: 131_072 } } }, |
| 76 | + }, |
| 77 | + }, |
| 78 | + model: "capture/probe", |
| 79 | + small_model: "capture/probe", |
| 80 | + // --config: extra opencode.json content to measure a change, e.g. |
| 81 | + // '{"permission":{"linear_*":"deny"}}' or a skill allowlist |
| 82 | + ...(args.config ? JSON.parse(args.config) : {}), |
| 83 | + } |
| 84 | + const child = spawn( |
| 85 | + process.execPath, |
| 86 | + ["run", path.join(opencode, "src", "index.ts"), "run", "--agent", args.agent!, "Reply with the single word ok."], |
| 87 | + { |
| 88 | + cwd: dir, |
| 89 | + env: { |
| 90 | + ...process.env, |
| 91 | + // `run` takes its directory from PWD when set; the inherited one is this script's |
| 92 | + PWD: dir, |
| 93 | + XDG_DATA_HOME: path.join(temp, "share"), |
| 94 | + XDG_STATE_HOME: path.join(temp, "state"), |
| 95 | + XDG_CACHE_HOME: path.join(temp, "cache"), |
| 96 | + OPENCODE_MODELS_PATH: path.join(opencode, "test", "tool", "fixtures", "models-api.json"), |
| 97 | + OPENCODE_DISABLE_MODELS_FETCH: "true", |
| 98 | + OPENCODE_DISABLE_AUTOUPDATE: "true", |
| 99 | + OPENCODE_CONFIG_CONTENT: JSON.stringify(config), |
| 100 | + ...extraEnv, |
| 101 | + }, |
| 102 | + stdio: ["ignore", "ignore", "pipe"], |
| 103 | + }, |
| 104 | + ) |
| 105 | + let stderr = "" |
| 106 | + child.stderr?.on("data", (data) => (stderr += data.toString())) |
| 107 | + const exited = new Promise<void>((done) => child.once("exit", () => done())) |
| 108 | + const timer = setTimeout( |
| 109 | + () => got.reject(new Error(`no request within ${args.timeout} ms\n${stderr.slice(-2000)}`)), |
| 110 | + Number(args.timeout), |
| 111 | + ) |
| 112 | + try { |
| 113 | + await Promise.race([got.promise, exited.then(() => got.promise)]) |
| 114 | + } finally { |
| 115 | + clearTimeout(timer) |
| 116 | + // the run starts MCP servers as its own children: stop the whole tree |
| 117 | + if (process.platform === "win32" && child.pid) |
| 118 | + await new Promise((done) => |
| 119 | + spawn("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" }).once("exit", done), |
| 120 | + ) |
| 121 | + else child.kill() |
| 122 | + await Promise.race([exited, new Promise((done) => setTimeout(done, 5000))]) |
| 123 | + server.stop(true) |
| 124 | + try { |
| 125 | + rmSync(temp, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 }) |
| 126 | + } catch { |
| 127 | + console.error(`note: could not remove ${temp} (still locked); remove it later`) |
| 128 | + } |
| 129 | + } |
| 130 | + return { body, stderr } |
| 131 | +} |
| 132 | + |
| 133 | +function breakdown(body: any) { |
| 134 | + const messages: any[] = body.messages ?? [] |
| 135 | + const text = (content: unknown) => |
| 136 | + typeof content === "string" |
| 137 | + ? content |
| 138 | + : Array.isArray(content) |
| 139 | + ? content.map((part: any) => (typeof part?.text === "string" ? part.text : "")).join("\n") |
| 140 | + : "" |
| 141 | + const system = messages |
| 142 | + .filter((m) => m.role === "system") |
| 143 | + .map((m) => text(m.content)) |
| 144 | + .join("\n") |
| 145 | + const rest = messages.filter((m) => m.role !== "system") |
| 146 | + |
| 147 | + // sections of the system prompt |
| 148 | + const sections: { name: string; bytes: number }[] = [] |
| 149 | + const skills = system.match(/Skills provide specialized[\s\S]*?<\/available_skills>/)?.[0] ?? "" |
| 150 | + if (skills) |
| 151 | + sections.push({ name: `skills list (${(skills.match(/<skill>/g) ?? []).length} skills)`, bytes: bytes(skills) }) |
| 152 | + const instructions = [...system.matchAll(/Instructions from: (.+)\n/g)] |
| 153 | + for (let i = 0; i < instructions.length; i++) { |
| 154 | + const start = instructions[i].index! |
| 155 | + const end = i + 1 < instructions.length ? instructions[i + 1].index! : system.length |
| 156 | + // an instruction file ends where the next known block starts |
| 157 | + const block = system |
| 158 | + .slice(start, end) |
| 159 | + .split(/\n(?=Skills provide specialized|<mcp_instructions>|<available_references>)/)[0] |
| 160 | + sections.push({ name: `instructions: ${instructions[i][1]}`, bytes: bytes(block) }) |
| 161 | + } |
| 162 | + const mcp = system.match(/<mcp_instructions>[\s\S]*?<\/mcp_instructions>/)?.[0] |
| 163 | + if (mcp) sections.push({ name: "mcp instructions", bytes: bytes(mcp) }) |
| 164 | + const known = sections.reduce((total, s) => total + s.bytes, 0) |
| 165 | + sections.push({ name: "everything else (base prompt, environment, agent)", bytes: bytes(system) - known }) |
| 166 | + |
| 167 | + // tools, grouped by the MCP server prefix where there is one |
| 168 | + const tools: any[] = body.tools ?? [] |
| 169 | + const groups = new Map<string, { count: number; bytes: number }>() |
| 170 | + for (const tool of tools) { |
| 171 | + const name: string = tool.function?.name ?? tool.name ?? "?" |
| 172 | + const server = |
| 173 | + name.includes("_") && !["todowrite", "webfetch", "websearch"].includes(name) ? name.split("_")[0] : name |
| 174 | + const group = groups.get(server) ?? { count: 0, bytes: 0 } |
| 175 | + group.count++ |
| 176 | + group.bytes += bytes(tool) |
| 177 | + groups.set(server, group) |
| 178 | + } |
| 179 | + return { |
| 180 | + system: bytes(system), |
| 181 | + sections, |
| 182 | + tools: bytes(tools), |
| 183 | + toolCount: tools.length, |
| 184 | + groups, |
| 185 | + messages: bytes(rest), |
| 186 | + total: bytes(body), |
| 187 | + } |
| 188 | +} |
| 189 | + |
| 190 | +for (const dir of dirs) { |
| 191 | + const { body } = await capture(dir) |
| 192 | + if (args.save) await Bun.write(args.save.replace("{n}", String(dirs.indexOf(dir))), JSON.stringify(body, null, 2)) |
| 193 | + const b = breakdown(body) |
| 194 | + console.log(`\n## ${dir} (agent ${args.agent})\n`) |
| 195 | + console.log(`Request body ${k(b.total)} bytes, ~${tok(b.total)} tokens at ${ratio} tokens/byte.\n`) |
| 196 | + console.log("| part | bytes | ~tokens |\n|---|---|---|") |
| 197 | + console.log(`| system prompt | ${k(b.system)} | ${tok(b.system)} |`) |
| 198 | + for (const s of b.sections.sort((a, z) => z.bytes - a.bytes)) |
| 199 | + console.log(`| ${s.name} | ${k(s.bytes)} | ${tok(s.bytes)} |`) |
| 200 | + console.log(`| tool definitions (${b.toolCount}) | ${k(b.tools)} | ${tok(b.tools)} |`) |
| 201 | + for (const [name, g] of [...b.groups.entries()].sort((a, z) => z[1].bytes - a[1].bytes).slice(0, 12)) |
| 202 | + console.log(`| ${name}${g.count > 1 ? ` (${g.count})` : ""} | ${k(g.bytes)} | ${tok(g.bytes)} |`) |
| 203 | + console.log(`| messages | ${k(b.messages)} | ${tok(b.messages)} |`) |
| 204 | +} |
0 commit comments