|
| 1 | +import { describe, expect, test } from "bun:test" |
| 2 | +import path from "path" |
| 3 | +import { tmpdir } from "../fixture/fixture" |
| 4 | + |
| 5 | +const entry = path.resolve(import.meta.dir, "../../src/index.ts") |
| 6 | +const command = process.env.AICTRL_TEST_BINARY |
| 7 | + ? [process.env.AICTRL_TEST_BINARY] |
| 8 | + : ["bun", "run", "--conditions=browser", entry] |
| 9 | + |
| 10 | +type Usage = { prompt_tokens: number; completion_tokens: number; total_tokens: number } |
| 11 | +type Tokens = { |
| 12 | + total: number |
| 13 | + input: number |
| 14 | + output: number |
| 15 | + reasoning: number |
| 16 | + cache: { read: number; write: number } |
| 17 | +} |
| 18 | +type Event = { |
| 19 | + type: string |
| 20 | + sessionID: string |
| 21 | + subagentSessionID?: string |
| 22 | + messageID?: string |
| 23 | + tokens?: Tokens | null |
| 24 | + usageStatus?: string |
| 25 | +} |
| 26 | + |
| 27 | +function usage(prompt: number, completion: number): Usage { |
| 28 | + return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion } |
| 29 | +} |
| 30 | + |
| 31 | +// Model turns in request order: the parent delegates through `task`, the child answers, the parent finishes. |
| 32 | +const turns = [ |
| 33 | + { |
| 34 | + delta: { |
| 35 | + tool_calls: [ |
| 36 | + { |
| 37 | + index: 0, |
| 38 | + id: "call_task", |
| 39 | + type: "function", |
| 40 | + function: { |
| 41 | + name: "task", |
| 42 | + arguments: JSON.stringify({ description: "Search", prompt: "Find the file.", subagent_type: "general" }), |
| 43 | + }, |
| 44 | + }, |
| 45 | + ], |
| 46 | + }, |
| 47 | + finish: "tool_calls", |
| 48 | + usage: usage(100, 10), |
| 49 | + }, |
| 50 | + { delta: { content: "Found it." }, finish: "stop", usage: usage(50, 5) }, |
| 51 | + { delta: { content: "Done." }, finish: "stop", usage: usage(120, 7) }, |
| 52 | +] |
| 53 | + |
| 54 | +async function run() { |
| 55 | + await using tmp = await tmpdir({ git: true }) |
| 56 | + const pending = [...turns] |
| 57 | + const server = Bun.serve({ |
| 58 | + hostname: "127.0.0.1", |
| 59 | + port: 0, |
| 60 | + idleTimeout: 0, |
| 61 | + fetch() { |
| 62 | + const turn = pending.shift() ?? { delta: { content: "extra" }, finish: "stop", usage: usage(0, 0) } |
| 63 | + const chunks = [ |
| 64 | + { choices: [{ index: 0, delta: { role: "assistant", ...turn.delta }, finish_reason: null }] }, |
| 65 | + { choices: [{ index: 0, delta: {}, finish_reason: turn.finish }], usage: turn.usage }, |
| 66 | + ].map( |
| 67 | + (chunk) => |
| 68 | + `data: ${JSON.stringify({ id: "fixture", object: "chat.completion.chunk", created: 1, model: "fixture", ...chunk })}\n\n`, |
| 69 | + ) |
| 70 | + return new Response(chunks.join("") + "data: [DONE]\n\n", { headers: { "content-type": "text/event-stream" } }) |
| 71 | + }, |
| 72 | + }) |
| 73 | + const config = { |
| 74 | + provider: { |
| 75 | + fixture: { |
| 76 | + npm: "@ai-sdk/openai-compatible", |
| 77 | + options: { apiKey: "fixture", baseURL: `http://127.0.0.1:${server.port}/v1` }, |
| 78 | + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 100000, output: 1000 } } }, |
| 79 | + }, |
| 80 | + }, |
| 81 | + agent: { title: { disable: true } }, |
| 82 | + permission: { "*": "allow" }, |
| 83 | + } |
| 84 | + const home = path.join(tmp.path, "home") |
| 85 | + const proc = Bun.spawn([...command, "run", "--format", "json", "--model", "fixture/fixture", "Delegate a search."], { |
| 86 | + cwd: tmp.path, |
| 87 | + env: { |
| 88 | + ...process.env, |
| 89 | + AICTRL_CONFIG_CONTENT: JSON.stringify(config), |
| 90 | + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", |
| 91 | + AICTRL_DISABLE_AUTOCOMPACT: "true", |
| 92 | + HOME: home, |
| 93 | + AICTRL_TEST_HOME: home, |
| 94 | + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(tmp.path, "managed"), |
| 95 | + XDG_DATA_HOME: path.join(tmp.path, "data"), |
| 96 | + XDG_CONFIG_HOME: path.join(tmp.path, "config"), |
| 97 | + XDG_STATE_HOME: path.join(tmp.path, "state"), |
| 98 | + XDG_CACHE_HOME: path.join(tmp.path, "cache"), |
| 99 | + }, |
| 100 | + stdout: "pipe", |
| 101 | + stderr: "pipe", |
| 102 | + }) |
| 103 | + const timeout = setTimeout(() => proc.kill("SIGKILL"), 20000) |
| 104 | + try { |
| 105 | + const [stdout, stderr, exit] = await Promise.all([ |
| 106 | + new Response(proc.stdout).text(), |
| 107 | + new Response(proc.stderr).text(), |
| 108 | + proc.exited, |
| 109 | + ]) |
| 110 | + const events: Event[] = stdout |
| 111 | + .split("\n") |
| 112 | + .filter((line) => line.startsWith("{")) |
| 113 | + .map((line) => JSON.parse(line)) |
| 114 | + return { exit, output: stderr + stdout, events, unused: pending.length } |
| 115 | + } finally { |
| 116 | + clearTimeout(timeout) |
| 117 | + proc.kill("SIGKILL") |
| 118 | + server.stop(true) |
| 119 | + } |
| 120 | +} |
| 121 | + |
| 122 | +describe("subagent token usage (#213)", () => { |
| 123 | + test("reports each child turn so parent + children add up to the provider total", async () => { |
| 124 | + const result = await run() |
| 125 | + expect(result.exit, result.output).toBe(0) |
| 126 | + expect(result.unused, result.output).toBe(0) |
| 127 | + |
| 128 | + const child = result.events.find((event) => event.type === "subagent_start")?.subagentSessionID |
| 129 | + expect(child, result.output).toBeDefined() |
| 130 | + const parentTurns = result.events.filter((event) => event.type === "message_complete") |
| 131 | + const childTurns = result.events.filter((event) => event.type === "subagent_message_complete") |
| 132 | + |
| 133 | + // message_complete keeps its primary-session meaning. |
| 134 | + expect(parentTurns).toHaveLength(2) |
| 135 | + expect(childTurns).toHaveLength(1) |
| 136 | + expect(childTurns[0]).toMatchObject({ subagentSessionID: child, usageStatus: "reported" }) |
| 137 | + expect(childTurns[0].messageID).not.toBe(parentTurns[0].messageID) |
| 138 | + |
| 139 | + const total = [...parentTurns, ...childTurns].reduce((sum, event) => sum + event.tokens!.total, 0) |
| 140 | + const provider = turns.reduce((sum, turn) => sum + turn.usage.total_tokens, 0) |
| 141 | + expect(childTurns[0].tokens!.total).toBe(55) |
| 142 | + expect(total).toBe(provider) |
| 143 | + }, 30_000) |
| 144 | +}) |
0 commit comments