|
| 1 | +import { afterAll, beforeAll, describe, expect, test } from "bun:test" |
| 2 | +import z from "zod" |
| 3 | +import { tmpdir } from "../fixture/fixture" |
| 4 | +import { Instance } from "../../src/project/instance" |
| 5 | +import { Agent } from "../../src/agent/agent" |
| 6 | +import { Provider } from "../../src/provider/provider" |
| 7 | +import { PermissionNext } from "../../src/permission/next" |
| 8 | +import { Skill } from "../../src/skill" |
| 9 | +import { SystemPrompt } from "../../src/session/system" |
| 10 | +import { InstructionPrompt } from "../../src/session/instruction" |
| 11 | +import { ToolRegistry } from "../../src/tool/registry" |
| 12 | +import { HeadlessRun } from "../../src/flag/headless-run" |
| 13 | + |
| 14 | +/** |
| 15 | + * Characters in the fixed prefix (system prompt plus tool definitions) that a |
| 16 | + * headless `aictrl run` of the build agent sends on every step, before any |
| 17 | + * conversation content. Set just above the current size: lower it whenever a |
| 18 | + * change shrinks the prefix, and never raise it without a reason (#160). |
| 19 | + */ |
| 20 | +const HEADLESS_PREFIX_BUDGET = 29_500 |
| 21 | + |
| 22 | +// The lite build's prompt contract. |
| 23 | +const HEADLESS_ENV = { AICTRL_HEADLESS: "true" } |
| 24 | + |
| 25 | +type Piece = { name: string; chars: number } |
| 26 | + |
| 27 | +/** System prompt pieces, assembled the way SessionPrompt and LLM.stream assemble them. */ |
| 28 | +async function systemPieces(model: Provider.Model): Promise<Piece[]> { |
| 29 | + const instructions = await InstructionPrompt.system() |
| 30 | + return [ |
| 31 | + { name: "system: provider prompt", chars: SystemPrompt.provider(model).join("\n").length }, |
| 32 | + { name: "system: environment", chars: (await SystemPrompt.environment(model)).join("\n").length }, |
| 33 | + { name: "system: instructions", chars: instructions.join("\n").length }, |
| 34 | + ] |
| 35 | +} |
| 36 | + |
| 37 | +/** Tool definitions the model receives: the registry list minus what LLM.stream drops. */ |
| 38 | +async function toolPieces(model: Provider.Model, agent: Agent.Info): Promise<Piece[]> { |
| 39 | + const tools = await ToolRegistry.tools({ providerID: model.providerID, modelID: model.api.id }, agent) |
| 40 | + const disabled = PermissionNext.disabled( |
| 41 | + tools.map((tool) => tool.id), |
| 42 | + agent.permission, |
| 43 | + ) |
| 44 | + return tools |
| 45 | + .filter((tool) => tool.id !== "invalid" && !disabled.has(tool.id)) |
| 46 | + .map((tool) => ({ |
| 47 | + name: `tool: ${tool.id}`, |
| 48 | + chars: tool.id.length + tool.description.length + JSON.stringify(z.toJSONSchema(tool.parameters)).length, |
| 49 | + })) |
| 50 | +} |
| 51 | + |
| 52 | +function breakdown(pieces: Piece[]) { |
| 53 | + return pieces |
| 54 | + .toSorted((a, b) => b.chars - a.chars) |
| 55 | + .map((piece) => ` ${piece.name.padEnd(28)} ${String(piece.chars).padStart(6)} chars`) |
| 56 | + .join("\n") |
| 57 | +} |
| 58 | + |
| 59 | +describe("headless request prefix budget (#160)", () => { |
| 60 | + const previous: Record<string, string | undefined> = {} |
| 61 | + |
| 62 | + beforeAll(() => { |
| 63 | + HeadlessRun.enable() |
| 64 | + for (const [key, value] of Object.entries(HEADLESS_ENV)) { |
| 65 | + previous[key] = process.env[key] |
| 66 | + process.env[key] = value |
| 67 | + } |
| 68 | + }) |
| 69 | + |
| 70 | + afterAll(() => { |
| 71 | + HeadlessRun.disable() |
| 72 | + for (const [key, value] of Object.entries(previous)) { |
| 73 | + if (value === undefined) delete process.env[key] |
| 74 | + else process.env[key] = value |
| 75 | + } |
| 76 | + }) |
| 77 | + |
| 78 | + test("system prompt plus tool definitions stay within the budget", async () => { |
| 79 | + await using tmp = await tmpdir({ |
| 80 | + git: true, |
| 81 | + config: { |
| 82 | + enabled_providers: ["alibaba"], |
| 83 | + provider: { alibaba: { options: { apiKey: "test-key" } } }, |
| 84 | + }, |
| 85 | + }) |
| 86 | + |
| 87 | + await Instance.provide({ |
| 88 | + directory: tmp.path, |
| 89 | + fn: async () => { |
| 90 | + // The skill tool lists every discovered skill; the fixture must not find any. |
| 91 | + expect(await Skill.all()).toEqual([]) |
| 92 | + |
| 93 | + const model = await Provider.getModel("alibaba", "qwen-plus") |
| 94 | + const agent = await Agent.get("build") |
| 95 | + const pieces = [...(await systemPieces(model)), ...(await toolPieces(model, agent))] |
| 96 | + const total = pieces.reduce((sum, piece) => sum + piece.chars, 0) |
| 97 | + |
| 98 | + const report = [ |
| 99 | + `Headless request prefix is ${total} chars (~${Math.round(total / 4)} tokens); budget is ${HEADLESS_PREFIX_BUDGET}.`, |
| 100 | + breakdown(pieces), |
| 101 | + ].join("\n") |
| 102 | + if (total > HEADLESS_PREFIX_BUDGET) throw new Error(report) |
| 103 | + console.log(report) |
| 104 | + }, |
| 105 | + }) |
| 106 | + }) |
| 107 | +}) |
0 commit comments