Skip to content

Commit 8ed4562

Browse files
authored
Merge pull request #164 from aictrl-dev/feat/issue-155-headless-todowrite
feat(run): drop todowrite from the default headless tool set (#155)
2 parents 246dfa6 + af1e484 commit 8ed4562

8 files changed

Lines changed: 282 additions & 3 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ and [JSON event reference](EVENTS.md).
7070

7171
In headless mode, Aictrl automatically rejects all interactive permission requests (like `question` or `plan_enter`), ensuring your pipelines never hang.
7272

73+
`aictrl run` does not offer the `todowrite` tool: its definition adds about 2.4k tokens to
74+
every request, and a one-shot run has nobody following a todo list. To track a plan, instruct
75+
the agent to keep one in a file.
76+
7377
### CI/CD Integration
7478

7579
Set `AICTRL_HEADLESS=true` in your environment to force headless behavior even in pseudo-TTYs.

‎packages/cli/README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,10 @@ Provider failures and stream timeouts retain **1**; SIGINT/SIGTERM retain
9797

9898
In headless mode, Aictrl automatically rejects permissions that would otherwise prompt a user.
9999

100+
`aictrl run` does not offer the `todowrite` tool: its definition adds about 2.4k tokens to
101+
every request, and a one-shot run has nobody following a todo list. To track a plan, instruct
102+
the agent to keep one in a file.
103+
100104
## GitHub Agent
101105

102106
Install the GitHub agent into any repository:

‎packages/cli/src/cli/cmd/run.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { terminal } from "./run.terminal"
1010
import { buildToolCatalogItems } from "./tool-catalog"
1111
import { withTimeout } from "../../util/timeout"
1212
import { Flag } from "../../flag/flag"
13+
import { HeadlessRun } from "../../flag/headless-run"
1314
import { bootstrap } from "../bootstrap"
1415
import { EOL } from "os"
1516
import { Filesystem } from "../../util/filesystem"
@@ -1098,6 +1099,8 @@ export const RunCommand = cmd({
10981099
return await invocation.run(() => execute(createAictrlClient({ baseUrl: args.attach, directory })))
10991100
}
11001101

1102+
// Sessions this process serves belong to the headless run until it finishes.
1103+
HeadlessRun.enable()
11011104
const execution = bootstrap(process.cwd(), async () => {
11021105
const sdk = {
11031106
session: {
@@ -1190,6 +1193,6 @@ export const RunCommand = cmd({
11901193
}
11911194
await execute(sdk)
11921195
})
1193-
await invocation.run(execution)
1196+
await invocation.run(execution).finally(() => HeadlessRun.disable())
11941197
},
11951198
})
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/**
2+
* Whether this process is serving a headless `aictrl run`. The run command marks it
3+
* at runtime, whichever entry point or build started it. The mark lives in memory
4+
* rather than in the environment, so it is not inherited by processes a run
5+
* spawns and cannot be switched on by a stray exported variable (#155).
6+
*/
7+
export namespace HeadlessRun {
8+
let active = false
9+
10+
export function enable() {
11+
active = true
12+
}
13+
14+
export function disable() {
15+
active = false
16+
}
17+
18+
export function enabled() {
19+
return active
20+
}
21+
}

‎packages/cli/src/tool/registry.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { Plugin } from "../plugin"
2323
import { WebSearchTool } from "./websearch"
2424
import { CodeSearchTool } from "./codesearch"
2525
import { Flag } from "@/flag/flag"
26+
import { HeadlessRun } from "@/flag/headless-run"
2627
import { Log } from "@/util/log"
2728
import { LspTool } from "./lsp"
2829
import { Truncate } from "./truncation"
@@ -111,10 +112,20 @@ export namespace ToolRegistry {
111112
custom.push(tool)
112113
}
113114

115+
/**
116+
* todowrite keeps a progress list for someone watching the session. A headless
117+
* `aictrl run` has no such reader, and the tool definition costs ~2.4k tokens on
118+
* every step, so a headless run does not offer it (#155).
119+
*/
120+
function todoEnabled() {
121+
return !HeadlessRun.enabled()
122+
}
123+
114124
async function all(): Promise<Tool.Info[]> {
115125
const custom = await state().then((x) => x.custom)
116126
const config = await Config.get()
117127
const question = ["app", "cli", "desktop"].includes(Flag.AICTRL_CLIENT) || Flag.AICTRL_ENABLE_QUESTION_TOOL
128+
const todo = todoEnabled()
118129

119130
return [
120131
InvalidTool,
@@ -127,7 +138,7 @@ export namespace ToolRegistry {
127138
WriteTool,
128139
TaskTool,
129140
WebFetchTool,
130-
TodoWriteTool,
141+
...(todo ? [TodoWriteTool] : []),
131142
// TodoReadTool,
132143
WebSearchTool,
133144
CodeSearchTool,
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import { describe, expect, test } from "bun:test"
2+
import path from "path"
3+
import { tmpdir } from "../fixture/fixture"
4+
5+
const entries = {
6+
full: path.resolve(import.meta.dir, "../../src/index.ts"),
7+
headless: path.resolve(import.meta.dir, "../../src/headless.ts"),
8+
}
9+
10+
/** Runs `aictrl run` against a fixture Gemini server and returns the tool names of the first request. */
11+
async function requestedTools(entry: string) {
12+
await using tmp = await tmpdir()
13+
const requests: any[] = []
14+
const server = Bun.serve({
15+
hostname: "127.0.0.1",
16+
port: 0,
17+
async fetch(request) {
18+
requests.push(await request.json())
19+
const chunk = {
20+
candidates: [{ index: 0, content: { role: "model", parts: [{ text: "Done." }] }, finishReason: "STOP" }],
21+
usageMetadata: { promptTokenCount: 7, candidatesTokenCount: 3, totalTokenCount: 10 },
22+
}
23+
return new Response(`data: ${JSON.stringify(chunk)}\n\n`, {
24+
headers: { "content-type": "text/event-stream" },
25+
})
26+
},
27+
})
28+
await Bun.write(
29+
path.join(tmp.path, "aictrl.json"),
30+
JSON.stringify({
31+
provider: {
32+
fixture: {
33+
npm: "@ai-sdk/google",
34+
options: { apiKey: "fixture", baseURL: `http://127.0.0.1:${server.port}` },
35+
models: { "gemini-fixture": { name: "fixture", limit: { context: 100000, output: 1000 } } },
36+
},
37+
},
38+
agent: { title: { disable: true } },
39+
}),
40+
)
41+
const environment = {
42+
...process.env,
43+
AICTRL_DISABLE_DEFAULT_PLUGINS: "true",
44+
AICTRL_DISABLE_MODELS_FETCH: "true",
45+
AICTRL_DISABLE_AUTOCOMPACT: "true",
46+
}
47+
const proc = Bun.spawn(
48+
["bun", "run", "--conditions=browser", entry, "run", "--model", "fixture/gemini-fixture", "Check this input."],
49+
{ cwd: tmp.path, env: environment, stdout: "pipe", stderr: "pipe" },
50+
)
51+
const timeout = setTimeout(() => proc.kill("SIGKILL"), 15000)
52+
try {
53+
const [stdout, stderr, exit] = await Promise.all([
54+
new Response(proc.stdout).text(),
55+
new Response(proc.stderr).text(),
56+
proc.exited,
57+
])
58+
expect(exit, stderr + stdout).toBe(0)
59+
expect(requests.length, stderr + stdout).toBeGreaterThan(0)
60+
const declarations: { name: string }[] = (requests[0].tools ?? []).flatMap(
61+
(tool: any) => tool.functionDeclarations ?? [],
62+
)
63+
expect(declarations.length, JSON.stringify(requests[0])).toBeGreaterThan(0)
64+
return declarations.map((declaration) => declaration.name)
65+
} finally {
66+
clearTimeout(timeout)
67+
proc.kill("SIGKILL")
68+
server.stop(true)
69+
}
70+
}
71+
72+
describe("headless run default tools (#155)", () => {
73+
test.each(Object.entries(entries))(
74+
"a %s-build run sends no todowrite or todoread tool",
75+
async (_, entry) => {
76+
const tools = await requestedTools(entry)
77+
expect(tools).toContain("bash")
78+
expect(tools).not.toContain("todowrite")
79+
expect(tools).not.toContain("todoread")
80+
},
81+
20000,
82+
)
83+
})
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import { afterAll, beforeAll, describe, expect, test } from "bun:test"
2+
import z from "zod"
3+
import { tmpdir } from "../fixture/fixture"
4+
import { Instance } from "../../src/project/instance"
5+
import { Agent } from "../../src/agent/agent"
6+
import { Provider } from "../../src/provider/provider"
7+
import { PermissionNext } from "../../src/permission/next"
8+
import { Skill } from "../../src/skill"
9+
import { SystemPrompt } from "../../src/session/system"
10+
import { InstructionPrompt } from "../../src/session/instruction"
11+
import { ToolRegistry } from "../../src/tool/registry"
12+
import { HeadlessRun } from "../../src/flag/headless-run"
13+
14+
/**
15+
* Characters in the fixed prefix (system prompt plus tool definitions) that a
16+
* headless `aictrl run` of the build agent sends on every step, before any
17+
* conversation content. Set just above the current size: lower it whenever a
18+
* change shrinks the prefix, and never raise it without a reason (#160).
19+
*/
20+
const HEADLESS_PREFIX_BUDGET = 29_500
21+
22+
// The lite build's prompt contract.
23+
const HEADLESS_ENV = { AICTRL_HEADLESS: "true" }
24+
25+
type Piece = { name: string; chars: number }
26+
27+
/** System prompt pieces, assembled the way SessionPrompt and LLM.stream assemble them. */
28+
async function systemPieces(model: Provider.Model): Promise<Piece[]> {
29+
const instructions = await InstructionPrompt.system()
30+
return [
31+
{ name: "system: provider prompt", chars: SystemPrompt.provider(model).join("\n").length },
32+
{ name: "system: environment", chars: (await SystemPrompt.environment(model)).join("\n").length },
33+
{ name: "system: instructions", chars: instructions.join("\n").length },
34+
]
35+
}
36+
37+
/** Tool definitions the model receives: the registry list minus what LLM.stream drops. */
38+
async function toolPieces(model: Provider.Model, agent: Agent.Info): Promise<Piece[]> {
39+
const tools = await ToolRegistry.tools({ providerID: model.providerID, modelID: model.api.id }, agent)
40+
const disabled = PermissionNext.disabled(
41+
tools.map((tool) => tool.id),
42+
agent.permission,
43+
)
44+
return tools
45+
.filter((tool) => tool.id !== "invalid" && !disabled.has(tool.id))
46+
.map((tool) => ({
47+
name: `tool: ${tool.id}`,
48+
chars: tool.id.length + tool.description.length + JSON.stringify(z.toJSONSchema(tool.parameters)).length,
49+
}))
50+
}
51+
52+
function breakdown(pieces: Piece[]) {
53+
return pieces
54+
.toSorted((a, b) => b.chars - a.chars)
55+
.map((piece) => ` ${piece.name.padEnd(28)} ${String(piece.chars).padStart(6)} chars`)
56+
.join("\n")
57+
}
58+
59+
describe("headless request prefix budget (#160)", () => {
60+
const previous: Record<string, string | undefined> = {}
61+
62+
beforeAll(() => {
63+
HeadlessRun.enable()
64+
for (const [key, value] of Object.entries(HEADLESS_ENV)) {
65+
previous[key] = process.env[key]
66+
process.env[key] = value
67+
}
68+
})
69+
70+
afterAll(() => {
71+
HeadlessRun.disable()
72+
for (const [key, value] of Object.entries(previous)) {
73+
if (value === undefined) delete process.env[key]
74+
else process.env[key] = value
75+
}
76+
})
77+
78+
test("system prompt plus tool definitions stay within the budget", async () => {
79+
await using tmp = await tmpdir({
80+
git: true,
81+
config: {
82+
enabled_providers: ["alibaba"],
83+
provider: { alibaba: { options: { apiKey: "test-key" } } },
84+
},
85+
})
86+
87+
await Instance.provide({
88+
directory: tmp.path,
89+
fn: async () => {
90+
// The skill tool lists every discovered skill; the fixture must not find any.
91+
expect(await Skill.all()).toEqual([])
92+
93+
const model = await Provider.getModel("alibaba", "qwen-plus")
94+
const agent = await Agent.get("build")
95+
const pieces = [...(await systemPieces(model)), ...(await toolPieces(model, agent))]
96+
const total = pieces.reduce((sum, piece) => sum + piece.chars, 0)
97+
98+
const report = [
99+
`Headless request prefix is ${total} chars (~${Math.round(total / 4)} tokens); budget is ${HEADLESS_PREFIX_BUDGET}.`,
100+
breakdown(pieces),
101+
].join("\n")
102+
if (total > HEADLESS_PREFIX_BUDGET) throw new Error(report)
103+
console.log(report)
104+
},
105+
})
106+
})
107+
})

‎packages/cli/test/tool/registry.test.ts‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
import { describe, expect, test } from "bun:test"
1+
import { afterEach, describe, expect, test } from "bun:test"
22
import path from "path"
33
import fs from "fs/promises"
44
import os from "os"
55
import { tmpdir } from "../fixture/fixture"
66
import { Instance } from "../../src/project/instance"
77
import { ToolRegistry } from "../../src/tool/registry"
88
import { Config } from "../../src/config/config"
9+
import { Agent } from "../../src/agent/agent"
10+
import { HeadlessRun } from "../../src/flag/headless-run"
911

1012
describe("tool.registry", () => {
1113
test("loads tools from .aictrl/tool (singular)", async () => {
@@ -179,3 +181,47 @@ describe("tool.registry", () => {
179181
}
180182
})
181183
})
184+
185+
describe("tool.registry todo tools (#155)", () => {
186+
afterEach(() => {
187+
HeadlessRun.disable()
188+
delete process.env.AICTRL_HEADLESS_RUN
189+
delete process.env.AICTRL_ENABLE_TODO_TOOL
190+
})
191+
192+
async function toolIDs() {
193+
await using tmp = await tmpdir()
194+
return Instance.provide({
195+
directory: tmp.path,
196+
fn: async () => {
197+
const agent = await Agent.get("build")
198+
const tools = await ToolRegistry.tools({ providerID: "anthropic", modelID: "claude-sonnet-4" }, agent)
199+
return tools.map((tool) => tool.id)
200+
},
201+
})
202+
}
203+
204+
test("a headless run gets neither todowrite nor todoread", async () => {
205+
HeadlessRun.enable()
206+
const ids = await toolIDs()
207+
expect(ids).toContain("bash")
208+
expect(ids).not.toContain("todowrite")
209+
expect(ids).not.toContain("todoread")
210+
})
211+
212+
test("a headless run has no environment switch back to todowrite", async () => {
213+
HeadlessRun.enable()
214+
process.env.AICTRL_ENABLE_TODO_TOOL = "1"
215+
expect(await toolIDs()).not.toContain("todowrite")
216+
})
217+
218+
test("sessions outside a headless run keep todowrite", async () => {
219+
expect(await toolIDs()).toContain("todowrite")
220+
})
221+
222+
test("the environment cannot mark a session as a headless run", async () => {
223+
// A run's child processes and stray shell exports must not drop the tool elsewhere.
224+
process.env.AICTRL_HEADLESS_RUN = "true"
225+
expect(await toolIDs()).toContain("todowrite")
226+
})
227+
})

0 commit comments

Comments
 (0)