Skip to content

Commit cd994a5

Browse files
authored
Merge pull request #255 from aictrl-dev/test/issue-248-token-budgets-20261021
test: enforce headless token budgets and release comparisons
2 parents ec18d10 + 44ae6b6 commit cd994a5

31 files changed

Lines changed: 3416 additions & 23 deletions

‎.github/workflows/ci.yml‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ on:
44
push:
55
branches: [main]
66
pull_request:
7-
branches: [main]
87

98
permissions:
109
contents: read
@@ -40,5 +39,33 @@ jobs:
4039
sudo apt-get install --no-install-recommends -y ripgrep
4140
rg --version
4241
42+
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
43+
with:
44+
python-version: "3.12"
45+
46+
# Setup may download pinned wheels/ranks. Capture and accounting are offline.
47+
- name: Provision reference tokenizer
48+
run: python3 -I -B script/token-budget/setup.py --directory "$RUNNER_TEMP/token-budget-environment"
49+
50+
- name: Enforce actual SDK prefix token budgets
51+
run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B script/token-budget/run.py --output "$RUNNER_TEMP/token-budget-evidence"'
52+
53+
- name: Verify token fixture loopback binding
54+
working-directory: script/token-budget/release-fixtures
55+
run: bun test token-stub.test.ts
56+
57+
- name: Verify token-budget fault controls
58+
run: '"$RUNNER_TEMP/token-budget-environment/venv/bin/python3" -I -B -m unittest discover -s script/token-budget -p "test_*.py" -v'
59+
env:
60+
TOKEN_BUDGET_TEST_CAMPAIGN: ${{ runner.temp }}/token-budget-evidence
61+
62+
- name: Retain token-budget captures
63+
if: always()
64+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
65+
with:
66+
name: token-budget-evidence
67+
path: ${{ runner.temp }}/token-budget-evidence
68+
if-no-files-found: warn
69+
4370
- name: Run Tests
4471
run: bun turbo test --concurrency=100%

‎EVENTS.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,16 +105,16 @@ Emitted once when the session begins.
105105

106106
### `tool_catalog`
107107

108-
Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Lists every tool and skill that was resolved for the session so consumers can structurally verify tool exposure without string-matching the model's prose.
108+
Emitted once per session, immediately after `session_start` and before the first model turn (present even for zero-turn or immediately-failing runs). Reports builtin registry and resolved MCP tool inventory, plus in-scope skills. This session-level snapshot supports inventory diagnostics; it is not the effective tool list for each model request.
109109

110110
```json
111111
{
112112
"type": "tool_catalog",
113113
"sessionID": "ses_…",
114114
"timestamp": 1741500000000,
115115
"tools": [
116-
{ "name": "record_finding", "source": "mcp", "server": "aictrl" },
117-
{ "name": "record_review_completed", "source": "mcp", "server": "aictrl" },
116+
{ "name": "aictrl_record_finding", "source": "mcp", "server": "aictrl" },
117+
{ "name": "aictrl_record_review_completed", "source": "mcp", "server": "aictrl" },
118118
{ "name": "bash", "source": "builtin" },
119119
{ "name": "read", "source": "builtin" }
120120
],
@@ -125,13 +125,13 @@ Emitted once per session, immediately after `session_start` and before the first
125125
}
126126
```
127127

128-
#### `tools[]` — builtin and MCP tools resolved for this session
128+
#### `tools[]` — builtin registry and resolved MCP inventory for this session
129129

130-
- **`name`** (string, **required**) — the tool/function name as exposed to the model. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`).
130+
- **`name`** (string, **required**) — the tool/function identifier used when a tool is dispatched. Mirrors upstream `ToolListItem.id`. For MCP tools this is `{serverName}_{toolName}` (both sanitised to `[a-zA-Z0-9_-]`). The identifier does not guarantee inclusion in a model request.
131131
- **`source`** (string, **required**) — `"builtin"` for tools built into the CLI; `"mcp"` for tools provided by a connected MCP server.
132132
- **`server`** (string, optional) — the MCP server (client name from config) that provides this tool. Present only when `source` is `"mcp"`.
133133

134-
`description` and `parameters` are intentionally omitted to keep the event lean. The primary consumer only needs `name` + `source` to verify tool exposure.
134+
`description` and `parameters` are intentionally omitted to keep the event lean. `name` + `source` identify inventory entries and their provenance; they do not prove dispatch exposure.
135135

136136
#### `skills[]` — skill packs resolved for this session (separate from `tools[]`)
137137

@@ -140,9 +140,9 @@ Only skills in scope for the run are listed; see [Skill Events](#skill-events).
140140
- **`name`** (string, **required**) — skill name from `SKILL.md` frontmatter.
141141
- **`version`** (string or null, **required**) — version string from the `version` field in `SKILL.md` frontmatter. `null` when no version is declared.
142142

143-
> **Use case:** The server-side completion gate (aictrl-dev/aictrl #3216) uses `tools[]` to verify that `record_finding` and `record_review_completed` were actually in the model's function list at dispatch time, and `skills[]` to record which skill version produced the review. Detects the "silent success" failure mode where a missing MCP server lets a run complete green without ever having the review tools available.
143+
> **Use case:** A server-side completion gate (aictrl-dev/aictrl #3216) can use `tools[]` to detect missing expected inventory and `skills[]` to record resolved skill versions. To verify that `aictrl_record_finding` and `aictrl_record_review_completed` were offered on a particular turn, it must inspect that actual model request's dispatched function list after filtering. Inventory presence alone proves neither dispatch nor a successful review-tool call.
144144
>
145-
> **Note on builtin tool filtering:** The `source: "builtin"` entries in `tools[]` reflect the _instance-level superset_ registered in `ToolRegistry`. Per-model filters (e.g. `apply_patch` only on gpt-\*, `codesearch`/`websearch` only for aictrl provider) are applied at dispatch time inside `resolveTools` and are **not** reflected here. Consumers should treat the presence of a builtin tool name as "registered and potentially available", not as "guaranteed to appear in the model's function list". The strong structural guarantee (tool present ↔ tool in dispatch list) applies only to `source: "mcp"` entries.
145+
> **Inventory versus dispatch:** Builtin entries reflect the _instance-level superset_ registered in `ToolRegistry`; model-specific builtin filters are applied when resolving each request. MCP entries come from connected servers' resolved tool catalogs. Both sources can be excluded by effective work-tool selection, permissions or request-specific disabling. The shared MCP naming helper guarantees identifier consistency, not inclusion in the dispatched function list.
146146
147147
### `session_complete`
148148

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ Stdin is read only when a standalone `-` is a message operand, or when no messag
6363

6464
The marker can appear before or after `--`; repeated markers still read stdin once. Markers are removed from the model message, command arguments and derived session title. `--` ends option parsing, so `run -- -` also selects stdin. An operand such as `"literal -"` remains ordinary text. An explicit empty operand (`run ""`) does not select implicit stdin. Selected stdin is read to EOF and appended after one actual newline. Ordinary prompt operands are joined with spaces without adding artificial quotes or backslash escapes; quotes, backslashes and whitespace inside an operand remain its text.
6565

66-
Option values belong to their options: `--model=-` or `--command=-` does not select stdin, and scalar options also accept a separated hyphen value. Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported.
66+
Scalar required-value options own a separated hyphen: `--model -` and `--command -` do not select stdin, nor do their `=` forms. Optional `--port` leaves a separated `-` as the stdin operand (`--port -`). Use `--file=-` to attach a file literally named `-`; `--file -` is missing its required file value. Multiple file values and repeated `--skill` flags retain their existing behavior. Required option values are checked by the argument parser; explicit `--title ""` remains supported.
6767

6868
An explicit marker reads stdin even from a terminal. Any selected stream can wait for EOF indefinitely; this change adds no stdin timeout. Callers must close selected input or apply their own supervisor deadline.
6969

‎RELEASING.md‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,29 @@
1010
bun test test/cli/run-mcp-discovery.test.ts test/mcp/discovery-recovery.test.ts test/cli/publish-workflow.test.ts test/cli/run-provider-finish.test.ts test/cli/run-signal-cancellation.test.ts test/cli/classify-session-error.test.ts test/session/idle.test.ts test/session/processor-idle.test.ts
1111
```
1212

13+
Before publication, run the separate current-versus-previous native release
14+
comparison. Prepare and review explicit candidate and previous published binary
15+
artifacts, their SHA256/version/provenance pins and one existing immutable
16+
fixture image using [the manifest contract](script/token-budget/release-fixtures/README.md).
17+
Provision the pinned Python counter in a fresh directory (requires Python3.12
18+
and network access during setup only), then capture and count offline:
19+
20+
```bash
21+
python3.12 -I -B script/token-budget/setup.py --directory /absolute/new-counter-env
22+
bun run --no-install --no-env-file script/token-budget/release.ts /absolute/release-inputs.json /absolute/new-release-evidence
23+
/absolute/new-counter-env/venv/bin/python3 -I -B script/token-budget/release-count.py /absolute/new-release-evidence/campaign.json
24+
```
25+
26+
Run these commands from the repository root. Require valid current-default and
27+
previous-default task runs, raw requests/responses and actual Read-result
28+
evidence, verified versions/binary hashes, successful cleanup and
29+
`release-comparison.json`. Review first-input and session input/output/total
30+
deltas before publication and record any accepted regression with rationale.
31+
Minimal arms are measured only when each actual binary advertises `--tools`;
32+
an unsupported previous arm must be labeled UNSUPPORTED, never synthesized
33+
through permission settings. This scripted reference-token gate is separate
34+
from source CI token budgets and from provider billing or model-quality claims.
35+
1336
Wait for CI build, workspace typecheck and tests to pass and address reviews.
1437

1538
3. Merge the approved PR and record the resulting `main` commit SHA. Confirm
@@ -43,15 +66,15 @@
4366
publish workflow publishes each as a platform package. It then lists them in
4467
the wrapper's `optionalDependencies`:
4568

46-
| Package | `os` | `cpu` | `libc` |
47-
| --- | --- | --- | --- |
48-
| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc |
49-
| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl |
50-
| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc |
51-
| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl |
52-
| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | |
53-
| `@aictrl/cli-darwin-arm64` | darwin | arm64 | |
54-
| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | |
69+
| Package | `os` | `cpu` | `libc` |
70+
| ------------------------------------- | ------ | ----- | ------ |
71+
| `@aictrl/cli-linux-x64-baseline` | linux | x64 | glibc |
72+
| `@aictrl/cli-linux-x64-baseline-musl` | linux | x64 | musl |
73+
| `@aictrl/cli-linux-arm64` | linux | arm64 | glibc |
74+
| `@aictrl/cli-linux-arm64-musl` | linux | arm64 | musl |
75+
| `@aictrl/cli-darwin-x64-baseline` | darwin | x64 | |
76+
| `@aictrl/cli-darwin-arm64` | darwin | arm64 | |
77+
| `@aictrl/cli-windows-x64-baseline` | win32 | x64 | |
5578

5679
npm installs only the package that matches the host. Every x64 package is the
5780
baseline (no AVX2) build: Bun ships the same binary for its baseline and AVX2

‎packages/cli/README.md‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ Aictrl is built to be used in non-interactive environments.
104104
### Stdin Piping
105105

106106
```bash
107-
cat diff.txt | aictrl run "review this code change"
107+
cat diff.txt | aictrl run "review this code change" -
108108
```
109109

110110
### JSON Events
@@ -277,6 +277,25 @@ If your agents should commit or open PRs, supply your own procedure through a
277277
skill, `AGENTS.md` or the `instructions` config, and run the CLI in an isolated
278278
environment with credentials scoped to what the task needs.
279279

280+
## Project instruction isolation
281+
282+
Set `AICTRL_DISABLE_PROJECT_CONFIG=true` to disable automatic project instruction
283+
discovery as well as project configuration. This excludes root `AGENTS.md`,
284+
`.opencode/AGENTS.md`, and nested `AGENTS.md` files that would otherwise be
285+
attached when a tool reads a project file.
286+
287+
Trusted global instruction files (`~/.config/aictrl/AGENTS.md` and
288+
`AICTRL_CONFIG_DIR/AGENTS.md`) and explicit `instructions` entries in caller
289+
configuration remain available. Relative caller instructions resolve from
290+
`AICTRL_CONFIG_DIR` when project configuration is disabled; without that directory,
291+
relative entries are skipped. Absolute paths and URLs remain explicit inputs.
292+
293+
This controls automatic instruction discovery. Project files and tool output
294+
remain untrusted task data, so CI jobs still need scoped credentials, configured
295+
permissions, and process isolation. Keep any existing instruction-file deletion
296+
workaround until the installed release includes this behavior and has been
297+
verified.
298+
280299
## Local Development
281300

282301
```bash

‎packages/cli/src/session/instruction.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ function globalFiles() {
1919
files.push(path.join(Flag.AICTRL_CONFIG_DIR, "AGENTS.md"))
2020
}
2121
files.push(path.join(Global.Path.config, "AGENTS.md"))
22-
files.push(path.join(Instance.directory, ".opencode", "AGENTS.md"))
22+
if (!Flag.AICTRL_DISABLE_PROJECT_CONFIG) {
23+
files.push(path.join(Instance.directory, ".opencode", "AGENTS.md"))
24+
}
2325
return files
2426
}
2527

@@ -181,6 +183,7 @@ export namespace InstructionPrompt {
181183
}
182184

183185
export async function resolve(messages: MessageV2.WithParts[], filepath: string, messageID: string) {
186+
if (Flag.AICTRL_DISABLE_PROJECT_CONFIG) return []
184187
const system = await systemPaths()
185188
const already = loaded(messages)
186189
const results: { filepath: string; content: string }[] = []

‎packages/cli/test/cli/fixture/command-template-capture.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@ export async function capture(input: Input) {
2525
const ownedFile = input.ownedRedirect ? path.join(cwd, "target; /usr/bin/printf INERT_REDIRECT") : undefined
2626
if (ownedFile) await fs.mkdir(path.dirname(ownedFile), { recursive: true })
2727
const requests: Record<string, unknown>[] = []
28+
let requestArrivals = 0
2829
const server = Bun.serve({
2930
hostname: "127.0.0.1",
3031
port: 0,
3132
async fetch(request) {
33+
requestArrivals++
3234
if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected provider path")
3335
requests.push((await request.json()) as Record<string, unknown>)
3436
const chunks = [
@@ -165,7 +167,7 @@ export async function capture(input: Input) {
165167
stages.push(
166168
setTimeout(() => {
167169
eofAt = performance.now() - started
168-
requestsBeforeEOF = requests.length
170+
requestsBeforeEOF = requestArrivals
169171
child.stdin!.end()
170172
}, 100),
171173
)
@@ -249,6 +251,7 @@ export async function capture(input: Input) {
249251
firstBytesAt,
250252
eofAt,
251253
requestsBeforeEOF,
254+
requestArrivals,
252255
command,
253256
cwd,
254257
code,

0 commit comments

Comments
 (0)