|
| 1 | +import path from "node:path" |
| 2 | +import fs from "node:fs/promises" |
| 3 | +import { guard } from "./guard" |
| 4 | +const out = process.env.TOKEN_BUDGET_OUTPUT |
| 5 | +const work = process.env.TOKEN_BUDGET_WORKSPACE |
| 6 | +const source = process.env.TOKEN_BUDGET_SOURCE |
| 7 | +if (!out || !work || !source) throw new Error("Explicit output/workspace/source required") |
| 8 | +const binpath = [path.dirname(process.execPath), "/usr/local/bin", "/usr/bin", "/bin"].join(path.delimiter) |
| 9 | +const rg = Bun.which("rg", { PATH: binpath }) |
| 10 | +if (!rg) throw new Error("Real ripgrep prerequisite missing") |
| 11 | +const requests: string[] = [] |
| 12 | +const server = Bun.serve({ |
| 13 | + hostname: "127.0.0.1", |
| 14 | + port: 0, |
| 15 | + idleTimeout: 0, |
| 16 | + async fetch(request) { |
| 17 | + if (new URL(request.url).pathname !== "/v1/chat/completions") throw new Error("Unexpected endpoint " + request.url) |
| 18 | + if (request.method !== "POST") throw new Error("Expected POST") |
| 19 | + requests.push(await request.text()) |
| 20 | + const chunks = [ |
| 21 | + { |
| 22 | + id: "eval", |
| 23 | + object: "chat.completion.chunk", |
| 24 | + choices: [{ index: 0, delta: { role: "assistant", content: "Fixture complete" } }], |
| 25 | + }, |
| 26 | + { id: "eval", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }, |
| 27 | + ] |
| 28 | + return new Response( |
| 29 | + chunks |
| 30 | + .map((chunk) => "data: " + JSON.stringify(chunk)) |
| 31 | + .concat("data: [DONE]") |
| 32 | + .join("\n\n") + "\n\n", |
| 33 | + { headers: { "content-type": "text/event-stream" } }, |
| 34 | + ) |
| 35 | + }, |
| 36 | +}) |
| 37 | +await fs.mkdir(out, { recursive: true }) |
| 38 | +await Bun.write(path.join(out, "guard.ts"), guard) |
| 39 | +await Bun.write(path.join(out, "network.jsonl"), "") |
| 40 | +const config = { |
| 41 | + enabled_providers: ["fixture"], |
| 42 | + provider: { |
| 43 | + fixture: { |
| 44 | + npm: "@ai-sdk/openai-compatible", |
| 45 | + options: { apiKey: "fixture", baseURL: server.url.origin + "/v1" }, |
| 46 | + models: { fixture: { name: "fixture", tool_call: true, limit: { context: 200000, output: 1000 } } }, |
| 47 | + }, |
| 48 | + }, |
| 49 | + agent: { title: { disable: true } }, |
| 50 | + permission: { "*": "allow" }, |
| 51 | +} |
| 52 | +const command = [ |
| 53 | + process.execPath, |
| 54 | + "run", |
| 55 | + "--no-install", |
| 56 | + "--no-env-file", |
| 57 | + "--preload", |
| 58 | + path.join(import.meta.dir, "clock.js"), |
| 59 | + "--preload", |
| 60 | + path.join(out, "guard.ts"), |
| 61 | + "--conditions=browser", |
| 62 | + source, |
| 63 | + "run", |
| 64 | + "--model", |
| 65 | + "fixture/fixture", |
| 66 | + "--agent", |
| 67 | + "build", |
| 68 | + "--format", |
| 69 | + "json", |
| 70 | + ...(process.env.TOKEN_BUDGET_ARM === "four" ? ["--tools", "read,bash,edit,write"] : []), |
| 71 | + "Say Fixture complete without tools.", |
| 72 | +] |
| 73 | +const env = { |
| 74 | + PATH: binpath, |
| 75 | + SHELL: "/bin/bash", |
| 76 | + TZ: "UTC", |
| 77 | + HOME: path.join(out, "home"), |
| 78 | + AICTRL_TEST_HOME: path.join(out, "home"), |
| 79 | + AICTRL_TEST_MANAGED_CONFIG_DIR: path.join(out, "managed"), |
| 80 | + XDG_DATA_HOME: path.join(out, "data"), |
| 81 | + XDG_CONFIG_HOME: path.join(out, "config"), |
| 82 | + XDG_STATE_HOME: path.join(out, "state"), |
| 83 | + XDG_CACHE_HOME: path.join(out, "cache"), |
| 84 | + BUN_RUNTIME_TRANSPILER_CACHE_PATH: path.join(out, "bun-cache"), |
| 85 | + GIT_CONFIG_NOSYSTEM: "1", |
| 86 | + GIT_CONFIG_GLOBAL: "/dev/null", |
| 87 | + AICTRL_DISABLE_PROJECT_CONFIG: "true", |
| 88 | + AICTRL_DISABLE_DEFAULT_PLUGINS: "true", |
| 89 | + AICTRL_DISABLE_AUTOCOMPACT: "true", |
| 90 | + AICTRL_MODEL_STREAM_IDLE_TIMEOUT_MS: "3000", |
| 91 | + AICTRL_CONFIG_CONTENT: JSON.stringify(config), |
| 92 | + SELECTOR_FIXTURE_ORIGIN: server.url.origin, |
| 93 | + SELECTOR_FIXTURE_GUARD_LOG: path.join(out, "network.jsonl"), |
| 94 | +} |
| 95 | +const child = Bun.spawn(command, { cwd: work, env, stdin: "ignore", stdout: "pipe", stderr: "pipe" }) |
| 96 | +let deadline = false |
| 97 | +const timer = setTimeout(() => { |
| 98 | + deadline = true |
| 99 | + child.kill("SIGKILL") |
| 100 | +}, 15000) |
| 101 | +try { |
| 102 | + const [stdout, stderr, exit] = await Promise.all([ |
| 103 | + new Response(child.stdout).text(), |
| 104 | + new Response(child.stderr).text(), |
| 105 | + child.exited, |
| 106 | + ]) |
| 107 | + clearTimeout(timer) |
| 108 | + await Bun.write(path.join(out, "stdout.jsonl"), stdout) |
| 109 | + await Bun.write(path.join(out, "stderr.log"), stderr) |
| 110 | + for (const [index, body] of requests.entries()) await Bun.write(path.join(out, `request-${index}.json`), body) |
| 111 | + const network = (await Bun.file(path.join(out, "network.jsonl")).text()) |
| 112 | + .split("\n") |
| 113 | + .filter(Boolean) |
| 114 | + .map((line) => JSON.parse(line)) |
| 115 | + const result = { exit, deadline, command, environment: env, rg, requests: requests.length, network } |
| 116 | + await Bun.write(path.join(out, "result.json"), JSON.stringify(result, null, 2) + "\n") |
| 117 | + if (deadline || exit !== 0 || requests.length !== 1 || network.some((row) => !row.allowed)) |
| 118 | + throw new Error( |
| 119 | + "Capture failed: " + JSON.stringify({ exit, deadline, requests: requests.length, network }) + stderr.slice(-2000), |
| 120 | + ) |
| 121 | + const events = stdout |
| 122 | + .split("\n") |
| 123 | + .filter(Boolean) |
| 124 | + .map((line) => JSON.parse(line)) |
| 125 | + if ( |
| 126 | + events |
| 127 | + .filter((event) => event.type === "text") |
| 128 | + .map((event) => event.part?.text) |
| 129 | + .join("") !== "Fixture complete" |
| 130 | + ) |
| 131 | + throw new Error("Exact assistant response missing") |
| 132 | + if ( |
| 133 | + events.filter((event) => event.type === "invocation_complete" && event.status === "completed").length !== 1 || |
| 134 | + events.some((event) => event.type === "error" || event.type === "tool_use") |
| 135 | + ) |
| 136 | + throw new Error("Unexpected invocation lifecycle") |
| 137 | +} catch (error) { |
| 138 | + console.error(String(error)) |
| 139 | + process.exitCode = 1 |
| 140 | +} finally { |
| 141 | + clearTimeout(timer) |
| 142 | + child.kill("SIGKILL") |
| 143 | + server.stop(true) |
| 144 | +} |
0 commit comments