Skip to content

Commit e19c574

Browse files
committed
fix(broker): shut down the app-server broker after an idle timeout
A broker outlives the client that spawned it. Nothing in the protocol tells it the client is gone for good, so a spawned broker stays resident forever holding its socket dir -- roughly 40MB of RSS each, until the machine reboots. This is reproducible from this repo's own test suite. A single `node --test tests/runtime.test.mjs` run leaves ~58 brokers behind: each test creates a fresh temp workspace, ensureBrokerSession spawns a broker per workspace, and nothing ever tears them down. Three suite runs on one machine left 174 idle brokers alive, all at 00:00:00 CPU with no connections, holding ~2.9GB of resident memory between them. The broker now shuts itself down after being idle for --idle-timeout milliseconds (default 30 minutes, overridable with CODEX_COMPANION_BROKER_IDLE_MS, 0 to disable). Idle means no connected sockets and nothing in flight, so an open connection -- including a long streaming turn -- always holds it open. The decision is made by the broker rather than by an external sweep because the broker is the only party that can observe whether it is serving anyone. Scanning the job registry from outside and then killing is racy by construction: a job can start between the scan and the kill. An idle exit leaves broker.json behind, which is already handled -- ensureBrokerSession probes the recorded endpoint and, when it does not answer, tears the session down and respawns. The broker deliberately does not clear broker.json itself: a replacement broker may already have rewritten that record, and clearing it would delete the live broker's endpoint. Blank and whitespace-only values fall back to the default rather than parsing as 0, so idle shutdown can only be disabled explicitly.
1 parent db52e28 commit e19c574

2 files changed

Lines changed: 227 additions & 3 deletions

File tree

‎plugins/codex/scripts/app-server-broker.mjs‎

Lines changed: 74 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,33 @@ import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs";
1111

1212
const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]);
1313

14+
// A broker outlives the client that spawned it: nothing in the protocol tells it the
15+
// client is gone for good, so without this it stays resident forever holding its socket
16+
// dir. Idle shutdown is decided BY THE BROKER because it is the only party that can see
17+
// whether it is serving anyone -- an external sweep cannot, and racing one is unsafe.
18+
const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
19+
const IDLE_TIMEOUT_ENV = "CODEX_COMPANION_BROKER_IDLE_MS";
20+
21+
function resolveIdleTimeoutMs(rawOption, env = {}) {
22+
const raw = rawOption ?? env[IDLE_TIMEOUT_ENV];
23+
// Trim before the emptiness test: Number(" ") is 0, so a blank or whitespace-only
24+
// value would otherwise DISABLE idle shutdown silently. Explicit "0" is the only
25+
// way to turn it off; anything blank falls back to the default.
26+
const text = raw === undefined || raw === null ? "" : String(raw).trim();
27+
if (text === "") {
28+
return DEFAULT_IDLE_TIMEOUT_MS;
29+
}
30+
const parsed = Number(text);
31+
// Reject rather than silently falling back: a typo'd timeout that quietly became
32+
// "never expire" would reintroduce the exact leak this exists to close.
33+
if (!Number.isFinite(parsed) || parsed < 0) {
34+
throw new Error(
35+
`Invalid idle timeout ${JSON.stringify(text)}: expected a non-negative number of milliseconds.`
36+
);
37+
}
38+
return parsed;
39+
}
40+
1441
function buildStreamThreadIds(method, params, result) {
1542
const threadIds = new Set();
1643
if (params?.threadId) {
@@ -48,11 +75,13 @@ function writePidFile(pidFile) {
4875
async function main() {
4976
const [subcommand, ...argv] = process.argv.slice(2);
5077
if (subcommand !== "serve") {
51-
throw new Error("Usage: node scripts/app-server-broker.mjs serve --endpoint <value> [--cwd <path>] [--pid-file <path>]");
78+
throw new Error(
79+
"Usage: node scripts/app-server-broker.mjs serve --endpoint <value> [--cwd <path>] [--pid-file <path>] [--idle-timeout <ms>]"
80+
);
5281
}
5382

5483
const { options } = parseArgs(argv, {
55-
valueOptions: ["cwd", "pid-file", "endpoint"]
84+
valueOptions: ["cwd", "pid-file", "endpoint", "idle-timeout"]
5685
});
5786

5887
if (!options.endpoint) {
@@ -63,13 +92,50 @@ async function main() {
6392
const endpoint = String(options.endpoint);
6493
const listenTarget = parseBrokerEndpoint(endpoint);
6594
const pidFile = options["pid-file"] ? path.resolve(options["pid-file"]) : null;
95+
const idleTimeoutMs = resolveIdleTimeoutMs(options["idle-timeout"], process.env);
6696
writePidFile(pidFile);
6797

6898
const appClient = await CodexAppServerClient.connect(cwd, { disableBroker: true });
6999
let activeRequestSocket = null;
70100
let activeStreamSocket = null;
71101
let activeStreamThreadIds = null;
72102
const sockets = new Set();
103+
let idleTimer = null;
104+
105+
// Idle means nobody is connected AND nothing is in flight. Holding an open socket is
106+
// enough to keep the broker alive, so a long streaming turn can never be cut short.
107+
function isIdle() {
108+
return sockets.size === 0 && activeRequestSocket === null && activeStreamSocket === null;
109+
}
110+
111+
function disarmIdleTimer() {
112+
if (idleTimer) {
113+
clearTimeout(idleTimer);
114+
idleTimer = null;
115+
}
116+
}
117+
118+
function armIdleTimer() {
119+
disarmIdleTimer();
120+
if (idleTimeoutMs <= 0 || !isIdle()) {
121+
return;
122+
}
123+
idleTimer = setTimeout(() => {
124+
idleTimer = null;
125+
// Re-check at fire time: a client may have connected while the timer was pending.
126+
if (!isIdle()) {
127+
armIdleTimer();
128+
return;
129+
}
130+
void shutdown(server).then(
131+
() => process.exit(0),
132+
() => process.exit(0)
133+
);
134+
}, idleTimeoutMs);
135+
// The listening server keeps the event loop alive; the timer must not do so itself,
136+
// or a broker with idle shutdown disabled could never exit cleanly.
137+
idleTimer.unref();
138+
}
73139

74140
function clearSocketOwnership(socket) {
75141
if (activeRequestSocket === socket) {
@@ -116,6 +182,7 @@ async function main() {
116182
appClient.setNotificationHandler(routeNotification);
117183

118184
const server = net.createServer((socket) => {
185+
disarmIdleTimer();
119186
sockets.add(socket);
120187
socket.setEncoding("utf8");
121188
let buffer = "";
@@ -225,11 +292,13 @@ async function main() {
225292
socket.on("close", () => {
226293
sockets.delete(socket);
227294
clearSocketOwnership(socket);
295+
armIdleTimer();
228296
});
229297

230298
socket.on("error", () => {
231299
sockets.delete(socket);
232300
clearSocketOwnership(socket);
301+
armIdleTimer();
233302
});
234303
});
235304

@@ -243,7 +312,9 @@ async function main() {
243312
process.exit(0);
244313
});
245314

246-
server.listen(listenTarget.path);
315+
server.listen(listenTarget.path, () => {
316+
armIdleTimer();
317+
});
247318
}
248319

249320
main().catch((error) => {

‎tests/broker-idle.test.mjs‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
import assert from "node:assert/strict";
2+
import fs from "node:fs";
3+
import net from "node:net";
4+
import path from "node:path";
5+
import test from "node:test";
6+
import { spawn } from "node:child_process";
7+
import { fileURLToPath } from "node:url";
8+
9+
import { makeTempDir } from "./helpers.mjs";
10+
import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs";
11+
12+
const ROOT = path.resolve(fileURLToPath(new URL("..", import.meta.url)));
13+
const BROKER = path.join(ROOT, "plugins", "codex", "scripts", "app-server-broker.mjs");
14+
15+
function delay(ms) {
16+
return new Promise((resolve) => setTimeout(resolve, ms));
17+
}
18+
19+
async function waitFor(predicate, { timeoutMs = 10000, intervalMs = 25 } = {}) {
20+
const start = Date.now();
21+
while (Date.now() - start < timeoutMs) {
22+
if (await predicate()) {
23+
return true;
24+
}
25+
await delay(intervalMs);
26+
}
27+
return false;
28+
}
29+
30+
function startBroker({ idleTimeout } = {}) {
31+
const binDir = makeTempDir();
32+
installFakeCodex(binDir);
33+
const sessionDir = makeTempDir("codex-broker-idle-");
34+
const cwd = makeTempDir("codex-broker-cwd-");
35+
const socketPath = path.join(sessionDir, "broker.sock");
36+
const pidFile = path.join(sessionDir, "broker.pid");
37+
38+
const args = [BROKER, "serve", "--endpoint", `unix:${socketPath}`, "--cwd", cwd, "--pid-file", pidFile];
39+
if (idleTimeout !== undefined) {
40+
args.push("--idle-timeout", String(idleTimeout));
41+
}
42+
43+
const child = spawn(process.execPath, args, { env: buildEnv(binDir), stdio: ["ignore", "pipe", "pipe"] });
44+
let stderr = "";
45+
child.stderr.on("data", (chunk) => {
46+
stderr += chunk;
47+
});
48+
49+
const exited = new Promise((resolve) => {
50+
child.on("exit", (code, signal) => resolve({ code, signal }));
51+
});
52+
53+
const alive = () => child.exitCode === null && child.signalCode === null;
54+
55+
// Bounded: a broker that never exits must turn into a RED test, not a hung run.
56+
// Awaiting `exited` unbounded is what made this file hang against an unpatched broker.
57+
const exitedWithin = (timeoutMs) =>
58+
Promise.race([exited, delay(timeoutMs).then(() => null)]);
59+
60+
const dispose = () => {
61+
if (alive()) {
62+
child.kill("SIGKILL");
63+
}
64+
};
65+
66+
return {
67+
child,
68+
socketPath,
69+
pidFile,
70+
exited,
71+
exitedWithin,
72+
dispose,
73+
stderr: () => stderr,
74+
alive,
75+
listening: () => waitFor(() => fs.existsSync(socketPath))
76+
};
77+
}
78+
79+
function connectTo(socketPath) {
80+
return new Promise((resolve, reject) => {
81+
const socket = net.createConnection({ path: socketPath });
82+
socket.on("connect", () => resolve(socket));
83+
socket.on("error", reject);
84+
});
85+
}
86+
87+
test("broker shuts itself down once it has been idle for the timeout", async (t) => {
88+
const broker = startBroker({ idleTimeout: 400 });
89+
t.after(() => broker.dispose());
90+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
91+
92+
const result = await broker.exitedWithin(8000);
93+
assert.ok(result, `broker never exited after the idle timeout, stderr: ${broker.stderr()}`);
94+
assert.equal(result.code, 0, `expected a clean idle exit, stderr: ${broker.stderr()}`);
95+
// shutdown() must still clean up after itself on the idle path, or the next
96+
// ensureBrokerSession would find a stale socket and a stale pidfile.
97+
assert.equal(fs.existsSync(broker.socketPath), false, "idle shutdown left the socket behind");
98+
assert.equal(fs.existsSync(broker.pidFile), false, "idle shutdown left the pidfile behind");
99+
});
100+
101+
test("broker stays alive while a client is connected, then exits after it disconnects", async (t) => {
102+
const broker = startBroker({ idleTimeout: 400 });
103+
t.after(() => broker.dispose());
104+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
105+
106+
const socket = await connectTo(broker.socketPath);
107+
// Well past the idle timeout: an open connection must hold the broker open, which is
108+
// what stops a long streaming turn from being cut off mid-flight.
109+
await delay(1600);
110+
assert.equal(broker.alive(), true, "broker exited while a client was still connected");
111+
112+
socket.destroy();
113+
const result = await broker.exitedWithin(8000);
114+
assert.ok(result, `broker never exited after the client disconnected, stderr: ${broker.stderr()}`);
115+
assert.equal(result.code, 0, `expected a clean idle exit after disconnect, stderr: ${broker.stderr()}`);
116+
});
117+
118+
test("broker with --idle-timeout 0 never idles out", async (t) => {
119+
const broker = startBroker({ idleTimeout: 0 });
120+
t.after(() => broker.dispose());
121+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
122+
123+
await delay(1600);
124+
assert.equal(broker.alive(), true, "idle shutdown ran even though it was disabled");
125+
126+
broker.child.kill("SIGTERM");
127+
await broker.exitedWithin(8000);
128+
});
129+
130+
test("broker rejects a non-numeric --idle-timeout instead of silently never expiring", async (t) => {
131+
const broker = startBroker({ idleTimeout: "not-a-number" });
132+
t.after(() => broker.dispose());
133+
const result = await broker.exitedWithin(8000);
134+
assert.ok(result, "broker did not exit on an invalid idle timeout");
135+
assert.equal(result.code, 1);
136+
assert.match(broker.stderr(), /Invalid idle timeout/);
137+
});
138+
139+
test("blank --idle-timeout falls back to the default instead of silently disabling", async (t) => {
140+
// Number(" ") === 0, so a whitespace value must NOT be read as "never expire".
141+
// Only an explicit 0 disables idle shutdown.
142+
const broker = startBroker({ idleTimeout: " " });
143+
t.after(() => broker.dispose());
144+
assert.equal(await broker.listening(), true, `broker never listened: ${broker.stderr()}`);
145+
146+
// The default is 30 minutes, so it must still be alive well past the short timeouts
147+
// used elsewhere in this file.
148+
await delay(1600);
149+
assert.equal(broker.alive(), true, "a blank idle timeout disabled idle shutdown");
150+
151+
broker.child.kill("SIGTERM");
152+
await broker.exitedWithin(8000);
153+
});

0 commit comments

Comments
 (0)