Skip to content

Commit 2b576cf

Browse files
Dooyong KIMclaude
authored andcommitted
fix(broker): P0 stability — idle timeout + PID liveness + shutdown timeout
Wave 1 (SUP-366/367/368) — three fixes for known stability issues, addressing user-reported pain points around orphan brokers, zombie running jobs, and SessionEnd hangs. Changes: 1. app-server-broker.mjs (SUP-366, cc#108): - 10 min idle auto-shutdown (configurable via CODEX_BROKER_IDLE_MS env) - Fires when sockets.size === 0 for the timeout window - Defense-in-depth alongside the SessionEnd hook - Pattern adapted from sanghyun-io/codex-app-server-plugin 2. lib/job-control.mjs (SUP-367, cc#264/openai#164/openai#202/openai#222): - PID liveness check via process.kill(pid, 0) - Auto-transition status:running -> crashed when worker is gone - Tail last 3 log lines into errorMessage for diagnostics 3. lib/broker-lifecycle.mjs (SUP-368, cc#245/openai#288, PR#293 reference): - 5 s safety timeout in sendBrokerShutdown - Prevents SessionEnd hook from hanging on unresponsive broker - timer.unref() so it doesn't block process exit INV-1 (no regression of existing /codex:* commands): preserved. INV-2 (clean upstream rebase): minimum diff to existing files. INV-3 (supalead skill compatibility): plugin/marketplace name unchanged. INV-4 (no CLAUDE.md import): not added. Refs Linear SUP-366, SUP-367, SUP-368 Project: codex-plugin-cc-plus fork (codex 안정성 + A+ delegate) Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent b0703c1 commit 2b576cf

3 files changed

Lines changed: 94 additions & 4 deletions

File tree

‎plugins/codex/scripts/app-server-broker.mjs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@ import { parseBrokerEndpoint } from "./lib/broker-endpoint.mjs";
1111

1212
const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact/start"]);
1313

14+
// Idle auto-shutdown — defense-in-depth against orphan brokers when SessionEnd hook fails.
15+
// Refs: cc#108, cc#163, cc#193. Pattern adapted from sanghyun-io/codex-app-server-plugin
16+
// `bin/broker.mjs:65, 491-499`.
17+
const DEFAULT_BROKER_IDLE_MS = 10 * 60 * 1000;
18+
const BROKER_IDLE_MS = (() => {
19+
const raw = Number.parseInt(process.env.CODEX_BROKER_IDLE_MS ?? "", 10);
20+
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_BROKER_IDLE_MS;
21+
})();
22+
1423
function buildStreamThreadIds(method, params, result) {
1524
const threadIds = new Set();
1625
if (params?.threadId) {
@@ -71,6 +80,24 @@ async function main() {
7180
let activeStreamThreadIds = null;
7281
const sockets = new Set();
7382

83+
// Idle shutdown timer — fires when no client sockets are active for BROKER_IDLE_MS.
84+
// Cancelled while any socket is connected. (Refs: cc#108)
85+
let idleTimer = null;
86+
function resetIdleTimer() {
87+
if (idleTimer) {
88+
clearTimeout(idleTimer);
89+
idleTimer = null;
90+
}
91+
if (sockets.size === 0) {
92+
idleTimer = setTimeout(async () => {
93+
process.stderr.write(`[broker] Idle timeout (${BROKER_IDLE_MS}ms) — shutting down\n`);
94+
await shutdown(server).catch(() => {});
95+
process.exit(0);
96+
}, BROKER_IDLE_MS);
97+
idleTimer.unref?.();
98+
}
99+
}
100+
74101
function clearSocketOwnership(socket) {
75102
if (activeRequestSocket === socket) {
76103
activeRequestSocket = null;
@@ -117,6 +144,7 @@ async function main() {
117144

118145
const server = net.createServer((socket) => {
119146
sockets.add(socket);
147+
resetIdleTimer();
120148
socket.setEncoding("utf8");
121149
let buffer = "";
122150

@@ -225,11 +253,13 @@ async function main() {
225253
socket.on("close", () => {
226254
sockets.delete(socket);
227255
clearSocketOwnership(socket);
256+
resetIdleTimer();
228257
});
229258

230259
socket.on("error", () => {
231260
sockets.delete(socket);
232261
clearSocketOwnership(socket);
262+
resetIdleTimer();
233263
});
234264
});
235265

@@ -244,6 +274,7 @@ async function main() {
244274
});
245275

246276
server.listen(listenTarget.path);
277+
resetIdleTimer();
247278
}
248279

249280
main().catch((error) => {

‎plugins/codex/scripts/lib/broker-lifecycle.mjs‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,19 +40,30 @@ export async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) {
4040
return false;
4141
}
4242

43-
export async function sendBrokerShutdown(endpoint) {
43+
// 5 s safety timeout — prevents SessionEnd hook from hanging when the broker is
44+
// unresponsive (e.g. SIGSTOP'd or stuck). Pattern matches `waitForBrokerEndpoint`.
45+
// Refs: cc#245, cc#288, cc PR#293.
46+
const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5000;
47+
48+
export async function sendBrokerShutdown(endpoint, timeoutMs = DEFAULT_SHUTDOWN_TIMEOUT_MS) {
4449
await new Promise((resolve) => {
50+
const timer = setTimeout(() => resolve(), timeoutMs);
51+
timer.unref?.();
52+
const finish = () => {
53+
clearTimeout(timer);
54+
resolve();
55+
};
4556
const socket = connectToEndpoint(endpoint);
4657
socket.setEncoding("utf8");
4758
socket.on("connect", () => {
4859
socket.write(`${JSON.stringify({ id: 1, method: "broker/shutdown", params: {} })}\n`);
4960
});
5061
socket.on("data", () => {
5162
socket.end();
52-
resolve();
63+
finish();
5364
});
54-
socket.on("error", resolve);
55-
socket.on("close", resolve);
65+
socket.on("error", finish);
66+
socket.on("close", finish);
5667
});
5768
}
5869

‎plugins/codex/scripts/lib/job-control.mjs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,36 @@ import { resolveWorkspaceRoot } from "./workspace.mjs";
88
export const DEFAULT_MAX_STATUS_JOBS = 8;
99
export const DEFAULT_MAX_PROGRESS_LINES = 4;
1010

11+
// PID liveness check — `kill -0 <pid>` semantics (signal 0 just probes).
12+
// Refs: cc#264 (status stuck after task_complete), cc#164/#202/#222 (zombie running jobs).
13+
// Pattern adapted from sanghyun-io/codex-app-server-plugin `bin/codex-review.mjs:1003-1020`.
14+
function checkProcessAlive(pid) {
15+
if (pid == null) return false;
16+
const num = Number(pid);
17+
if (!Number.isFinite(num) || num <= 0) return false;
18+
try {
19+
process.kill(num, 0);
20+
return true;
21+
} catch {
22+
return false;
23+
}
24+
}
25+
26+
function tailLogLines(logFile, count = 3) {
27+
if (!logFile || !fs.existsSync(logFile)) return "";
28+
try {
29+
return fs
30+
.readFileSync(logFile, "utf8")
31+
.split(/\r?\n/)
32+
.map((line) => line.trim())
33+
.filter(Boolean)
34+
.slice(-count)
35+
.join(" | ");
36+
} catch {
37+
return "";
38+
}
39+
}
40+
1141
export function sortJobsNewestFirst(jobs) {
1242
return [...jobs].sort((left, right) => String(right.updatedAt ?? "").localeCompare(String(left.updatedAt ?? "")));
1343
}
@@ -174,6 +204,24 @@ export function enrichJob(job, options = {}) {
174204
: null
175205
};
176206

207+
// Detect zombie jobs — status says running/queued but the worker PID is gone.
208+
// Auto-transitions to `crashed` and surfaces last log lines so callers don't
209+
// wait forever on a dead worker. (Refs: cc#264/#164/#202/#222)
210+
if (
211+
(enriched.status === "running" || enriched.status === "queued") &&
212+
enriched.pid &&
213+
!checkProcessAlive(enriched.pid)
214+
) {
215+
let errorMessage = `Worker process exited unexpectedly (PID ${enriched.pid} no longer alive).`;
216+
const tail = tailLogLines(enriched.logFile, 3);
217+
if (tail) {
218+
errorMessage += ` Last log: ${tail}`;
219+
}
220+
enriched.status = "crashed";
221+
enriched.phase = "crashed";
222+
enriched.errorMessage = enriched.errorMessage ?? errorMessage;
223+
}
224+
177225
return {
178226
...enriched,
179227
phase: enriched.phase ?? inferLegacyJobPhase(enriched, enriched.progressPreview)

0 commit comments

Comments
 (0)