Skip to content

Commit 8be4ab3

Browse files
Jafeegeoff-uxclaude
committed
feat: harden Codex turn lifecycle — bound stalls, interrupt abandoned turns, idle broker shutdown (openai#361)
Reconcile upstream PR openai#361 into the already-merged openai#302 wall-clock timeouts. Replace openai#302's captureTurn idle timer (armIdle / CODEX_TURN_IDLE_TIMEOUT_MS) with openai#361's Promise.race([work, ceiling, stall, exit]) + interruptTurnBestEffort. This also covers a start RPC that hangs or rejects before any ACK, and interrupts an abandoned turn upstream so it cannot overlap a retry. Keep openai#302's app-server request() wall-clock timeout untouched (complementary transport-layer guard). Add broker idle self-shutdown (CODEX_COMPANION_BROKER_IDLE_MS, default 30m) and onSocketGone teardown so an orphaned broker/turn self-heals instead of leaking; close the listener before the upstream app-server on shutdown. Surface Codex invocation failures verbatim instead of swallowing them into an empty response. openai#361 supersedes openai#312 (the per-turn watchdog openai#312 attempted, implemented correctly here). Dual security review (Opus + Codex gpt-5.5 xhigh): both APPROVE, all 9 red lines clear, 0 security findings. Converged LOW non-blockers: onSocketGone error+close double-event is idempotent but untested; CEILING_MS backstop untested; openai#355 background-job interaction noted (not merged). Full suite 98/98 in a clean env. Upstream PR: openai#361 Co-Authored-By: geoff-ux <[email protected]> Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent c7d071a commit 8be4ab3

7 files changed

Lines changed: 388 additions & 81 deletions

File tree

‎plugins/codex/agents/codex-rescue.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ Forwarding rules:
3939
- Otherwise forward the task as a fresh `task` run.
4040
- Preserve the user's task text as-is apart from stripping routing flags.
4141
- Return the stdout of the `codex-companion` command exactly as-is.
42-
- If the Bash call fails or Codex cannot be invoked, return nothing.
42+
- If the Bash call fails or Codex cannot be invoked, return its error output (stderr and any stdout) verbatim so the failure is visible — never swallow it into an empty response.
4343

4444
Response style:
4545

‎plugins/codex/scripts/app-server-broker.mjs‎

Lines changed: 74 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,35 @@ async function main() {
6969
let activeRequestSocket = null;
7070
let activeStreamSocket = null;
7171
let activeStreamThreadIds = null;
72+
// The socket whose request is currently awaiting an upstream response. If that
73+
// socket disconnects before the response arrives (e.g. its turn watchdog fired
74+
// on a start that never ACKed), the upstream work is orphaned — see onSocketGone.
75+
let pendingUpstreamSocket = null;
7276
const sockets = new Set();
7377

78+
// Idle self-shutdown: the broker is keyed per-cwd and is only reaped on a
79+
// later ensureBrokerSession for the SAME cwd. A broker for a cwd that is never
80+
// revisited (deleted worktree, ended session) would otherwise run forever and
81+
// accumulate as orphaned processes. Exit after IDLE_MS with no connected
82+
// clients so the leak self-heals; the companion transparently respawns one on
83+
// the next task. Env-overridable for long idle gaps.
84+
const IDLE_MS = Number(process.env.CODEX_COMPANION_BROKER_IDLE_MS) || 1_800_000;
85+
let idleTimer = null;
86+
function armIdle() {
87+
if (idleTimer) {
88+
clearTimeout(idleTimer);
89+
}
90+
idleTimer = setTimeout(async () => {
91+
if (sockets.size > 0 || activeRequestSocket || activeStreamSocket) {
92+
armIdle();
93+
return;
94+
}
95+
await shutdown(server);
96+
process.exit(0);
97+
}, IDLE_MS);
98+
idleTimer.unref?.();
99+
}
100+
74101
function clearSocketOwnership(socket) {
75102
if (activeRequestSocket === socket) {
76103
activeRequestSocket = null;
@@ -81,6 +108,28 @@ async function main() {
81108
}
82109
}
83110

111+
function onSocketGone(socket) {
112+
sockets.delete(socket);
113+
// The client that owned in-progress upstream work vanished — either a
114+
// request still in flight (a start that never returned), or a streaming turn
115+
// that ACKed but has not reached turn/completed (e.g. the companion crashed
116+
// or was killed before it could interrupt). That turn is now orphaned and
117+
// may keep running. The broker is single-flight, so there is no other
118+
// in-flight work to preserve: tear the broker down (closing the upstream
119+
// app-server) so the abandoned turn cannot keep mutating the workspace or
120+
// collide with the next task, which reconnects to a fresh runtime. A turn
121+
// that reached turn/completed (or was interrupted) has already cleared
122+
// activeStreamSocket via routeNotification, so a normal close does not
123+
// trigger this.
124+
if (socket === pendingUpstreamSocket || socket === activeStreamSocket) {
125+
pendingUpstreamSocket = null;
126+
shutdown(server).finally(() => process.exit(0));
127+
return;
128+
}
129+
clearSocketOwnership(socket);
130+
armIdle();
131+
}
132+
84133
function routeNotification(message) {
85134
const target = activeRequestSocket ?? activeStreamSocket;
86135
if (!target) {
@@ -103,8 +152,14 @@ async function main() {
103152
for (const socket of sockets) {
104153
socket.end();
105154
}
106-
await appClient.close().catch(() => {});
155+
// Stop accepting new connections BEFORE closing the upstream app-server. On
156+
// an idle self-shutdown a new task may connect at the teardown boundary;
157+
// closing the listener first means that client is either served by the
158+
// still-open appClient (during drain) or refused with ECONNREFUSED — which
159+
// the companion retries into a fresh broker — rather than accepted and then
160+
// failed against a closed app-server (an error withAppServer does not retry).
107161
await new Promise((resolve) => server.close(resolve));
162+
await appClient.close().catch(() => {});
108163
if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) {
109164
fs.unlinkSync(listenTarget.path);
110165
}
@@ -117,10 +172,12 @@ async function main() {
117172

118173
const server = net.createServer((socket) => {
119174
sockets.add(socket);
175+
armIdle();
120176
socket.setEncoding("utf8");
121177
let buffer = "";
122178

123179
socket.on("data", async (chunk) => {
180+
armIdle();
124181
buffer += chunk;
125182
let newlineIndex = buffer.indexOf("\n");
126183
while (newlineIndex !== -1) {
@@ -182,6 +239,7 @@ async function main() {
182239
}
183240

184241
if (allowInterruptDuringActiveStream) {
242+
pendingUpstreamSocket = socket;
185243
try {
186244
const result = await appClient.request(message.method, message.params ?? {});
187245
send(socket, { id: message.id, result });
@@ -190,12 +248,17 @@ async function main() {
190248
id: message.id,
191249
error: buildJsonRpcError(error.rpcCode ?? -32000, error.message)
192250
});
251+
} finally {
252+
if (pendingUpstreamSocket === socket) {
253+
pendingUpstreamSocket = null;
254+
}
193255
}
194256
continue;
195257
}
196258

197259
const isStreaming = STREAMING_METHODS.has(message.method);
198260
activeRequestSocket = socket;
261+
pendingUpstreamSocket = socket;
199262

200263
try {
201264
const result = await appClient.request(message.method, message.params ?? {});
@@ -218,19 +281,20 @@ async function main() {
218281
if (activeStreamSocket === socket && !isStreaming) {
219282
activeStreamSocket = null;
220283
}
284+
} finally {
285+
// The request settled (ACK for a streaming turn, or a terminal result):
286+
// it is no longer in flight, so the socket closing after this is normal
287+
// teardown, not an abandoned upstream request.
288+
if (pendingUpstreamSocket === socket) {
289+
pendingUpstreamSocket = null;
290+
}
221291
}
222292
}
223293
});
224294

225-
socket.on("close", () => {
226-
sockets.delete(socket);
227-
clearSocketOwnership(socket);
228-
});
295+
socket.on("close", () => onSocketGone(socket));
229296

230-
socket.on("error", () => {
231-
sockets.delete(socket);
232-
clearSocketOwnership(socket);
233-
});
297+
socket.on("error", () => onSocketGone(socket));
234298
});
235299

236300
process.on("SIGTERM", async () => {
@@ -244,6 +308,7 @@ async function main() {
244308
});
245309

246310
server.listen(listenTarget.path);
311+
armIdle();
247312
}
248313

249314
main().catch((error) => {

‎plugins/codex/scripts/lib/codex.mjs‎

Lines changed: 109 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -550,54 +550,49 @@ function applyTurnNotification(state, message) {
550550
}
551551
}
552552

553-
const DEFAULT_TURN_IDLE_TIMEOUT_MS = 240_000;
554-
555-
function resolveTurnIdleTimeoutMs() {
556-
const raw = process.env.CODEX_TURN_IDLE_TIMEOUT_MS;
557-
if (raw === undefined || raw === "") {
558-
return DEFAULT_TURN_IDLE_TIMEOUT_MS;
553+
// Best-effort interrupt of a turn being abandoned by a watchdog, bounded so the
554+
// teardown cannot itself hang. On the broker transport this is forwarded to the
555+
// upstream app-server, which stops the turn so it cannot overlap a retry.
556+
async function interruptTurnBestEffort(client, threadId, turnId) {
557+
if (!threadId || !turnId) {
558+
return;
559559
}
560-
const parsed = Number(raw);
561-
if (!Number.isFinite(parsed) || parsed < 0) {
562-
return DEFAULT_TURN_IDLE_TIMEOUT_MS;
560+
try {
561+
await Promise.race([
562+
client.request("turn/interrupt", { threadId, turnId }),
563+
new Promise((_resolve, reject) => {
564+
const timer = setTimeout(() => reject(new Error("turn/interrupt timed out")), 5_000);
565+
timer.unref?.();
566+
})
567+
]);
568+
} catch {
569+
// Best-effort: the turn is being abandoned regardless of the interrupt outcome.
563570
}
564-
return parsed;
565571
}
566572

567573
async function captureTurn(client, threadId, startRequest, options = {}) {
568574
const state = createTurnCaptureState(threadId, options);
569575
const previousHandler = client.notificationHandler;
570576

571-
const idleTimeoutMs = resolveTurnIdleTimeoutMs();
572-
let idleTimer = null;
573-
const clearIdle = () => {
574-
if (idleTimer) {
575-
clearTimeout(idleTimer);
576-
idleTimer = null;
577-
}
578-
};
579-
const armIdle = () => {
580-
if (idleTimeoutMs <= 0 || state.completed) {
581-
return;
582-
}
583-
clearIdle();
584-
idleTimer = setTimeout(() => {
585-
idleTimer = null;
586-
if (state.completed) {
587-
return;
588-
}
589-
state.completed = true;
590-
clearCompletionTimer(state);
591-
const tid = state.turnId ?? "(pending)";
592-
state.rejectCompletion(
593-
new Error(`codex turn idle timeout after ${idleTimeoutMs}ms (thread=${state.threadId}, turn=${tid})`)
594-
);
595-
}, idleTimeoutMs);
596-
idleTimer.unref?.();
597-
};
577+
// The completion promise resolves only on a terminal `turn/completed`
578+
// notification (or the inferred-completion heuristic) and is otherwise
579+
// uncoupled from time and from the app-server process. A wedged turn (e.g. a
580+
// hung MCP server that never emits another notification) or an app-server that
581+
// dies mid-turn would leave `await state.completion` hanging forever, which
582+
// surfaces as a "stuck" Codex run. Bound the wait three ways so it always
583+
// fails toward "didn't finish" instead of hanging:
584+
// - STALL_MS: no inbound traffic at all for this long => treat as stalled.
585+
// - CEILING_MS: absolute backstop on a single turn.
586+
// - exit: app-server process death rejects immediately.
587+
// Both timeouts are env-overridable for genuinely long turns.
588+
const CEILING_MS = Number(process.env.CODEX_COMPANION_TURN_TIMEOUT_MS) || 1_800_000;
589+
const STALL_MS = Number(process.env.CODEX_COMPANION_TURN_STALL_MS) || 600_000;
590+
let lastActivity = Date.now();
591+
let ceilingTimer = null;
592+
let stallTimer = null;
598593

599594
client.setNotificationHandler((message) => {
600-
armIdle();
595+
lastActivity = Date.now();
601596
if (!state.turnId) {
602597
state.bufferedNotifications.push(message);
603598
return;
@@ -618,11 +613,55 @@ async function captureTurn(client, threadId, startRequest, options = {}) {
618613
applyTurnNotification(state, message);
619614
});
620615

621-
armIdle();
616+
// `turnAbandoned` marks a failure where the turn may still be running upstream
617+
// (a watchdog fired, or the app-server vanished) — as opposed to a turn that
618+
// errored and is therefore already over. The catch below uses it to decide
619+
// whether the upstream work needs to be actively stopped.
620+
const ceiling = new Promise((_resolve, reject) => {
621+
ceilingTimer = setTimeout(() => {
622+
reject(
623+
Object.assign(
624+
new Error(
625+
`Codex turn exceeded the ${Math.round(CEILING_MS / 1000)}s ceiling without completing (raise CODEX_COMPANION_TURN_TIMEOUT_MS for long turns). Treating as stalled.`
626+
),
627+
{ turnAbandoned: true }
628+
)
629+
);
630+
}, CEILING_MS);
631+
ceilingTimer.unref?.();
632+
});
633+
634+
const stall = new Promise((_resolve, reject) => {
635+
stallTimer = setInterval(() => {
636+
if (Date.now() - lastActivity >= STALL_MS) {
637+
reject(
638+
Object.assign(
639+
new Error(
640+
`Codex turn produced no activity for ${Math.round(STALL_MS / 1000)}s (raise CODEX_COMPANION_TURN_STALL_MS for long silent turns) — likely a hung MCP server or wedged turn. Treating as stalled.`
641+
),
642+
{ turnAbandoned: true }
643+
)
644+
);
645+
}
646+
}, Math.min(STALL_MS, 30_000));
647+
stallTimer.unref?.();
648+
});
622649

623-
try {
650+
const exit = client.exitPromise.then(() => {
651+
if (state.completed) {
652+
return state;
653+
}
654+
throw Object.assign(
655+
new Error(client.exitError?.message ?? "codex app-server exited before the turn completed."),
656+
{ turnAbandoned: true }
657+
);
658+
});
659+
660+
// Wrap the start RPC and response processing so the guards also cover a start
661+
// request that hangs or rejects before the turn is ever established, and so
662+
// every guard is observed by Promise.race from the outset.
663+
const work = (async () => {
624664
const response = await startRequest();
625-
armIdle();
626665
options.onResponse?.(response, state);
627666
state.turnId = response.turn?.id ?? null;
628667
if (state.turnId) {
@@ -644,9 +683,37 @@ async function captureTurn(client, threadId, startRequest, options = {}) {
644683
}
645684

646685
return await state.completion;
686+
})();
687+
688+
try {
689+
// `work` wins under normal operation; the guards only fire on a wedge, an
690+
// over-long turn, or app-server death — including a start RPC that hangs or
691+
// rejects before the turn is established. Promise.race attaches a handler to
692+
// every racer up front, so a guard that rejects (even before the start RPC
693+
// returns, or after the race has settled) is always handled and never
694+
// surfaces as an unhandledRejection.
695+
return await Promise.race([work, ceiling, stall, exit]);
696+
} catch (error) {
697+
// Stopping the wait is not enough: on a shared broker, abandoning the turn
698+
// here leaves it running upstream, where it can keep mutating the workspace
699+
// and collide with the next task. (On the direct transport the app-server is
700+
// killed by close(), so only the broker path needs this.) When the turn id is
701+
// known, interrupt it so the broker stays alive and reusable. When the start
702+
// RPC hung before a turn id existed there is nothing to interrupt — the broker
703+
// detects the abandoned in-flight request when this client disconnects and
704+
// tears itself down, which covers owned and env-shared brokers alike.
705+
if (error?.turnAbandoned && !state.completed && client.transport === "broker" && state.turnId) {
706+
await interruptTurnBestEffort(client, state.threadId, state.turnId);
707+
}
708+
throw error;
647709
} finally {
648-
clearIdle();
649710
clearCompletionTimer(state);
711+
if (ceilingTimer) {
712+
clearTimeout(ceilingTimer);
713+
}
714+
if (stallTimer) {
715+
clearInterval(stallTimer);
716+
}
650717
client.setNotificationHandler(previousHandler ?? null);
651718
}
652719
}

‎plugins/codex/skills/codex-cli-runtime/SKILL.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ Safety rules:
4040
- Preserve the user's task text as-is apart from stripping routing flags.
4141
- Do not inspect the repository, read files, grep, monitor progress, poll status, fetch results, cancel jobs, summarize output, or do any follow-up work of your own.
4242
- Return the stdout of the `task` command exactly as-is.
43-
- If the Bash call fails or Codex cannot be invoked, return nothing.
43+
- If the Bash call fails or Codex cannot be invoked, return its error output (stderr and any stdout) verbatim so the failure is visible — never swallow it into an empty response.

‎tests/commands.test.mjs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ test("rescue command absorbs continue semantics", () => {
136136
assert.match(agent, /If the user asks for `spark`, map that to `--model gpt-5\.3-codex-spark`/i);
137137
assert.match(agent, /If the user asks for a concrete model name such as `gpt-5\.4-mini`, pass it through with `--model`/i);
138138
assert.match(agent, /Return the stdout of the `codex-companion` command exactly as-is/i);
139-
assert.match(agent, /If the Bash call fails or Codex cannot be invoked, return nothing/i);
139+
assert.match(agent, /If the Bash call fails or Codex cannot be invoked, return its error output/i);
140140
assert.match(agent, /gpt-5-4-prompting/);
141141
assert.match(agent, /only to tighten the user's request into a better Codex prompt/i);
142142
assert.match(agent, /Do not use that skill to inspect the repository, reason through the problem yourself, draft a solution, or do any independent work/i);
@@ -151,7 +151,7 @@ test("rescue command absorbs continue semantics", () => {
151151
assert.match(runtimeSkill, /Strip it before calling `task`/i);
152152
assert.match(runtimeSkill, /`--effort`: accepted values are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`/i);
153153
assert.match(runtimeSkill, /Do not inspect the repository, read files, grep, monitor progress, poll status, fetch results, cancel jobs, summarize output, or do any follow-up work of your own/i);
154-
assert.match(runtimeSkill, /If the Bash call fails or Codex cannot be invoked, return nothing/i);
154+
assert.match(runtimeSkill, /If the Bash call fails or Codex cannot be invoked, return its error output/i);
155155
assert.match(readme, /`codex:codex-rescue` subagent/i);
156156
assert.match(readme, /if you do not pass `--model` or `--effort`, Codex chooses its own defaults/i);
157157
assert.match(readme, /--model gpt-5\.4-mini --effort medium/i);

0 commit comments

Comments
 (0)