Skip to content

Commit e545512

Browse files
authored
fix(server): harden drain-and-restart spawn failure path (#580 follow-up) (#594)
* fix(server): exit cleanly when drain-and-restart spawn fails Follow-up to #580: wait for detached start spawn success, and on sync throw or pre-start error exit(1) without markRecycling so we never leave a drained process latched or exit(0) with no replacement. * fix(server): sanitize restart spawn errors and restore fences Address Codex review on #594: log only errno codes (no pathful messages), and clear inherited OCX_SERVICE on spawn failure so ensure/tray daemons still run syncCleanup restore.
1 parent 903b62c commit e545512

2 files changed

Lines changed: 157 additions & 10 deletions

File tree

‎src/server/management/system-restart.ts‎

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@
1111
* Task Scheduler ERRORLEVEL loop) bring the proxy back.
1212
* - Otherwise: detached `ocx start --port <live>` (bypasses ensure's
1313
* codexAutoStart gate), mark recycle so exit cleanup keeps injection, exit(0).
14+
* - If detached spawn fails (sync throw or pre-start `error`): exit(1) without
15+
* markRecycling — after drain the listen socket is already closed, so a latch
16+
* reset cannot recover serving. Clear inherited `OCX_SERVICE` so exit cleanup
17+
* can restore Codex/Grok fences (ensure/tray daemons set the marker without a
18+
* real supervisor). Log only a stable errno code — never the raw message
19+
* (paths in ENOENT often include the OS username).
1420
*/
1521
import { spawn } from "node:child_process";
1622
import {
@@ -31,7 +37,8 @@ export interface SystemRestartIo {
3137
drainAndShutdown?: typeof drainAndShutdown;
3238
isServiceInstalled?: () => boolean;
3339
isSupervisedServiceChild?: () => boolean;
34-
spawnStart?: (port?: number) => void;
40+
/** Must resolve only after the replacement process has actually started. */
41+
spawnStart?: (port?: number) => void | Promise<void>;
3542
markRecycling?: () => void;
3643
exitProcess?: (code: number) => void;
3744
schedule?: (fn: () => void | Promise<void>, ms: number) => void;
@@ -63,18 +70,49 @@ function isSupervisedServiceChild(): boolean {
6370
return process.env.OCX_SERVICE === "1" && isServiceInstalled();
6471
}
6572

66-
function spawnDetachedStart(port?: number): void {
73+
/** Stable, path-free spawn failure label for logs (never interpolate err.message). */
74+
function spawnFailureCode(err: unknown): string {
75+
if (err && typeof err === "object" && "code" in err) {
76+
const code = (err as NodeJS.ErrnoException).code;
77+
if (typeof code === "string" && code.length > 0 && code.length <= 64) return code;
78+
}
79+
return "spawn_failed";
80+
}
81+
82+
function spawnDetachedStart(port?: number): Promise<void> {
6783
const args = [process.argv[1], "start"];
6884
if (typeof port === "number" && Number.isFinite(port) && port > 0 && port <= 65535) {
6985
args.push("--port", String(Math.trunc(port)));
7086
}
71-
const child = spawn(process.execPath, args, {
72-
detached: true,
73-
stdio: "ignore",
74-
windowsHide: true,
75-
env: { ...process.env, OCX_SERVICE: "1" },
87+
return new Promise<void>((resolve, reject) => {
88+
let child: ReturnType<typeof spawn>;
89+
try {
90+
child = spawn(process.execPath, args, {
91+
detached: true,
92+
stdio: "ignore",
93+
windowsHide: true,
94+
env: { ...process.env, OCX_SERVICE: "1" },
95+
});
96+
} catch (err) {
97+
reject(err);
98+
return;
99+
}
100+
let settled = false;
101+
const finish = (fn: () => void) => {
102+
if (settled) return;
103+
settled = true;
104+
fn();
105+
};
106+
child.once("error", (err) => {
107+
finish(() => reject(err));
108+
});
109+
child.once("spawn", () => {
110+
finish(() => {
111+
child.unref();
112+
resolve();
113+
});
114+
});
76115
});
77-
child.unref();
78116
}
79117

80118
/**
@@ -106,9 +144,22 @@ export function acceptSystemRestart(io: SystemRestartIo = restartIo): {
106144
return;
107145
}
108146
const port = (io.listenPort ?? resolveListenPort)();
109-
(io.spawnStart ?? spawnDetachedStart)(port);
147+
const exitProcess = io.exitProcess ?? ((code: number) => { process.exit(code); });
148+
try {
149+
await (io.spawnStart ?? spawnDetachedStart)(port);
150+
} catch (err) {
151+
console.warn(
152+
`⚠️ Drain-and-restart spawn failed (${spawnFailureCode(err)}); exiting without replacement`,
153+
);
154+
// Listen socket is already stopped; do not markRecycling — no child to inherit fences.
155+
// ensure/tray children inherit OCX_SERVICE=1 without an installed service; clear it so
156+
// syncCleanup can restore Codex/Grok fences instead of leaving clients pointed at a dead port.
157+
delete process.env.OCX_SERVICE;
158+
exitProcess(1);
159+
return;
160+
}
110161
(io.markRecycling ?? markRecyclingForExit)();
111-
(io.exitProcess ?? ((code: number) => { process.exit(code); }))(0);
162+
exitProcess(0);
112163
}, 200);
113164
}
114165

‎tests/system-restart.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,102 @@ describe("acceptSystemRestart", () => {
8484
expect(calls).toEqual(["drain", "exit:1"]);
8585
});
8686

87+
test("spawn sync throw exits 1 without marking recycle", async () => {
88+
const calls: string[] = [];
89+
let scheduled: (() => void | Promise<void>) | null = null;
90+
const warnings: string[] = [];
91+
const originalWarn = console.warn;
92+
console.warn = (...args: unknown[]) => {
93+
warnings.push(args.map(String).join(" "));
94+
};
95+
96+
try {
97+
acceptSystemRestart({
98+
isDraining: () => false,
99+
getActiveTurnCount: () => 0,
100+
isSupervisedServiceChild: () => false,
101+
listenPort: () => 10123,
102+
schedule: (fn) => { scheduled = fn; },
103+
setDraining: () => {},
104+
drainAndShutdown: async () => { calls.push("drain"); },
105+
spawnStart: () => {
106+
calls.push("start");
107+
const err = new Error(
108+
"ENOENT: no such file or directory, uv_spawn 'C:\\Users\\Alice\\AppData\\Local\\bun\\bun.exe'",
109+
) as NodeJS.ErrnoException;
110+
err.code = "ENOENT";
111+
throw err;
112+
},
113+
markRecycling: () => { calls.push("recycle"); },
114+
exitProcess: (code) => { calls.push(`exit:${code}`); },
115+
});
116+
117+
await scheduled!();
118+
expect(calls).toEqual(["drain", "start", "exit:1"]);
119+
expect(warnings.some((w) => w.includes("ENOENT"))).toBe(true);
120+
expect(warnings.some((w) => /Alice|AppData|bun\.exe/i.test(w))).toBe(false);
121+
} finally {
122+
console.warn = originalWarn;
123+
}
124+
});
125+
126+
test("spawn async rejection exits 1 without marking recycle", async () => {
127+
const calls: string[] = [];
128+
let scheduled: (() => void | Promise<void>) | null = null;
129+
130+
acceptSystemRestart({
131+
isDraining: () => false,
132+
getActiveTurnCount: () => 0,
133+
isSupervisedServiceChild: () => false,
134+
listenPort: () => 10123,
135+
schedule: (fn) => { scheduled = fn; },
136+
setDraining: () => {},
137+
drainAndShutdown: async () => { calls.push("drain"); },
138+
spawnStart: async () => {
139+
calls.push("start");
140+
throw new Error("spawn error before start");
141+
},
142+
markRecycling: () => { calls.push("recycle"); },
143+
exitProcess: (code) => { calls.push(`exit:${code}`); },
144+
});
145+
146+
await scheduled!();
147+
expect(calls).toEqual(["drain", "start", "exit:1"]);
148+
});
149+
150+
test("spawn failure clears OCX_SERVICE so exit cleanup can restore fences", async () => {
151+
const calls: string[] = [];
152+
let scheduled: (() => void | Promise<void>) | null = null;
153+
const prev = process.env.OCX_SERVICE;
154+
process.env.OCX_SERVICE = "1";
155+
156+
try {
157+
acceptSystemRestart({
158+
isDraining: () => false,
159+
getActiveTurnCount: () => 0,
160+
// ensure/tray: marker set, but no installed service → unsupervised spawn path
161+
isSupervisedServiceChild: () => false,
162+
listenPort: () => 10123,
163+
schedule: (fn) => { scheduled = fn; },
164+
setDraining: () => {},
165+
drainAndShutdown: async () => { calls.push("drain"); },
166+
spawnStart: () => {
167+
calls.push("start");
168+
throw Object.assign(new Error("ENOENT"), { code: "ENOENT" });
169+
},
170+
markRecycling: () => { calls.push("recycle"); },
171+
exitProcess: (code) => { calls.push(`exit:${code}`); },
172+
});
173+
174+
await scheduled!();
175+
expect(calls).toEqual(["drain", "start", "exit:1"]);
176+
expect(process.env.OCX_SERVICE).toBeUndefined();
177+
} finally {
178+
if (prev === undefined) delete process.env.OCX_SERVICE;
179+
else process.env.OCX_SERVICE = prev;
180+
}
181+
});
182+
87183
test("does not schedule a second drain while already draining", async () => {
88184
let scheduled = 0;
89185
const result = acceptSystemRestart({

0 commit comments

Comments
 (0)