Skip to content

Commit acb0acd

Browse files
committed
fix: add gateway supervisor restart handoff
1 parent f9da484 commit acb0acd

5 files changed

Lines changed: 627 additions & 5 deletions

File tree

src/cli/gateway-cli/lifecycle.runtime.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ export {
1919
resetGatewayRestartStateForInProcessRestart,
2020
scheduleGatewaySigusr1Restart,
2121
} from "../../infra/restart.js";
22+
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
2223
export { markUpdateRestartSentinelFailure } from "../../infra/restart-sentinel.js";
2324
export { detectRespawnSupervisor } from "../../infra/supervisor-markers.js";
2425
export { writeDiagnosticStabilityBundleForFailureSync } from "../../logging/diagnostic-stability-bundle.js";

src/cli/gateway-cli/run-loop.test.ts

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,17 @@ const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false);
1414
const markGatewaySigusr1RestartHandled = vi.fn();
1515
const peekGatewaySigusr1RestartReason = vi.fn<() => string | undefined>(() => undefined);
1616
const resetGatewayRestartStateForInProcessRestart = vi.fn();
17+
const writeGatewayRestartHandoffSync = vi.fn((_opts: unknown) => ({
18+
kind: "gateway-supervisor-restart-handoff" as const,
19+
version: 1 as const,
20+
intentId: "test-intent",
21+
pid: process.pid,
22+
createdAt: Date.now(),
23+
expiresAt: Date.now() + 60_000,
24+
source: "unknown" as const,
25+
restartKind: "full-process" as const,
26+
supervisorMode: "external" as const,
27+
}));
1728
const scheduleGatewaySigusr1Restart = vi.fn((_opts?: { delayMs?: number; reason?: string }) => ({
1829
ok: true,
1930
pid: process.pid,
@@ -107,6 +118,10 @@ vi.mock("../../infra/restart-sentinel.js", () => ({
107118
markUpdateRestartSentinelFailure: (reason: string) => markUpdateRestartSentinelFailure(reason),
108119
}));
109120

121+
vi.mock("../../infra/restart-handoff.js", () => ({
122+
writeGatewayRestartHandoffSync: (opts: unknown) => writeGatewayRestartHandoffSync(opts),
123+
}));
124+
110125
vi.mock("../../process/command-queue.js", () => ({
111126
getActiveTaskCount: () => getActiveTaskCount(),
112127
markGatewayDraining: () => markGatewayDraining(),
@@ -595,6 +610,7 @@ describe("runGatewayLoop", () => {
595610
expect(lockRelease).toHaveBeenCalled();
596611
expect(runtime.exit).toHaveBeenCalledWith(0);
597612
expect(exitCallOrder).toEqual(["lockRelease", "exit"]);
613+
expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled();
598614
});
599615
});
600616

@@ -616,6 +632,12 @@ describe("runGatewayLoop", () => {
616632
sigusr1();
617633
await expect(exited).resolves.toBe(0);
618634
expect(runtime.exit).toHaveBeenCalledWith(0);
635+
expect(writeGatewayRestartHandoffSync).toHaveBeenCalledWith({
636+
restartKind: "full-process",
637+
reason: undefined,
638+
processInstanceId: expect.any(String),
639+
supervisorMode: "launchd",
640+
});
619641
expect(Date.now() - startedAt).toBeGreaterThanOrEqual(1400);
620642
});
621643
} finally {
@@ -719,7 +741,38 @@ describe("runGatewayLoop", () => {
719741
expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1);
720742
expect(start).toHaveBeenCalledTimes(1);
721743
expect(markUpdateRestartSentinelFailure).not.toHaveBeenCalled();
744+
expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled();
745+
});
746+
});
747+
748+
it("writes a handoff before exiting for supervised update restarts", async () => {
749+
vi.clearAllMocks();
750+
peekGatewaySigusr1RestartReason.mockReturnValue("update.run");
751+
respawnGatewayProcessForUpdate.mockReturnValueOnce({
752+
mode: "supervised",
722753
});
754+
try {
755+
setPlatform("freebsd");
756+
await withIsolatedSignals(async ({ captureSignal }) => {
757+
const { runtime, exited } = await createSignaledLoopHarness();
758+
const sigusr1 = captureSignal("SIGUSR1");
759+
760+
sigusr1();
761+
762+
await expect(exited).resolves.toBe(0);
763+
expect(runtime.exit).toHaveBeenCalledWith(0);
764+
expect(writeGatewayRestartHandoffSync).toHaveBeenCalledWith({
765+
restartKind: "update-process",
766+
reason: "update.run",
767+
processInstanceId: expect.any(String),
768+
supervisorMode: "external",
769+
});
770+
});
771+
} finally {
772+
if (originalPlatformDescriptor) {
773+
Object.defineProperty(process, "platform", originalPlatformDescriptor);
774+
}
775+
}
723776
});
724777

725778
it("probes the configured gateway host for update respawn health", async () => {

src/cli/gateway-cli/run-loop.ts

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { randomUUID } from "node:crypto";
12
import net from "node:net";
23
import type { startGatewayServer } from "../../gateway/server.js";
34
import { formatErrorMessage } from "../../infra/errors.js";
@@ -94,6 +95,7 @@ export async function runGatewayLoop(params: {
9495
let server: Awaited<ReturnType<typeof startGatewayServer>> | null = null;
9596
let shuttingDown = false;
9697
let restartResolver: (() => void) | null = null;
98+
const processInstanceId = randomUUID();
9799
const waitForHealthyChild = params.waitForHealthyChild ?? waitForHealthyGatewayChild;
98100

99101
const cleanupSignals = () => {
@@ -140,6 +142,7 @@ export async function runGatewayLoop(params: {
140142
markUpdateRestartSentinelFailure,
141143
respawnGatewayProcessForUpdate,
142144
restartGatewayProcessWithFreshPid,
145+
writeGatewayRestartHandoffSync,
143146
} = await loadGatewayLifecycleRuntimeModule();
144147

145148
if (isUpdateRestart) {
@@ -176,8 +179,15 @@ export async function runGatewayLoop(params: {
176179
return;
177180
}
178181
if (respawn.mode === "supervised") {
182+
const supervisorMode = detectRespawnSupervisor(process.env, process.platform);
183+
writeGatewayRestartHandoffSync({
184+
restartKind: "update-process",
185+
reason: restartReason,
186+
processInstanceId,
187+
supervisorMode: supervisorMode ?? "external",
188+
});
179189
gatewayLog.info("restart mode: update process respawn (supervisor restart)");
180-
if (detectRespawnSupervisor(process.env, process.platform) === "launchd") {
190+
if (supervisorMode === "launchd") {
181191
await new Promise((resolve) => {
182192
setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS);
183193
});
@@ -208,15 +218,24 @@ export async function runGatewayLoop(params: {
208218
// Release the lock BEFORE spawning so the child can acquire it immediately.
209219
const respawn = restartGatewayProcessWithFreshPid();
210220
if (respawn.mode === "spawned" || respawn.mode === "supervised") {
221+
const supervisorMode =
222+
respawn.mode === "supervised"
223+
? detectRespawnSupervisor(process.env, process.platform)
224+
: null;
211225
const modeLabel =
212226
respawn.mode === "spawned"
213227
? `spawned pid ${respawn.pid ?? "unknown"}`
214228
: "supervisor restart";
229+
if (respawn.mode === "supervised") {
230+
writeGatewayRestartHandoffSync({
231+
restartKind: "full-process",
232+
reason: restartReason,
233+
processInstanceId,
234+
supervisorMode: supervisorMode ?? "external",
235+
});
236+
}
215237
gatewayLog.info(`restart mode: full process restart (${modeLabel})`);
216-
if (
217-
respawn.mode === "supervised" &&
218-
detectRespawnSupervisor(process.env, process.platform) === "launchd"
219-
) {
238+
if (supervisorMode === "launchd") {
220239
// A short clean-exit pause keeps rapid SIGUSR1/config restarts from
221240
// tripping launchd crash-loop throttling before KeepAlive relaunches.
222241
await new Promise((resolve) => {

src/infra/restart-handoff.test.ts

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
import fs from "node:fs";
2+
import os from "node:os";
3+
import path from "node:path";
4+
import { afterEach, describe, expect, it } from "vitest";
5+
import {
6+
consumeGatewayRestartHandoffForExitedProcessSync,
7+
GATEWAY_SUPERVISOR_RESTART_HANDOFF_FILENAME,
8+
GATEWAY_SUPERVISOR_RESTART_HANDOFF_KIND,
9+
readGatewayRestartHandoffSync,
10+
writeGatewayRestartHandoffSync,
11+
} from "./restart-handoff.js";
12+
13+
const tempDirs: string[] = [];
14+
15+
function createHandoffEnv(): NodeJS.ProcessEnv {
16+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-restart-handoff-"));
17+
tempDirs.push(dir);
18+
return {
19+
...process.env,
20+
OPENCLAW_STATE_DIR: dir,
21+
};
22+
}
23+
24+
function handoffPath(env: NodeJS.ProcessEnv): string {
25+
return path.join(env.OPENCLAW_STATE_DIR ?? "", GATEWAY_SUPERVISOR_RESTART_HANDOFF_FILENAME);
26+
}
27+
28+
describe("gateway restart handoff", () => {
29+
afterEach(() => {
30+
for (const dir of tempDirs.splice(0)) {
31+
fs.rmSync(dir, { force: true, recursive: true });
32+
}
33+
});
34+
35+
it("writes a supervisor handoff for an exited gateway process", () => {
36+
const env = createHandoffEnv();
37+
38+
const handoff = writeGatewayRestartHandoffSync({
39+
env,
40+
pid: 12_345,
41+
processInstanceId: "gateway-instance-1",
42+
reason: "plugin source changed",
43+
restartKind: "full-process",
44+
supervisorMode: "launchd",
45+
createdAt: 1_000,
46+
});
47+
48+
expect(handoff).toMatchObject({
49+
kind: GATEWAY_SUPERVISOR_RESTART_HANDOFF_KIND,
50+
version: 1,
51+
pid: 12_345,
52+
processInstanceId: "gateway-instance-1",
53+
reason: "plugin source changed",
54+
source: "plugin-change",
55+
restartKind: "full-process",
56+
supervisorMode: "launchd",
57+
createdAt: 1_000,
58+
expiresAt: 61_000,
59+
});
60+
expect(fs.statSync(handoffPath(env)).mode & 0o777).toBe(0o600);
61+
expect(readGatewayRestartHandoffSync(env, 1_500)).toMatchObject({
62+
pid: 12_345,
63+
reason: "plugin source changed",
64+
});
65+
});
66+
67+
it("consumes a fresh handoff by exited pid instead of current process pid", () => {
68+
const env = createHandoffEnv();
69+
70+
expect(
71+
writeGatewayRestartHandoffSync({
72+
env,
73+
pid: process.pid + 1,
74+
reason: "update.run",
75+
restartKind: "update-process",
76+
supervisorMode: "systemd",
77+
createdAt: 2_000,
78+
}),
79+
).not.toBeNull();
80+
81+
expect(
82+
consumeGatewayRestartHandoffForExitedProcessSync({
83+
env,
84+
exitedPid: process.pid + 1,
85+
now: 2_001,
86+
}),
87+
).toMatchObject({
88+
pid: process.pid + 1,
89+
source: "gateway-update",
90+
restartKind: "update-process",
91+
supervisorMode: "systemd",
92+
});
93+
expect(fs.existsSync(handoffPath(env))).toBe(false);
94+
});
95+
96+
it("rejects handoffs for a different exited pid and clears them", () => {
97+
const env = createHandoffEnv();
98+
99+
expect(
100+
writeGatewayRestartHandoffSync({
101+
env,
102+
pid: 111,
103+
restartKind: "full-process",
104+
supervisorMode: "external",
105+
createdAt: 1_000,
106+
}),
107+
).not.toBeNull();
108+
109+
expect(
110+
consumeGatewayRestartHandoffForExitedProcessSync({
111+
env,
112+
exitedPid: 222,
113+
now: 1_001,
114+
}),
115+
).toBeNull();
116+
expect(fs.existsSync(handoffPath(env))).toBe(false);
117+
});
118+
119+
it("rejects a handoff when the supplied process instance does not match", () => {
120+
const env = createHandoffEnv();
121+
122+
expect(
123+
writeGatewayRestartHandoffSync({
124+
env,
125+
pid: 111,
126+
processInstanceId: "gateway-instance-1",
127+
restartKind: "full-process",
128+
supervisorMode: "external",
129+
createdAt: 1_000,
130+
}),
131+
).not.toBeNull();
132+
133+
expect(
134+
consumeGatewayRestartHandoffForExitedProcessSync({
135+
env,
136+
exitedPid: 111,
137+
processInstanceId: "gateway-instance-2",
138+
now: 1_001,
139+
}),
140+
).toBeNull();
141+
expect(fs.existsSync(handoffPath(env))).toBe(false);
142+
});
143+
144+
it("rejects malformed handoff payloads", () => {
145+
const env = createHandoffEnv();
146+
147+
fs.writeFileSync(
148+
handoffPath(env),
149+
`${JSON.stringify({
150+
kind: GATEWAY_SUPERVISOR_RESTART_HANDOFF_KIND,
151+
version: 1,
152+
intentId: "bad",
153+
pid: 111,
154+
createdAt: 1_000,
155+
expiresAt: 61_000,
156+
reason: 123,
157+
source: "bad-source",
158+
restartKind: "full-process",
159+
supervisorMode: "external",
160+
})}\n`,
161+
{ encoding: "utf8", mode: 0o600 },
162+
);
163+
164+
expect(readGatewayRestartHandoffSync(env, 1_001)).toBeNull();
165+
});
166+
167+
it("rejects expired and oversized handoff files", () => {
168+
const env = createHandoffEnv();
169+
170+
expect(
171+
writeGatewayRestartHandoffSync({
172+
env,
173+
pid: 111,
174+
restartKind: "full-process",
175+
supervisorMode: "external",
176+
createdAt: 1_000,
177+
ttlMs: 1_000,
178+
}),
179+
).not.toBeNull();
180+
expect(readGatewayRestartHandoffSync(env, 2_001)).toBeNull();
181+
182+
fs.writeFileSync(handoffPath(env), "x".repeat(8192), { encoding: "utf8", mode: 0o600 });
183+
expect(
184+
consumeGatewayRestartHandoffForExitedProcessSync({
185+
env,
186+
exitedPid: 111,
187+
now: 2_001,
188+
}),
189+
).toBeNull();
190+
expect(fs.existsSync(handoffPath(env))).toBe(false);
191+
});
192+
193+
it("does not follow an existing handoff-path symlink when writing", () => {
194+
const env = createHandoffEnv();
195+
const targetPath = path.join(env.OPENCLAW_STATE_DIR ?? "", "attacker-target.txt");
196+
fs.writeFileSync(targetPath, "keep", "utf8");
197+
try {
198+
fs.symlinkSync(targetPath, handoffPath(env));
199+
} catch {
200+
return;
201+
}
202+
203+
expect(
204+
writeGatewayRestartHandoffSync({
205+
env,
206+
pid: 12_345,
207+
restartKind: "full-process",
208+
supervisorMode: "external",
209+
}),
210+
).not.toBeNull();
211+
212+
expect(fs.readFileSync(targetPath, "utf8")).toBe("keep");
213+
expect(fs.lstatSync(handoffPath(env)).isSymbolicLink()).toBe(false);
214+
expect(
215+
consumeGatewayRestartHandoffForExitedProcessSync({
216+
env,
217+
exitedPid: 12_345,
218+
}),
219+
).toMatchObject({ pid: 12_345 });
220+
});
221+
});

0 commit comments

Comments
 (0)