Skip to content

Commit e30ce44

Browse files
849261680849261680
committed
fix(gateway): skip ingress sweep for spawned restarts
1 parent 46485f8 commit e30ce44

6 files changed

Lines changed: 81 additions & 5 deletions

File tree

‎src/cli/gateway-cli/lifecycle.runtime.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ export {
2424
resetGatewayRestartStateForInProcessRestart,
2525
scheduleGatewaySigusr1Restart,
2626
} from "../../infra/restart.js";
27-
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
27+
export {
28+
withGatewayRestartSkipStartupIngressSweepEnv,
29+
writeGatewayRestartHandoffSync,
30+
} from "../../infra/restart-handoff.js";
2831
export { markUpdateRestartSentinelFailure } from "../../infra/restart-sentinel.js";
2932
export { detectRespawnSupervisor } from "../../infra/supervisor-markers.js";
3033
export { writeDiagnosticStabilityBundleForFailureSync } from "../../logging/diagnostic-stability-bundle.js";

‎src/cli/gateway-cli/run-loop.test.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@ const writeGatewayRestartHandoffSync = vi.fn((_opts: unknown) => ({
3030
restartKind: "full-process" as const,
3131
supervisorMode: "external" as const,
3232
}));
33+
const withGatewayRestartSkipStartupIngressSweepEnv = vi.fn(
34+
(env: NodeJS.ProcessEnv | undefined) => ({
35+
...env,
36+
OPENCLAW_GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP: "1",
37+
}),
38+
);
3339
const scheduleGatewaySigusr1Restart = vi.fn((_opts?: { delayMs?: number; reason?: string }) => ({
3440
ok: true,
3541
pid: process.pid,
@@ -140,6 +146,8 @@ vi.mock("../../infra/restart-sentinel.js", () => ({
140146
}));
141147

142148
vi.mock("../../infra/restart-handoff.js", () => ({
149+
withGatewayRestartSkipStartupIngressSweepEnv: (env: NodeJS.ProcessEnv | undefined) =>
150+
withGatewayRestartSkipStartupIngressSweepEnv(env),
143151
writeGatewayRestartHandoffSync: (opts: unknown) => writeGatewayRestartHandoffSync(opts),
144152
}));
145153

@@ -757,7 +765,10 @@ describe("runGatewayLoop", () => {
757765
waitForActiveTasks.mockResolvedValueOnce({ drained: false });
758766
waitForActiveEmbeddedRuns.mockResolvedValueOnce({ drained: true });
759767

760-
type StartServer = () => Promise<{
768+
type StartServer = (params?: {
769+
startupStartedAt?: number;
770+
inProcessRestart?: boolean;
771+
}) => Promise<{
761772
close: GatewayCloseFn;
762773
}>;
763774

@@ -1369,6 +1380,7 @@ describe("runGatewayLoop", () => {
13691380
const [respawnOpts] = restartGatewayProcessWithFreshPid.mock.calls[0] ?? [];
13701381
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_STARTED_AT_MS).toMatch(/^\d/u);
13711382
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_LAST_AT_MS).toMatch(/^\d/u);
1383+
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP).toBe("1");
13721384
expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled();
13731385
});
13741386
} finally {
@@ -1548,6 +1560,8 @@ describe("runGatewayLoop", () => {
15481560
await expect(exited).resolves.toBe(0);
15491561
expect(waitForHealthyChild).toHaveBeenCalledWith(18789, 7777, "127.0.0.1");
15501562
expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1);
1563+
const [respawnOpts] = respawnGatewayProcessForUpdate.mock.calls[0] ?? [];
1564+
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP).toBe("1");
15511565
expect(start).toHaveBeenCalledTimes(1);
15521566
expect(markUpdateRestartSentinelFailure).not.toHaveBeenCalled();
15531567
expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled();

‎src/cli/gateway-cli/run-loop.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,13 +180,16 @@ export async function runGatewayLoop(params: {
180180
markUpdateRestartSentinelFailure,
181181
respawnGatewayProcessForUpdate,
182182
restartGatewayProcessWithFreshPid,
183+
withGatewayRestartSkipStartupIngressSweepEnv,
183184
writeGatewayRestartHandoffSync,
184185
} = await loadGatewayLifecycleRuntimeModule();
185186

186187
if (isUpdateRestart) {
187188
const restartTraceHandoff = captureGatewayRestartTraceHandoff();
188189
const respawn = respawnGatewayProcessForUpdate({
189-
env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
190+
env: withGatewayRestartSkipStartupIngressSweepEnv(
191+
createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
192+
),
190193
});
191194
if (respawn.mode === "spawned") {
192195
const port = params.lockPort;
@@ -265,7 +268,9 @@ export async function runGatewayLoop(params: {
265268
// Release the lock BEFORE spawning so the child can acquire it immediately.
266269
const restartTraceHandoff = captureGatewayRestartTraceHandoff();
267270
const respawn = restartGatewayProcessWithFreshPid({
268-
env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
271+
env: withGatewayRestartSkipStartupIngressSweepEnv(
272+
createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
273+
),
269274
});
270275
if (respawn.mode === "spawned" || respawn.mode === "supervised") {
271276
const supervisorMode =

‎src/gateway/server-startup-ingress-sweep.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { describe, expect, it, vi } from "vitest";
2+
import { GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV } from "../infra/restart-handoff.js";
23
import { runStartupIngressClaimSweep } from "./server-startup-ingress-sweep.js";
34

45
describe("runStartupIngressClaimSweep", () => {
@@ -40,6 +41,28 @@ describe("runStartupIngressClaimSweep", () => {
4041
expect(warn).not.toHaveBeenCalled();
4142
});
4243

44+
it("skips and consumes the sweep marker during spawned restart handoff", async () => {
45+
const info = vi.fn();
46+
const warn = vi.fn();
47+
const mockSweep = vi.fn();
48+
const env = {
49+
[GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV]: "1",
50+
};
51+
52+
await runStartupIngressClaimSweep({
53+
env,
54+
log: { info, warn },
55+
deps: { recoverAllStaleChannelIngressClaims: mockSweep },
56+
});
57+
58+
expect(mockSweep).not.toHaveBeenCalled();
59+
expect(env[GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV]).toBeUndefined();
60+
expect(info).toHaveBeenCalledWith(
61+
"gateway: skipping stale ingress claim sweep during spawned restart handoff",
62+
);
63+
expect(warn).not.toHaveBeenCalled();
64+
});
65+
4366
it("skips the sweep during supervised restart handoff", async () => {
4467
const info = vi.fn();
4568
const warn = vi.fn();

‎src/gateway/server-startup-ingress-sweep.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { recoverAllStaleChannelIngressClaims } from "../channels/message/ingress-queue.js";
2-
import { readGatewayRestartHandoffSync } from "../infra/restart-handoff.js";
2+
import {
3+
consumeGatewayRestartSkipStartupIngressSweepEnv,
4+
readGatewayRestartHandoffSync,
5+
} from "../infra/restart-handoff.js";
36

47
type IngressSweepLogger = {
58
info: (message: string) => void;
@@ -25,6 +28,7 @@ export async function runStartupIngressClaimSweep(params: {
2528
inProcessRestart?: boolean;
2629
log: IngressSweepLogger;
2730
deps?: {
31+
consumeGatewayRestartSkipStartupIngressSweepEnv?: typeof consumeGatewayRestartSkipStartupIngressSweepEnv;
2832
recoverAllStaleChannelIngressClaims?: typeof recoverAllStaleChannelIngressClaims;
2933
readGatewayRestartHandoffSync?: typeof readGatewayRestartHandoffSync;
3034
};
@@ -38,6 +42,13 @@ export async function runStartupIngressClaimSweep(params: {
3842
const sweep =
3943
params.deps?.recoverAllStaleChannelIngressClaims ?? recoverAllStaleChannelIngressClaims;
4044
const env = params.env ?? process.env;
45+
const consumeSkipEnv =
46+
params.deps?.consumeGatewayRestartSkipStartupIngressSweepEnv ??
47+
consumeGatewayRestartSkipStartupIngressSweepEnv;
48+
if (consumeSkipEnv(env)) {
49+
params.log.info("gateway: skipping stale ingress claim sweep during spawned restart handoff");
50+
return;
51+
}
4152
const readHandoff = params.deps?.readGatewayRestartHandoffSync ?? readGatewayRestartHandoffSync;
4253
if (readHandoff(env)) {
4354
params.log.info(

‎src/infra/restart-handoff.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@ import path from "node:path";
55
import { isRecord } from "@openclaw/normalization-core/record-coerce";
66
import { resolveStateDir } from "../config/paths.js";
77
import { createSubsystemLogger } from "../logging/subsystem.js";
8+
import { isTruthyEnvValue } from "./env.js";
89

910
// Restart handoff files let a supervisor explain a recent gateway restart after
1011
// the old process exits. The file is short-lived, bounded, and regular-file only.
1112
export const GATEWAY_SUPERVISOR_RESTART_HANDOFF_FILENAME =
1213
"gateway-supervisor-restart-handoff.json";
1314
export const GATEWAY_SUPERVISOR_RESTART_HANDOFF_KIND = "gateway-supervisor-restart-handoff";
15+
export const GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV =
16+
"OPENCLAW_GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP";
1417
const GATEWAY_RESTART_HANDOFF_TTL_MS = 60_000;
1518
const GATEWAY_RESTART_TRACE_HANDOFF_MAX_DURATION_MS = 10 * 60_000;
1619
const GATEWAY_RESTART_HANDOFF_MAX_BYTES = 4096;
@@ -48,6 +51,23 @@ export type GatewayRestartHandoff = {
4851
};
4952
};
5053

54+
export function withGatewayRestartSkipStartupIngressSweepEnv(
55+
env: NodeJS.ProcessEnv | undefined,
56+
): NodeJS.ProcessEnv {
57+
return {
58+
...env,
59+
[GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV]: "1",
60+
};
61+
}
62+
63+
export function consumeGatewayRestartSkipStartupIngressSweepEnv(
64+
env: NodeJS.ProcessEnv = process.env,
65+
): boolean {
66+
const shouldSkip = isTruthyEnvValue(env[GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV]);
67+
delete env[GATEWAY_RESTART_SKIP_STARTUP_INGRESS_SWEEP_ENV];
68+
return shouldSkip;
69+
}
70+
5171
function formatShortDuration(ms: number): string {
5272
const clamped = Math.max(0, Math.floor(ms));
5373
if (clamped < 1000) {

0 commit comments

Comments
 (0)