fix(gateway): add safe restart coordinator (#76923)

NikolaFC · galiniliev · web-flow · commit 103cdd9d96f8 · 2026-05-04T10:58:36.000-07:00
Add a safe restart coordinator that preflights active Gateway work before restart.

- expose gateway.restart.preflight and gateway.restart.request RPC methods
- add explicit openclaw gateway restart --safe / openclaw daemon restart --safe path
- narrow restart blockers to running non-ended tasks so queued records no longer block indefinitely
- keep existing restart behavior unchanged; --force remains the immediate override

Co-authored-by: NikolaFC &lt;54186359+NikolaFC@users.noreply.github.com&gt;
Co-authored-by: galiniliev &lt;5711535+galiniliev@users.noreply.github.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -457,6 +457,7 @@ Docs: https://docs.openclaw.ai
 - Status/update: resolve beta update-channel checks from the installed version when config still says `stable`, and let `status --deep` reuse live gateway channel credential state instead of warning on command-path-only token misses.
 - Doctor/plugins: preserve unmanaged third-party plugin `node_modules` during `doctor --fix`, while still pruning OpenClaw-managed runtime dependency caches.
 - Gateway/restart: add `openclaw gateway restart --force` and `--wait <duration>`, log active task run IDs before restart deferral timers, and report timeout restarts as explicit forced restarts.
+- Gateway/restart: align `gateway.restart.safe` preflight with scheduled restart deferral by counting only active restart blockers (running non-ended tasks), so queued task records no longer keep "safe" restarts deferred indefinitely.
 - Discord: persist slash-command deploy hashes across process restarts so unchanged command sets skip redeploy and avoid restart-loop 429s.
 - Providers/LM Studio: normalize binary `off`/`on` reasoning metadata from Gemma 4 and other local models to LM Studio's accepted OpenAI-compatible `reasoning_effort` values.
 - Plugins/externalization: keep official external install docs, update examples, and live Codex npm checks on default npm tags instead of `@beta`. Thanks @vincentkoc.
diff --git a/docs/cli/daemon.md b/docs/cli/daemon.md
@@ -36,7 +36,7 @@ openclaw daemon uninstall
 
 - `status`: `--url`, `--token`, `--password`, `--timeout`, `--no-probe`, `--require-rpc`, `--deep`, `--json`
 - `install`: `--port`, `--runtime <node|bun>`, `--token`, `--force`, `--json`
-- `restart`: `--force`, `--wait <duration>`, `--json`
+- `restart`: `--safe`, `--force`, `--wait <duration>`, `--json`
 - lifecycle (`uninstall|start|stop`): `--json`
 
 Notes:
@@ -53,6 +53,7 @@ Notes:
 - If both `gateway.auth.token` and `gateway.auth.password` are configured and `gateway.auth.mode` is unset, install is blocked until mode is set explicitly.
 - On macOS, `install` keeps LaunchAgent plists owner-only and loads managed service environment values through an owner-only file and wrapper instead of serializing API keys or auth-profile env refs into `EnvironmentVariables`.
 - If you intentionally run multiple gateways on one host, isolate ports, config/state, and workspaces; see [/gateway#multiple-gateways-same-host](/gateway#multiple-gateways-same-host).
+- `restart --safe` asks the running Gateway to preflight active work and schedule one coalesced restart after active work drains. Plain `restart` keeps the existing service-manager behavior; `--force` remains the immediate override path.
 
 ## Prefer
 
diff --git a/docs/cli/gateway.md b/docs/cli/gateway.md
@@ -105,6 +105,16 @@ openclaw gateway run
   Raw stream jsonl path.
 </ParamField>
 
+## Restart the Gateway
+
+```bash
+openclaw gateway restart
+openclaw gateway restart --safe
+openclaw gateway restart --force
+```
+
+`openclaw gateway restart --safe` asks the running Gateway to preflight active OpenClaw work before restarting. If queued operations, reply delivery, embedded runs, or task runs are active, the Gateway reports the blockers, coalesces duplicate safe restart requests, and restarts once the active work drains. Plain `restart` keeps the existing service-manager behavior for compatibility. Use `--force` only when you explicitly want the immediate override path.
+
 <Warning>
 Inline `--password` can be exposed in local process listings. Prefer `--password-file`, env, or a SecretRef-backed `gateway.auth.password`.
 </Warning>
diff --git a/src/cli/daemon-cli/lifecycle.test.ts b/src/cli/daemon-cli/lifecycle.test.ts
@@ -49,6 +49,7 @@ const probeGateway = vi.fn<
     configSnapshot: unknown;
   }>
 >();
+const callGatewayCli = vi.fn();
 const isRestartEnabled = vi.fn<(config?: { commands?: unknown }) => boolean>(() => true);
 const loadConfig = vi.hoisted(() => vi.fn(() => ({})));
 const recoverInstalledLaunchAgent = vi.hoisted(() => vi.fn());
@@ -77,6 +78,10 @@ vi.mock("../../gateway/probe.js", () => ({
   }) => probeGateway(opts),
 }));
 
+vi.mock("../../gateway/call.js", () => ({
+  callGatewayCli: (opts: unknown) => callGatewayCli(opts),
+}));
+
 vi.mock("../../config/commands.js", () => ({
   isRestartEnabled: (config?: { commands?: unknown }) => isRestartEnabled(config),
 }));
@@ -113,7 +118,11 @@ vi.mock("./lifecycle-core.js", () => ({
 
 describe("runDaemonRestart health checks", () => {
   let runDaemonStart: (opts?: { json?: boolean }) => Promise<void>;
-  let runDaemonRestart: (opts?: { json?: boolean }) => Promise<boolean>;
+  let runDaemonRestart: (opts?: {
+    json?: boolean;
+    safe?: boolean;
+    force?: boolean;
+  }) => Promise<boolean>;
   let runDaemonStop: (opts?: { json?: boolean }) => Promise<void>;
   let envSnapshot: ReturnType<typeof captureEnv>;
 
@@ -162,6 +171,7 @@ describe("runDaemonRestart health checks", () => {
     signalVerifiedGatewayPidSync.mockReset();
     formatGatewayPidList.mockReset();
     probeGateway.mockReset();
+    callGatewayCli.mockReset();
     isRestartEnabled.mockReset();
     loadConfig.mockReset();
     recoverInstalledLaunchAgent.mockReset();
@@ -204,6 +214,31 @@ describe("runDaemonRestart health checks", () => {
       ok: true,
       configSnapshot: { commands: { restart: true } },
     });
+    callGatewayCli.mockResolvedValue({
+      ok: true,
+      status: "deferred",
+      preflight: {
+        safe: false,
+        counts: {
+          queueSize: 1,
+          pendingReplies: 0,
+          embeddedRuns: 0,
+          activeTasks: 0,
+          totalActive: 1,
+        },
+        blockers: [{ kind: "queue", count: 1, message: "1 queued or active operation(s)" }],
+        summary: "restart deferred: 1 queued or active operation(s)",
+      },
+      restart: {
+        ok: true,
+        pid: 123,
+        signal: "SIGUSR1",
+        delayMs: 0,
+        mode: "emit",
+        coalesced: false,
+        cooldownMsApplied: 0,
+      },
+    });
     isRestartEnabled.mockReturnValue(true);
     signalVerifiedGatewayPidSync.mockImplementation(() => {});
     formatGatewayPidList.mockImplementation((pids) => pids.join(", "));
@@ -230,6 +265,24 @@ describe("runDaemonRestart health checks", () => {
     expect(recoverInstalledLaunchAgent).toHaveBeenCalledWith({ result: "started" });
   });
 
+  it("requests a safe gateway restart over RPC without touching the service manager", async () => {
+    await runDaemonRestart({ json: true, safe: true });
+
+    expect(callGatewayCli).toHaveBeenCalledWith({
+      method: "gateway.restart.request",
+      params: { reason: "gateway.restart.safe" },
+      timeoutMs: 10_000,
+    });
+    expect(runServiceRestart).not.toHaveBeenCalled();
+  });
+
+  it("keeps force restart on the existing non-safe path", async () => {
+    await runDaemonRestart({ json: true, force: true });
+
+    expect(callGatewayCli).not.toHaveBeenCalled();
+    expect(runServiceRestart).toHaveBeenCalled();
+  });
+
   it("repairs stale loaded service definitions from gateway start", async () => {
     repairLoadedGatewayServiceForStart.mockResolvedValue({
       result: "started",
diff --git a/src/cli/daemon-cli/lifecycle.ts b/src/cli/daemon-cli/lifecycle.ts
@@ -1,12 +1,14 @@
 import { isRestartEnabled } from "../../config/commands.flags.js";
 import { readBestEffortConfig, resolveGatewayPort } from "../../config/config.js";
 import { resolveGatewayService } from "../../daemon/service.js";
+import { callGatewayCli } from "../../gateway/call.js";
 import { probeGateway } from "../../gateway/probe.js";
 import {
   findVerifiedGatewayListenerPidsOnPortSync,
   formatGatewayPidList,
   signalVerifiedGatewayPidSync,
 } from "../../infra/gateway-processes.js";
+import type { SafeGatewayRestartRequestResult } from "../../infra/restart-coordinator.js";
 import { type GatewayRestartIntent, writeGatewayRestartIntentSync } from "../../infra/restart.js";
 import { defaultRuntime } from "../../runtime.js";
 import { normalizeOptionalString } from "../../shared/string-coerce.js";
@@ -139,6 +141,50 @@ function resolveGatewayRestartIntentOptions(
   return undefined;
 }
 
+function formatSafeRestartWarnings(result: SafeGatewayRestartRequestResult): string[] | undefined {
+  if (result.preflight.blockers.length === 0) {
+    return undefined;
+  }
+  return [result.preflight.summary];
+}
+
+async function requestSafeGatewayRestart(opts: DaemonLifecycleOptions): Promise<boolean> {
+  if (opts.force) {
+    throw new Error("--safe cannot be combined with --force; omit --safe to force restart now");
+  }
+  if (opts.wait !== undefined) {
+    throw new Error("--safe cannot be combined with --wait; safe restart uses gateway deferral");
+  }
+  const result = await callGatewayCli<SafeGatewayRestartRequestResult>({
+    method: "gateway.restart.request",
+    params: { reason: "gateway.restart.safe" },
+    timeoutMs: 10_000,
+  });
+  const message =
+    result.status === "coalesced"
+      ? "safe restart request joined an existing pending gateway restart"
+      : result.status === "deferred"
+        ? "safe restart requested; gateway will restart after active work drains"
+        : "safe restart requested; gateway will restart momentarily";
+  const payload = {
+    ok: true,
+    result: result.status,
+    message,
+    preflight: result.preflight,
+    restart: result.restart,
+    warnings: formatSafeRestartWarnings(result),
+  };
+  if (opts.json) {
+    defaultRuntime.log(JSON.stringify(payload, null, 2));
+  } else {
+    defaultRuntime.log(message);
+    if (result.preflight.blockers.length > 0) {
+      defaultRuntime.log(theme.warn(result.preflight.summary));
+    }
+  }
+  return true;
+}
+
 async function restartGatewayWithoutServiceManager(
   port: number,
   restartIntent?: GatewayRestartIntent,
@@ -218,6 +264,9 @@ export async function runDaemonStop(opts: DaemonLifecycleOptions = {}) {
  * Throws/exits on check or restart failures.
  */
 export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promise<boolean> {
+  if (opts.safe) {
+    return await requestSafeGatewayRestart(opts);
+  }
   const json = Boolean(opts.json);
   const service = resolveGatewayService();
   let restartedWithoutServiceManager = false;
diff --git a/src/cli/daemon-cli/register-service-commands.test.ts b/src/cli/daemon-cli/register-service-commands.test.ts
@@ -70,6 +70,17 @@ describe("addGatewayServiceCommands", () => {
         );
       },
     },
+    {
+      name: "forwards restart safe control",
+      argv: ["restart", "--safe"],
+      assert: () => {
+        expect(runDaemonRestart).toHaveBeenCalledWith(
+          expect.objectContaining({
+            safe: true,
+          }),
+        );
+      },
+    },
     {
       name: "forwards restart force control",
       argv: ["restart", "--force"],
diff --git a/src/cli/daemon-cli/register-service-commands.ts b/src/cli/daemon-cli/register-service-commands.ts
@@ -49,6 +49,7 @@ function resolveRestartOptions(cmdOpts: DaemonLifecycleOptions, command?: Comman
   return {
     ...cmdOpts,
     force: Boolean(cmdOpts.force || parentForce),
+    safe: Boolean(cmdOpts.safe),
   };
 }
 
@@ -122,6 +123,7 @@ export function addGatewayServiceCommands(parent: Command, opts?: { statusDescri
     .command("restart")
     .description("Restart the Gateway service (launchd/systemd/schtasks)")
     .option("--force", "Restart immediately without waiting for active gateway work", false)
+    .option("--safe", "Request an OpenClaw-aware restart after active work drains", false)
     .option(
       "--wait <duration>",
       "Wait duration before forcing restart (ms, 10s, 5m; 0 waits indefinitely)",
diff --git a/src/cli/daemon-cli/types.ts b/src/cli/daemon-cli/types.ts
@@ -27,5 +27,6 @@ export type DaemonInstallOptions = {
 export type DaemonLifecycleOptions = {
   json?: boolean;
   force?: boolean;
+  safe?: boolean;
   wait?: string;
 };
diff --git a/src/cli/gateway-cli/run-loop.test.ts b/src/cli/gateway-cli/run-loop.test.ts
@@ -390,7 +390,9 @@ describe("runGatewayLoop", () => {
       expect(waitForActiveEmbeddedRuns).not.toHaveBeenCalled();
       expect(abortEmbeddedPiRun).toHaveBeenCalledWith(undefined, { mode: "all" });
       expect(gatewayLog.warn).toHaveBeenCalledWith(
-        expect.stringContaining("restart blocked by active task run(s): taskId=task-force"),
+        expect.stringContaining(
+          "restart blocked by active background task run(s): taskId=task-force",
+        ),
       );
       expect(gatewayLog.warn).toHaveBeenCalledWith(
         "forced restart requested; skipping active work drain",
diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts
@@ -392,7 +392,7 @@ export async function runGatewayLoop(params: {
               `draining ${activeTasks} active task(s) and ${activeRuns} active embedded run(s) before restart ${formatRestartDrainBudget()}`,
             );
             if (taskBlockers) {
-              gatewayLog.warn(`restart blocked by active task run(s): ${taskBlockers}`);
+              gatewayLog.warn(`restart blocked by active background task run(s): ${taskBlockers}`);
             }
             if (restartIntent?.force) {
               gatewayLog.warn("forced restart requested; skipping active work drain");
diff --git a/src/gateway/method-scopes.ts b/src/gateway/method-scopes.ts
@@ -112,6 +112,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
     "cron.status",
     "cron.runs",
     "gateway.identity.get",
+    "gateway.restart.preflight",
     "system-presence",
     "last-heartbeat",
     "node.list",
@@ -199,6 +200,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
     "system-event",
     "agents.files.set",
     "update.status",
+    "gateway.restart.request",
   ],
   [TALK_SECRETS_SCOPE]: [],
 };
diff --git a/src/gateway/server-methods-list.ts b/src/gateway/server-methods-list.ts
@@ -148,6 +148,8 @@ const BASE_METHODS = [
   "cron.run",
   "cron.runs",
   "gateway.identity.get",
+  "gateway.restart.preflight",
+  "gateway.restart.request",
   "system-presence",
   "system-event",
   "message.action",
diff --git a/src/gateway/server-methods.ts b/src/gateway/server-methods.ts
@@ -30,6 +30,7 @@ import { nodePendingHandlers } from "./server-methods/nodes-pending.js";
 import { nodeHandlers } from "./server-methods/nodes.js";
 import { pluginHostHookHandlers } from "./server-methods/plugin-host-hooks.js";
 import { pushHandlers } from "./server-methods/push.js";
+import { restartHandlers } from "./server-methods/restart.js";
 import { sendHandlers } from "./server-methods/send.js";
 import { sessionsHandlers } from "./server-methods/sessions.js";
 import { skillsHandlers } from "./server-methods/skills.js";
@@ -47,7 +48,12 @@ import { voicewakeHandlers } from "./server-methods/voicewake.js";
 import { webHandlers } from "./server-methods/web.js";
 import { wizardHandlers } from "./server-methods/wizard.js";
 
-const CONTROL_PLANE_WRITE_METHODS = new Set(["config.apply", "config.patch", "update.run"]);
+const CONTROL_PLANE_WRITE_METHODS = new Set([
+  "config.apply",
+  "config.patch",
+  "gateway.restart.request",
+  "update.run",
+]);
 function authorizeGatewayMethod(method: string, client: GatewayRequestOptions["client"]) {
   if (!client?.connect) {
     return null;
@@ -110,6 +116,7 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
   ...nodeHandlers,
   ...nodePendingHandlers,
   ...pushHandlers,
+  ...restartHandlers,
   ...sendHandlers,
   ...usageHandlers,
   ...agentHandlers,
diff --git a/src/gateway/server-methods/restart.ts b/src/gateway/server-methods/restart.ts
@@ -0,0 +1,22 @@
+import {
+  createSafeGatewayRestartPreflight,
+  requestSafeGatewayRestart,
+} from "../../infra/restart-coordinator.js";
+import type { GatewayRequestHandlers } from "./types.js";
+
+function normalizeReason(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value.trim().slice(0, 200) : undefined;
+}
+
+export const restartHandlers: GatewayRequestHandlers = {
+  "gateway.restart.request": async ({ respond, params }) => {
+    const result = requestSafeGatewayRestart({
+      reason: normalizeReason(params.reason),
+      delayMs: 0,
+    });
+    respond(true, result);
+  },
+  "gateway.restart.preflight": async ({ respond }) => {
+    respond(true, createSafeGatewayRestartPreflight());
+  },
+};
diff --git a/src/gateway/server-reload-handlers.test.ts b/src/gateway/server-reload-handlers.test.ts
@@ -126,7 +126,9 @@ describe("gateway restart deferral preflight", () => {
       );
 
       expect(logReload.warn).toHaveBeenCalledWith(
-        expect.stringContaining("restart blocked by active task run(s): taskId=task-nightly"),
+        expect.stringContaining(
+          "restart blocked by active background task run(s): taskId=task-nightly",
+        ),
       );
       expect(logReload.warn).toHaveBeenCalledWith(expect.stringContaining("runId=run-nightly"));
 
diff --git a/src/gateway/server-reload-handlers.ts b/src/gateway/server-reload-handlers.ts
@@ -22,7 +22,6 @@ import {
 } from "../secrets/runtime.js";
 import {
   getInspectableActiveTaskRestartBlockers,
-  getInspectableTaskRegistrySummary,
   type ActiveTaskRestartBlocker,
 } from "../tasks/task-registry.maintenance.js";
 import type { ChannelHealthMonitor } from "./channel-health-monitor.js";
@@ -143,7 +142,7 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
     const queueSize = getTotalQueueSize();
     const pendingReplies = getTotalPendingReplies();
     const embeddedRuns = getActiveEmbeddedRunCount();
-    const activeTasks = getInspectableTaskRegistrySummary().active;
+    const activeTasks = getInspectableActiveTaskRestartBlockers().length;
     return {
       queueSize,
       pendingReplies,
@@ -164,7 +163,7 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
       details.push(`${counts.embeddedRuns} embedded run(s)`);
     }
     if (counts.activeTasks > 0) {
-      details.push(`${counts.activeTasks} task run(s)`);
+      details.push(`${counts.activeTasks} background task run(s)`);
     }
     return details;
   };
@@ -420,7 +419,7 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
       );
       const taskBlockers = formatTaskBlockers();
       if (taskBlockers) {
-        params.logReload.warn(`restart blocked by active task run(s): ${taskBlockers}`);
+        params.logReload.warn(`restart blocked by active background task run(s): ${taskBlockers}`);
       }
 
       deferGatewayRestartUntilIdle({
diff --git a/src/gateway/server-startup-early.ts b/src/gateway/server-startup-early.ts
diff --git a/src/gateway/server.reload.test.ts b/src/gateway/server.reload.test.ts
diff --git a/src/infra/restart-coordinator.test.ts b/src/infra/restart-coordinator.test.ts
diff --git a/src/infra/restart-coordinator.ts b/src/infra/restart-coordinator.ts
diff --git a/src/tasks/task-registry.maintenance.issue-60299.test.ts b/src/tasks/task-registry.maintenance.issue-60299.test.ts
diff --git a/src/tasks/task-registry.maintenance.ts b/src/tasks/task-registry.maintenance.ts

Original file line number	Diff line number	Diff line change
`@@ -392,7 +392,7 @@ export async function runGatewayLoop(params: {`
`392`	`392`	`draining ${activeTasks} active task(s) and ${activeRuns} active embedded run(s) before restart ${formatRestartDrainBudget()}`,
`393`	`393`	`);`
`394`	`394`	`if (taskBlockers) {`
`395`		- gatewayLog.warn(`restart blocked by active task run(s): ${taskBlockers}`);
	`395`	+ gatewayLog.warn(`restart blocked by active background task run(s): ${taskBlockers}`);
`396`	`396`	`}`
`397`	`397`	`if (restartIntent?.force) {`
`398`	`398`	`gatewayLog.warn("forced restart requested; skipping active work drain");`