feat: harden live gateway and mac restarts (#104314)

steipete · web-flow · commit 2ed24246182f · 2026-07-11T01:41:27.000-07:00
* feat: audit gateway logs after live restarts

* fix: keep mac app running during replacement build
diff --git a/.agents/skills/openclaw-live-updater/SKILL.md b/.agents/skills/openclaw-live-updater/SKILL.md
@@ -32,6 +32,7 @@ Keep `/Users/steipete/openclaw` a read-only-to-the-agent deployment mirror: clea
    - A dependency-input change, absent `node_modules`, or missing/invalid build provenance first runs `pnpm install --frozen-lockfile`.
    - `pnpm build` must leave both canonical stamp heads and `dist/build-info.json.commit` equal to post-update `afterSha`; any missing/mismatched stamp or required artifact blocks restart.
    - Only after exact-SHA build proof may it restart the managed Gateway and require `gateway status --deep --require-rpc --json` plus `health --verbose --json`.
+   - After every managed restart, query Gateway logs through RPC, restrict the audit to entries emitted since that restart began, report warning summaries, and fail the pass on any error/fatal entry. Never accept supervisor or RPC health without this restart-window log audit.
 
    Treat supervisor state alone as insufficient. If build or proof fails, leave the new mirror head intact and retry the stale/missing build on the next heartbeat; never run the old `dist` against new source.
 
diff --git a/.agents/skills/openclaw-live-updater/scripts/update-main.mjs b/.agents/skills/openclaw-live-updater/scripts/update-main.mjs
@@ -524,7 +524,9 @@ function assertExactBuild(checkout, expectedSha) {
 
 function restartGateway(runCommand, checkout, expectedSha) {
   assertExactBuild(checkout, expectedSha);
+  const startedAtMs = Date.now();
   runCommand("pnpm", ["openclaw", "gateway", "restart"], checkout);
+  return startedAtMs;
 }
 
 function verifyGateway(runCommand, checkout, expectedSha) {
@@ -537,6 +539,69 @@ function verifyGateway(runCommand, checkout, expectedSha) {
   runCommand("pnpm", ["openclaw", "health", "--verbose", "--json"], checkout);
 }
 
+function summarizeGatewayLogEntry(entry) {
+  return {
+    time: entry.time,
+    level: entry.level,
+    subsystem: entry.subsystem ?? null,
+    message: String(entry.message ?? "").slice(0, 500),
+  };
+}
+
+export function parseGatewayLogAudit(output, sinceMs) {
+  const entries = output
+    .split("\n")
+    .filter(Boolean)
+    .flatMap((line) => {
+      try {
+        const entry = JSON.parse(line);
+        const timestamp = Date.parse(entry.time ?? "");
+        return entry.type === "log" && Number.isFinite(timestamp) && timestamp >= sinceMs
+          ? [entry]
+          : [];
+      } catch {
+        return [];
+      }
+    });
+  const errors = entries
+    .filter((entry) => entry.level === "error" || entry.level === "fatal")
+    .map(summarizeGatewayLogEntry);
+  const warnings = entries.filter((entry) => entry.level === "warn").map(summarizeGatewayLogEntry);
+  return {
+    entries: entries.length,
+    errorCount: errors.length,
+    warningCount: warnings.length,
+    errors: errors.slice(0, 20),
+    warnings: warnings.slice(0, 20),
+  };
+}
+
+function defaultAuditGatewayLogs(checkout, sinceMs) {
+  const output = execFileSync(
+    process.execPath,
+    [
+      "openclaw.mjs",
+      "logs",
+      "--json",
+      "--limit",
+      "1000",
+      "--max-bytes",
+      "1000000",
+      "--timeout",
+      "10000",
+    ],
+    { cwd: checkout, encoding: "utf8", maxBuffer: 4 * 1024 * 1024 },
+  );
+  const audit = parseGatewayLogAudit(output, sinceMs);
+  if (audit.errorCount > 0) {
+    throw new UpdateInvariantError(
+      "gateway_restart_log_errors",
+      `Gateway emitted ${audit.errorCount} error/fatal log entries after restart: ${JSON.stringify(audit.errors.slice(0, 5))}`,
+    );
+  }
+  return audit;
+}
+
 export function findExactMacTarget(processes, executable) {
   const target = processes
     .split("\n")
@@ -601,6 +666,8 @@ export function maintainMain(options, dependencies = {}) {
     }
     const runCommand = dependencies.runCommand ?? defaultRunCommand;
     const verifyMacTarget = dependencies.verifyMacTarget ?? defaultVerifyMacTarget;
+    const auditGatewayLogs = dependencies.auditGatewayLogs ?? defaultAuditGatewayLogs;
+    let gatewayLogAudit = null;
     let queuedMacState = null;
     if (actions.macAppRebuild) {
       queuedMacState = {
@@ -619,16 +686,18 @@ export function maintainMain(options, dependencies = {}) {
     if (actions.gatewayBuild) {
       runCommand("pnpm", ["build"], update.checkout);
       assertExactBuild(update.checkout, update.afterSha);
-      restartGateway(runCommand, update.checkout, update.afterSha);
+      const restartStartedAt = restartGateway(runCommand, update.checkout, update.afterSha);
       verifyGateway(runCommand, update.checkout, update.afterSha);
+      gatewayLogAudit = auditGatewayLogs(update.checkout, restartStartedAt);
     } else {
       try {
         verifyGateway(runCommand, update.checkout, update.afterSha);
       } catch {
         actions.gatewayRestart = true;
         actions.gatewaySelfHeal = true;
-        restartGateway(runCommand, update.checkout, update.afterSha);
+        const restartStartedAt = restartGateway(runCommand, update.checkout, update.afterSha);
         verifyGateway(runCommand, update.checkout, update.afterSha);
+        gatewayLogAudit = auditGatewayLogs(update.checkout, restartStartedAt);
       }
     }
     if (actions.macAppRebuild) {
@@ -664,6 +733,7 @@ export function maintainMain(options, dependencies = {}) {
       buildBefore,
       buildChangedPaths,
       actions,
+      ...(gatewayLogAudit ? { gatewayLogAudit } : {}),
       ...(maintenanceState.macTarget ? { macTarget: maintenanceState.macTarget } : {}),
     };
   } finally {
diff --git a/scripts/restart-mac.sh b/scripts/restart-mac.sh
@@ -269,15 +269,13 @@ stop_launch_agent() {
   launchctl bootout gui/"$UID"/ai.openclaw.mac 2>/dev/null || true
 }
 
-# 1) Stop only the process set selected by the requested mode.
+# 1) Validate the process set selected by the requested mode. Target-only keeps
+# the current managed app alive while the replacement builds and signs.
 if [[ "$TARGET_ONLY" -eq 1 ]]; then
   if [[ -n "$(foreign_openclaw_process_pids)" ]]; then
     fail "Another OpenClaw app or test process is active; target-only restart deferred"
   fi
-  log "==> Killing managed installed and exact target OpenClaw instances"
-  if ! kill_managed_openclaw; then
-    fail "Managed OpenClaw instances did not exit after cleanup attempts"
-  fi
+  log "==> Keeping managed OpenClaw running while the replacement builds"
 else
   stop_launch_agent
   log "==> Killing existing OpenClaw instances"
@@ -378,6 +376,16 @@ if [[ "$ATTACH_ONLY" -eq 1 ]]; then
   ATTACH_ONLY_ARGS+=(--args --attach-only)
 fi
 
+if [[ "$TARGET_ONLY" -eq 1 ]]; then
+  if [[ -n "$(foreign_openclaw_process_pids)" ]]; then
+    fail "Another OpenClaw app or test process appeared during build; target-only restart deferred"
+  fi
+  log "==> Switching managed installed and exact target OpenClaw instances"
+  if ! kill_managed_openclaw; then
+    fail "Managed OpenClaw instances did not exit after cleanup attempts"
+  fi
+fi
+
 # 4) Launch the installed app in the foreground so the menu bar extra appears.
 # LaunchServices can inherit a huge environment from this shell (secrets, prompt vars, etc.).
 # That can cause launchd spawn failures and is undesirable for a GUI app anyway.
diff --git a/test/scripts/openclaw-live-updater.test.ts b/test/scripts/openclaw-live-updater.test.ts
@@ -5,6 +5,7 @@ import {
   mkdirSync,
   mkdtempSync,
   readFileSync,
+  realpathSync,
   renameSync,
   rmSync,
   symlinkSync,
@@ -21,6 +22,7 @@ import {
   inspectBuildState,
   maintainMain,
   originMatches,
+  parseGatewayLogAudit,
 } from "../../.agents/skills/openclaw-live-updater/scripts/update-main.mjs";
 import {
   BUILD_STAMP_FILE,
@@ -48,7 +50,17 @@ function maintainFixture(
   options: Record<string, unknown>,
   dependencies: Record<string, unknown> = {},
 ) {
-  return maintainMain(options, { fetchMain: fetchFixtureMain, ...dependencies });
+  return maintainMain(options, {
+    fetchMain: fetchFixtureMain,
+    auditGatewayLogs: () => ({
+      entries: 0,
+      errorCount: 0,
+      warningCount: 0,
+      errors: [],
+      warnings: [],
+    }),
+    ...dependencies,
+  });
 }
 
 function makeFixture() {
@@ -72,6 +84,7 @@ function makeFixture() {
   const canonicalOrigin = "https://github.com/openclaw/openclaw.git";
   git(mirror, "remote", "set-url", "origin", canonicalOrigin);
   fixtureOrigins.set(mirror, origin);
+  fixtureOrigins.set(realpathSync(mirror), origin);
   return { root, mirror, origin, seed };
 }
 
@@ -116,6 +129,53 @@ function fakeCommands(mirror: string) {
 }
 
 describe("openclaw live updater", () => {
+  test("audits only error and warning logs emitted after Gateway restart", () => {
+    const output = [
+      { type: "meta", file: "/tmp/openclaw.log" },
+      { type: "log", time: "2026-07-11T08:00:00.000Z", level: "error", message: "old" },
+      { type: "log", time: "2026-07-11T08:00:02.000Z", level: "info", message: "ready" },
+      {
+        type: "log",
+        time: "2026-07-11T08:00:03.000Z",
+        level: "warn",
+        subsystem: "gateway",
+        message: "degraded",
+      },
+      {
+        type: "log",
+        time: "2026-07-11T08:00:04.000Z",
+        level: "fatal",
+        subsystem: "gateway",
+        message: "failed",
+      },
+      { type: "notice", message: "done" },
+    ]
+      .map((entry) => JSON.stringify(entry))
+      .join("\n");
+
+    expect(parseGatewayLogAudit(output, Date.parse("2026-07-11T08:00:02.000Z"))).toEqual({
+      entries: 3,
+      errorCount: 1,
+      warningCount: 1,
+      errors: [
+        {
+          time: "2026-07-11T08:00:04.000Z",
+          level: "fatal",
+          subsystem: "gateway",
+          message: "failed",
+        },
+      ],
+      warnings: [
+        {
+          time: "2026-07-11T08:00:03.000Z",
+          level: "warn",
+          subsystem: "gateway",
+          message: "degraded",
+        },
+      ],
+    });
+  });
+
   test("accepts supported OpenClaw GitHub origins", () => {
     expect(originMatches("https://github.com/openclaw/openclaw.git")).toBe(true);
     expect(originMatches("git@github.com:openclaw/openclaw.git")).toBe(true);
@@ -275,6 +335,13 @@ describe("openclaw live updater", () => {
       macAppRebuild: true,
       macUiVerification: true,
     });
+    expect(output.gatewayLogAudit).toEqual({
+      entries: 0,
+      errorCount: 0,
+      warningCount: 0,
+      errors: [],
+      warnings: [],
+    });
     expect(commands.calls).toEqual([
       "pnpm install --frozen-lockfile",
       "pnpm build",
diff --git a/test/scripts/restart-mac.test.ts b/test/scripts/restart-mac.test.ts
@@ -372,20 +372,37 @@ describe("scripts/restart-mac.sh", () => {
 
   it("target-only mode refuses foreign app processes without broad cleanup", () => {
     const script = readFileSync(restartScriptPath, "utf8");
-    const targetBlock = script.slice(
+    const initialTargetBlock = script.slice(
       script.indexOf('if [[ "$TARGET_ONLY" -eq 1 ]]; then', script.indexOf("# 1)")),
       script.indexOf("else", script.indexOf("# 1)")),
     );
+    const switchTargetBlock = script.slice(
+      script.indexOf('if [[ "$TARGET_ONLY" -eq 1 ]]; then', script.indexOf("ATTACH_ONLY_ARGS")),
+      script.indexOf("# 4) Launch"),
+    );
 
-    expect(targetBlock).toContain("foreign_openclaw_process_pids");
-    expect(targetBlock).toContain("kill_managed_openclaw");
-    expect(targetBlock).not.toContain("stop_launch_agent");
-    expect(targetBlock).not.toContain("kill_all_openclaw");
+    expect(initialTargetBlock).toContain("foreign_openclaw_process_pids");
+    expect(initialTargetBlock).not.toContain("kill_managed_openclaw");
+    expect(initialTargetBlock).not.toContain("stop_launch_agent");
+    expect(initialTargetBlock).not.toContain("kill_all_openclaw");
+    expect(switchTargetBlock).toContain("foreign_openclaw_process_pids");
+    expect(switchTargetBlock).toContain("kill_managed_openclaw");
     expect(script).toContain('[[ "${executable}" == "${TARGET_EXECUTABLE}" ]] && continue');
     expect(script).toContain('process_pids_for_executable "${TARGET_EXECUTABLE}"');
     expect(script).toContain("target-only restart deferred");
   });
 
+  it("keeps the managed app alive until the signed replacement is ready", () => {
+    const script = readFileSync(restartScriptPath, "utf8");
+    const packageIndex = script.indexOf('run_step "package app"');
+    const switchIndex = script.indexOf('log "==> Switching managed installed');
+    const launchIndex = script.indexOf('run_step "launch app"');
+
+    expect(packageIndex).toBeGreaterThan(-1);
+    expect(switchIndex).toBeGreaterThan(packageIndex);
+    expect(launchIndex).toBeGreaterThan(switchIndex);
+  });
+
   it("escalates only exact managed app processes when graceful shutdown stalls", () => {
     const script = readFileSync(restartScriptPath, "utf8");
     const managedKillBlock = script.slice(
