fix: live updater survives launchd teardown and retargets wrapped gateways (#105022)

steipete · web-flow · commit 68553bed8437 · 2026-07-12T09:35:53.000+01:00
* fix(updater): tolerate launchd teardown and replace plist args safely

* test(updater): declare plist argument helper
diff --git a/.agents/skills/openclaw-live-updater/SKILL.md b/.agents/skills/openclaw-live-updater/SKILL.md
@@ -32,7 +32,7 @@ Keep `/Users/steipete/openclaw` a read-only-to-the-agent deployment mirror: clea
    - A dependency-input change, absent `node_modules`, or missing/invalid build provenance requires `pnpm install --frozen-lockfile`. When a build is required, do not install before acquiring the maintenance suspension and stopping the managed Gateway.
    - Before mutating any entrypoint currently executed by the Gateway, invoke an exact trusted source CLI to acquire `gateway.suspend.prepare`, binding both prepare and resume to this checkout's managed LaunchAgent loopback port and service auth even when normal CLI configuration points at a remote Gateway. The LaunchAgent may execute either this checkout's `dist/index.js` or a clean detached canonical snapshot under `~/.openclaw/runtime/gateway-<sha>` whose commit is an ancestor of the checkout; reject every other entrypoint. Never execute snapshot code: capture an exact source control build before the Git fast-forward for its prepare and failure-resume calls, preserve any validated generated service-environment wrapper, and stop the managed LaunchAgent with native launchd bootout. If that control build is missing, first accept native proof that the snapshot job is already booted out with its port free; when the isolated snapshot is still running, build the verified clean source checkout to obtain an exact suspension client, then use only that source client for prepare and failure-resume. Never use this recovery build while launchd targets the source checkout. This atomically pauses cron scheduling, closes new work admission, and refuses while active work remains. A busy result defers further mutation to the next heartbeat; never replace this fence with `cron list` polling. Once ready, stop directly without a source launcher, install frozen dependencies when required, then build unless the exact recovery build already produced the deployment artifact; source launchers can auto-build stale output before dispatching the stop. Resume the suspension if stop fails. If suspension RPC is unavailable on macOS, proceed only when native inspection proves this checkout's managed LaunchAgent is booted out and its configured port has no listener; never accept a loaded KeepAlive job's transient stopped state. On other platforms, require the existing CLI to prove the managed service is stopped with no PID, listener, or RPC. This preserves retry after a post-stop failure without weakening the live-work fence. Preserve `dist/OpenClaw.app` outside `dist` for the build and restore it even when the build fails, because the JS build cleans `dist` regardless of Mac impact classification. Never mutate the live `dist` tree while an old Gateway can dynamically import from it. `pnpm build` must leave both canonical stamp heads and `dist/build-info.json.commit` equal to post-update `afterSha`; any missing/mismatched stamp or required artifact blocks restart.
    - Snapshot ownership validation must never invoke Git inside the snapshot. Treat its worktree, local Git configuration, filters, hooks, attributes, and build artifacts as untrusted; prove only that the regular owned LaunchAgent entrypoint is under the canonical detached ancestor snapshot path. Snapshot validation authorizes native bootout and retargeting only. Every CLI path must reject snapshot execution and use the trusted source checkout build instead.
-   - Only after exact-SHA build proof may it restart the managed Gateway and require `gateway status --deep --require-rpc --json` plus `health --verbose --json`. A validated ancestor snapshot is suspension-only: prove the old launchd job is booted out with its port free, then atomically retarget only the owned LaunchAgent entrypoint to this checkout's exact `dist/index.js`, including on a retry where the build is already current. Preserve all other service arguments and environment unchanged. After restart, prove the loaded launchd PID owns the configured listener.
+   - Only after exact-SHA build proof may it restart the managed Gateway and require `gateway status --deep --require-rpc --json` plus `health --verbose --json`. A validated ancestor snapshot is suspension-only: prove the old launchd job is booted out with its port free, allowing bounded retries while launchd and the listener finish teardown, then atomically retarget only the owned LaunchAgent entrypoint to this checkout's exact `dist/index.js`, including on a retry where the build is already current. Replace the verified `ProgramArguments` array as one value; never use array-index plist mutation that can insert a duplicate argument. Preserve all other service arguments and environment unchanged. After restart, prove the loaded launchd PID owns the configured listener.
    - After every managed restart, query Gateway logs through RPC, restrict the audit to entries emitted since that restart began, report warning summaries, and fail the pass on any error/fatal entry. If RPC verification or log retrieval fails, still inspect the local structured log for that restart window. Never accept supervisor or RPC health without this restart-window log audit.
 
    Treat supervisor state alone as insufficient. If build or proof fails, leave the new mirror head intact and retry the stale/missing build on the next heartbeat; never run the old `dist` against new source.
diff --git a/.agents/skills/openclaw-live-updater/scripts/update-main.mjs b/.agents/skills/openclaw-live-updater/scripts/update-main.mjs
@@ -37,6 +37,8 @@ const DEFAULT_EXPECTED_ORIGIN = "openclaw/openclaw";
 const FULL_SHA_RE = /^[0-9a-f]{40}$/u;
 const GATEWAY_READINESS_ATTEMPTS = 3;
 const GATEWAY_READINESS_RETRY_DELAY_MS = 5_000;
+const GATEWAY_STOP_PROOF_ATTEMPTS = 100;
+const GATEWAY_STOP_PROOF_RETRY_DELAY_MS = 100;
 const GATEWAY_SUSPEND_TIMEOUT_MS = 10_000;
 const GENERATED_LAUNCH_AGENT_ENV_WRAPPER = `#!/bin/sh
 set -eu
@@ -875,23 +877,43 @@ export function repointManagedGatewayDeployment(
   };
 }
 
+export function replaceLaunchAgentProgramArgument(programArguments, index, expected, replacement) {
+  if (!Array.isArray(programArguments) || programArguments[index] !== expected) {
+    throw new UpdateInvariantError(
+      "gateway_repoint_failed",
+      "managed Gateway LaunchAgent changed before its entrypoint could be replaced",
+    );
+  }
+  return programArguments.with(index, replacement);
+}
+
 function replaceLaunchAgentEntrypoint(deployment, entrypoint) {
-  const original = readFileSync(deployment.plistPath);
   const temporaryPath = `${deployment.plistPath}.openclaw-live-updater-${randomUUID()}`;
-  writeFileSync(temporaryPath, original, {
+  writeFileSync(temporaryPath, readFileSync(deployment.plistPath), {
     flag: "wx",
     mode: statSync(deployment.plistPath).mode,
   });
   try {
+    const plistResult = spawnSync(
+      "/usr/bin/plutil",
+      ["-convert", "json", "-o", "-", temporaryPath],
+      { encoding: "utf8" },
+    );
+    if (plistResult.status !== 0) {
+      throw new UpdateInvariantError(
+        "gateway_repoint_failed",
+        `could not read the managed Gateway LaunchAgent: ${String(plistResult.stderr).trim()}`,
+      );
+    }
+    const programArguments = replaceLaunchAgentProgramArgument(
+      JSON.parse(plistResult.stdout)?.ProgramArguments,
+      deployment.entrypointIndex,
+      deployment.entrypoint,
+      entrypoint,
+    );
     execFileSync(
       "/usr/bin/plutil",
-      [
-        "-replace",
-        `ProgramArguments.${deployment.entrypointIndex}`,
-        "-string",
-        entrypoint,
-        temporaryPath,
-      ],
+      ["-replace", "ProgramArguments", "-json", JSON.stringify(programArguments), temporaryPath],
       { stdio: ["ignore", "ignore", "pipe"] },
     );
     execFileSync("/usr/bin/plutil", ["-lint", temporaryPath], {
@@ -1143,6 +1165,33 @@ function stopManagedGateway(runCommand, checkout, deployment) {
   );
 }
 
+function stopManagedGatewayAndProve(runCommand, checkout, deployment, proveGatewayStopped, sleep) {
+  let stopError;
+  try {
+    stopManagedGateway(runCommand, checkout, deployment);
+  } catch (error) {
+    stopError = error;
+  }
+  let proofError;
+  for (let attempt = 0; attempt < GATEWAY_STOP_PROOF_ATTEMPTS; attempt += 1) {
+    try {
+      return proveGatewayStopped(checkout);
+    } catch (error) {
+      proofError = error;
+      if (attempt + 1 < GATEWAY_STOP_PROOF_ATTEMPTS) {
+        sleep(GATEWAY_STOP_PROOF_RETRY_DELAY_MS);
+      }
+    }
+  }
+  if (!stopError) {
+    throw proofError;
+  }
+  throw new AggregateError(
+    [stopError, proofError],
+    "Gateway stop command failed and native stopped proof did not converge",
+  );
+}
+
 function isTrustedSourceControlBuild(checkout, buildState, currentHead) {
   if (buildState.current) {
     return true;
@@ -1802,10 +1851,15 @@ export function maintainMain(options, dependencies = {}) {
         // Native bootout prevents launchd from retaining old ProgramArguments
         // and avoids source launchers that can rebuild stale dist before stopping.
         try {
-          stopManagedGateway(runCommand, update.checkout, gatewayDeploymentBefore);
-          // Retarget only after launchd has discarded the old ProgramArguments.
-          // Otherwise a later kickstart can revive its cached snapshot command.
-          proveGatewayStopped(update.checkout);
+          // launchctl can return before the job and listener have disappeared.
+          // Retarget only after bounded native proof prevents cached snapshot revival.
+          stopManagedGatewayAndProve(
+            runCommand,
+            update.checkout,
+            gatewayDeploymentBefore,
+            proveGatewayStopped,
+            sleep,
+          );
         } catch (error) {
           try {
             resumeSuspension(
diff --git a/test/external-script-modules.d.ts b/test/external-script-modules.d.ts
@@ -159,6 +159,12 @@ declare module "*openclaw-live-updater/scripts/update-main.mjs" {
     home: string,
     stateDir?: string,
   ): string | null;
+  export function replaceLaunchAgentProgramArgument(
+    programArguments: unknown,
+    index: number,
+    expected: string,
+    replacement: string,
+  ): string[];
   export function repointManagedGatewayDeployment(
     checkout: string,
     deployment: GatewayDeployment,
diff --git a/test/scripts/openclaw-live-updater.test.ts b/test/scripts/openclaw-live-updater.test.ts
@@ -27,6 +27,7 @@ import {
   parseGatewayLogAudit,
   parseLaunchctlArguments,
   prepareGatewaySuspension,
+  replaceLaunchAgentProgramArgument,
   repointManagedGatewayDeployment,
   resolveManagedGatewayEntrypoint,
   runBuiltGatewayCall,
@@ -502,6 +503,29 @@ describe("openclaw live updater", () => {
     });
   });
 
+  test("replaces a wrapped LaunchAgent entrypoint without inserting another argument", () => {
+    const snapshot = "/Users/test/.openclaw/runtime/gateway-1234567/dist/index.js";
+    const source = "/Users/test/openclaw/dist/index.js";
+    const original = [
+      "/bin/sh",
+      "/Users/test/.openclaw/service-env/ai.openclaw.gateway-env-wrapper.sh",
+      "/Users/test/.openclaw/service-env/ai.openclaw.gateway.env",
+      "/opt/homebrew/bin/node",
+      snapshot,
+      "gateway",
+      "--port",
+      "18789",
+    ];
+
+    expect(replaceLaunchAgentProgramArgument(original, 4, snapshot, source)).toEqual([
+      ...original.slice(0, 4),
+      source,
+      ...original.slice(5),
+    ]);
+    expect(original).toHaveLength(8);
+    expect(original[4]).toBe(snapshot);
+  });
+
   test("fails closed when Gateway service retargeting does not stick", () => {
     const checkout = "/Users/test/openclaw";
     const snapshot = "/Users/test/.openclaw/runtime/gateway-1234567/dist/index.js";
@@ -790,26 +814,57 @@ describe("openclaw live updater", () => {
     expect(inspectBuildState(mirror, git(mirror, "rev-parse", "HEAD")).current).toBe(false);
   });
 
-  test("resumes a prepared suspension when the managed Gateway stop fails", () => {
+  test("accepts native stopped proof when the stop command reports an error", () => {
     const { root, mirror } = makeFixture();
     mkdirSync(path.join(mirror, "node_modules"));
+    const commands = fakeCommands(mirror);
     const resumed: string[] = [];
 
+    const output = maintainFixture(
+      { checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
+      {
+        runCommand(command: string, args: string[]) {
+          if (command === process.execPath && args.includes("stop")) {
+            throw new Error("stop failed after stopping");
+          }
+          commands.runCommand(command, args);
+        },
+        resumeGatewaySuspension: (_checkout: string, suspensionId: string) => {
+          resumed.push(suspensionId);
+        },
+      },
+    );
+
+    expect(output.ok).toBe(true);
+    expect(resumed).toEqual([]);
+  });
+
+  test("resumes a prepared suspension when stopped proof never converges", () => {
+    const { root, mirror } = makeFixture();
+    mkdirSync(path.join(mirror, "node_modules"));
+    const resumed: string[] = [];
+    let proofAttempts = 0;
+    let sleepAttempts = 0;
+
     expect(() =>
       maintainFixture(
         { checkout: mirror, remote: "origin", lockPath: path.join(root, "maintenance.lock") },
         {
-          runCommand(command: string, args: string[]) {
-            if (command === process.execPath && args.includes("stop")) {
-              throw new Error("stop failed");
-            }
+          proveGatewayStopped: () => {
+            proofAttempts += 1;
+            throw new Error("listener still present");
+          },
+          sleep: () => {
+            sleepAttempts += 1;
           },
           resumeGatewaySuspension: (_checkout: string, suspensionId: string) => {
             resumed.push(suspensionId);
           },
         },
       ),
-    ).toThrow("stop failed");
+    ).toThrow("native stopped proof did not converge");
+    expect(proofAttempts).toBe(100);
+    expect(sleepAttempts).toBe(99);
     expect(resumed).toEqual(["fixture-suspension"]);
   });
 
@@ -1138,7 +1193,7 @@ describe("openclaw live updater", () => {
         proveGatewayStopped: () => {
           stoppedProofAttempts += 1;
           commands.calls.push("prove gateway stopped");
-          if (stoppedProofAttempts === 1) {
+          if (stoppedProofAttempts <= 2) {
             throw new Error("snapshot still owns its listener");
           }
           return {
@@ -1148,6 +1203,9 @@ describe("openclaw live updater", () => {
             proofSource: "fixture",
           };
         },
+        sleep: (ms: number) => {
+          commands.calls.push(`sleep ${ms}`);
+        },
         repointGatewayDeployment: (
           _checkout: string,
           deployment: { entrypoint: string; invocationPrefix: string[] },
@@ -1171,7 +1229,7 @@ describe("openclaw live updater", () => {
     );
 
     expect(controlEntrypoint).toBe(source);
-    expect(stoppedProofAttempts).toBe(2);
+    expect(stoppedProofAttempts).toBe(3);
     expect(output.gatewayDeployment).toMatchObject({
       changed: true,
       entrypoint: source,
@@ -1184,6 +1242,8 @@ describe("openclaw live updater", () => {
       "pnpm build",
       `/bin/launchctl bootout gui/${uid}/ai.openclaw.gateway`,
       "prove gateway stopped",
+      "sleep 100",
+      "prove gateway stopped",
       `/bin/launchctl enable gui/${uid}/ai.openclaw.gateway`,
       `/bin/launchctl bootstrap gui/${uid} ${plistPath}`,
     ]);
