fix(daemon): stabilize LaunchAgent restart and proxy env passthrough (#27276)

frankekn · gumadeiras · web-flow · commit b9757114290f · 2026-02-26T02:40:48.000-05:00
Merged via /review-pr -> /prepare-pr -> /merge-pr. Prepared head SHA: b08797a Co-authored-by: frankekn <4488090+frankekn@users.noreply.github.com> Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com> Reviewed-by: @gumadeiras
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ Docs: https://docs.openclaw.ai
 
 ### Fixes
 
+- Daemon/macOS launchd: forward proxy env vars into supervised service environments, switch LaunchAgent keepalive policy to crash-only with throttling, and harden restart sequencing to `print -> bootout -> wait old pid exit -> bootstrap -> kickstart`. (#27276) thanks @frankekn.
 - Android/Node invoke: remove native gateway WebSocket `Origin` header to avoid false origin rejections, unify invoke command registry/policy/error parsing paths, and keep command availability checks centralized to reduce dispatcher/advertisement drift. (#27257) Thanks @obviyus.
 - CI/Windows: shard the Windows `checks-windows` test lane into two matrix jobs and honor explicit shard index overrides in `scripts/test-parallel.mjs` to reduce CI critical-path wall time. (#27234) Thanks @joshavant.
 
diff --git a/src/daemon/launchd-plist.ts b/src/daemon/launchd-plist.ts
@@ -1,5 +1,7 @@
 import fs from "node:fs/promises";
 
+const LAUNCHD_THROTTLE_INTERVAL_SECONDS = 5;
+
 const plistEscape = (value: string): string =>
   value
     .replaceAll("&", "&amp;")
@@ -106,5 +108,5 @@ export function buildLaunchAgentPlist({
     ? `\n    <key>Comment</key>\n    <string>${plistEscape(comment.trim())}</string>`
     : "";
   const envXml = renderEnvDict(environment);
-  return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0">\n  <dict>\n    <key>Label</key>\n    <string>${plistEscape(label)}</string>\n    ${commentXml}\n    <key>RunAtLoad</key>\n    <true/>\n    <key>KeepAlive</key>\n    <true/>\n    <key>ProgramArguments</key>\n    <array>${argsXml}\n    </array>\n    ${workingDirXml}\n    <key>StandardOutPath</key>\n    <string>${plistEscape(stdoutPath)}</string>\n    <key>StandardErrorPath</key>\n    <string>${plistEscape(stderrPath)}</string>${envXml}\n  </dict>\n</plist>\n`;
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0">\n  <dict>\n    <key>Label</key>\n    <string>${plistEscape(label)}</string>\n    ${commentXml}\n    <key>RunAtLoad</key>\n    <true/>\n    <key>KeepAlive</key>\n    <dict>\n      <key>SuccessfulExit</key>\n      <false/>\n    </dict>\n    <key>ThrottleInterval</key>\n    <integer>${LAUNCHD_THROTTLE_INTERVAL_SECONDS}</integer>\n    <key>ProgramArguments</key>\n    <array>${argsXml}\n    </array>\n    ${workingDirXml}\n    <key>StandardOutPath</key>\n    <string>${plistEscape(stdoutPath)}</string>\n    <key>StandardErrorPath</key>\n    <string>${plistEscape(stderrPath)}</string>${envXml}\n  </dict>\n</plist>\n`;
 }
diff --git a/src/daemon/launchd.integration.test.ts b/src/daemon/launchd.integration.test.ts
@@ -0,0 +1,112 @@
+import { spawnSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { PassThrough } from "node:stream";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import {
+  installLaunchAgent,
+  readLaunchAgentRuntime,
+  restartLaunchAgent,
+  resolveLaunchAgentPlistPath,
+  uninstallLaunchAgent,
+} from "./launchd.js";
+import type { GatewayServiceEnv } from "./service-types.js";
+
+const WAIT_INTERVAL_MS = 200;
+const WAIT_TIMEOUT_MS = 15_000;
+
+function canRunLaunchdIntegration(): boolean {
+  if (process.platform !== "darwin") {
+    return false;
+  }
+  if (typeof process.getuid !== "function") {
+    return false;
+  }
+  const domain = `gui/${process.getuid()}`;
+  const probe = spawnSync("launchctl", ["print", domain], { encoding: "utf8" });
+  if (probe.error) {
+    return false;
+  }
+  return probe.status === 0;
+}
+
+const describeLaunchdIntegration = canRunLaunchdIntegration() ? describe : describe.skip;
+
+async function waitForRunningRuntime(params: {
+  env: GatewayServiceEnv;
+  pidNot?: number;
+  timeoutMs?: number;
+}): Promise<{ pid: number }> {
+  const timeoutMs = params.timeoutMs ?? WAIT_TIMEOUT_MS;
+  const deadline = Date.now() + timeoutMs;
+  let lastStatus = "unknown";
+  let lastPid: number | undefined;
+  while (Date.now() < deadline) {
+    const runtime = await readLaunchAgentRuntime(params.env);
+    lastStatus = runtime.status;
+    lastPid = runtime.pid;
+    if (
+      runtime.status === "running" &&
+      typeof runtime.pid === "number" &&
+      runtime.pid > 1 &&
+      (params.pidNot === undefined || runtime.pid !== params.pidNot)
+    ) {
+      return { pid: runtime.pid };
+    }
+    await new Promise((resolve) => {
+      setTimeout(resolve, WAIT_INTERVAL_MS);
+    });
+  }
+  throw new Error(
+    `Timed out waiting for launchd runtime (status=${lastStatus}, pid=${lastPid ?? "none"})`,
+  );
+}
+
+describeLaunchdIntegration("launchd integration", () => {
+  let env: GatewayServiceEnv | undefined;
+  let homeDir = "";
+  const stdout = new PassThrough();
+
+  beforeAll(async () => {
+    const testId = randomUUID().slice(0, 8);
+    homeDir = await fs.mkdtemp(path.join(os.tmpdir(), `openclaw-launchd-int-${testId}-`));
+    env = {
+      HOME: homeDir,
+      OPENCLAW_LAUNCHD_LABEL: `ai.openclaw.launchd-int-${testId}`,
+      OPENCLAW_LOG_PREFIX: `gateway-launchd-int-${testId}`,
+    };
+    await installLaunchAgent({
+      env,
+      stdout,
+      programArguments: [process.execPath, "-e", "setInterval(() => {}, 1000);"],
+    });
+    await waitForRunningRuntime({ env });
+  }, 30_000);
+
+  afterAll(async () => {
+    if (env) {
+      try {
+        await uninstallLaunchAgent({ env, stdout });
+      } catch {
+        // Best-effort cleanup in case launchctl state already changed.
+      }
+    }
+    if (homeDir) {
+      await fs.rm(homeDir, { recursive: true, force: true });
+    }
+  }, 30_000);
+
+  it("restarts launchd service and keeps it running with a new pid", async () => {
+    if (!env) {
+      throw new Error("launchd integration env was not initialized");
+    }
+    const before = await waitForRunningRuntime({ env });
+    await restartLaunchAgent({ env, stdout });
+    const after = await waitForRunningRuntime({ env, pidNot: before.pid });
+    expect(after.pid).toBeGreaterThan(1);
+    expect(after.pid).not.toBe(before.pid);
+    await fs.access(resolveLaunchAgentPlistPath(env));
+  }, 30_000);
+});
diff --git a/src/daemon/launchd.test.ts b/src/daemon/launchd.test.ts
@@ -5,12 +5,14 @@ import {
   isLaunchAgentListed,
   parseLaunchctlPrint,
   repairLaunchAgentBootstrap,
+  restartLaunchAgent,
   resolveLaunchAgentPlistPath,
 } from "./launchd.js";
 
 const state = vi.hoisted(() => ({
   launchctlCalls: [] as string[][],
   listOutput: "",
+  printOutput: "",
   bootstrapError: "",
   dirs: new Set<string>(),
   files: new Map<string, string>(),
@@ -35,6 +37,9 @@ vi.mock("./exec-file.js", () => ({
     if (call[0] === "list") {
       return { stdout: state.listOutput, stderr: "", code: 0 };
     }
+    if (call[0] === "print") {
+      return { stdout: state.printOutput, stderr: "", code: 0 };
+    }
     if (call[0] === "bootstrap" && state.bootstrapError) {
       return { stdout: "", stderr: state.bootstrapError, code: 1 };
     }
@@ -71,6 +76,7 @@ vi.mock("node:fs/promises", async (importOriginal) => {
 beforeEach(() => {
   state.launchctlCalls.length = 0;
   state.listOutput = "";
+  state.printOutput = "";
   state.bootstrapError = "";
   state.dirs.clear();
   state.files.clear();
@@ -179,6 +185,86 @@ describe("launchd install", () => {
     expect(plist).toContain(`<string>${tmpDir}</string>`);
   });
 
+  it("writes crash-only KeepAlive policy with throttle interval", async () => {
+    const env = createDefaultLaunchdEnv();
+    await installLaunchAgent({
+      env,
+      stdout: new PassThrough(),
+      programArguments: defaultProgramArguments,
+    });
+
+    const plistPath = resolveLaunchAgentPlistPath(env);
+    const plist = state.files.get(plistPath) ?? "";
+    expect(plist).toContain("<key>KeepAlive</key>");
+    expect(plist).toContain("<key>SuccessfulExit</key>");
+    expect(plist).toContain("<false/>");
+    expect(plist).toContain("<key>ThrottleInterval</key>");
+    expect(plist).toContain("<integer>5</integer>");
+  });
+
+  it("restarts LaunchAgent with bootout-bootstrap-kickstart order", async () => {
+    const env = createDefaultLaunchdEnv();
+    await restartLaunchAgent({
+      env,
+      stdout: new PassThrough(),
+    });
+
+    const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501";
+    const label = "ai.openclaw.gateway";
+    const plistPath = resolveLaunchAgentPlistPath(env);
+    const bootoutIndex = state.launchctlCalls.findIndex(
+      (c) => c[0] === "bootout" && c[1] === `${domain}/${label}`,
+    );
+    const bootstrapIndex = state.launchctlCalls.findIndex(
+      (c) => c[0] === "bootstrap" && c[1] === domain && c[2] === plistPath,
+    );
+    const kickstartIndex = state.launchctlCalls.findIndex(
+      (c) => c[0] === "kickstart" && c[1] === "-k" && c[2] === `${domain}/${label}`,
+    );
+
+    expect(bootoutIndex).toBeGreaterThanOrEqual(0);
+    expect(bootstrapIndex).toBeGreaterThanOrEqual(0);
+    expect(kickstartIndex).toBeGreaterThanOrEqual(0);
+    expect(bootoutIndex).toBeLessThan(bootstrapIndex);
+    expect(bootstrapIndex).toBeLessThan(kickstartIndex);
+  });
+
+  it("waits for previous launchd pid to exit before bootstrapping", async () => {
+    const env = createDefaultLaunchdEnv();
+    state.printOutput = ["state = running", "pid = 4242"].join("\n");
+    const killSpy = vi.spyOn(process, "kill");
+    killSpy
+      .mockImplementationOnce(() => true)
+      .mockImplementationOnce(() => {
+        const err = new Error("no such process") as NodeJS.ErrnoException;
+        err.code = "ESRCH";
+        throw err;
+      });
+
+    vi.useFakeTimers();
+    try {
+      const restartPromise = restartLaunchAgent({
+        env,
+        stdout: new PassThrough(),
+      });
+      await vi.advanceTimersByTimeAsync(250);
+      await restartPromise;
+      expect(killSpy).toHaveBeenCalledWith(4242, 0);
+      const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501";
+      const label = "ai.openclaw.gateway";
+      const bootoutIndex = state.launchctlCalls.findIndex(
+        (c) => c[0] === "bootout" && c[1] === `${domain}/${label}`,
+      );
+      const bootstrapIndex = state.launchctlCalls.findIndex((c) => c[0] === "bootstrap");
+      expect(bootoutIndex).toBeGreaterThanOrEqual(0);
+      expect(bootstrapIndex).toBeGreaterThanOrEqual(0);
+      expect(bootoutIndex).toBeLessThan(bootstrapIndex);
+    } finally {
+      vi.useRealTimers();
+      killSpy.mockRestore();
+    }
+  });
+
   it("shows actionable guidance when launchctl gui domain does not support bootstrap", async () => {
     state.bootstrapError = "Bootstrap failed: 125: Domain does not support specified action";
     const env = createDefaultLaunchdEnv();
diff --git a/src/daemon/launchd.ts b/src/daemon/launchd.ts
@@ -331,6 +331,34 @@ function isUnsupportedGuiDomain(detail: string): boolean {
   );
 }
 
+const RESTART_PID_WAIT_TIMEOUT_MS = 10_000;
+const RESTART_PID_WAIT_INTERVAL_MS = 200;
+
+async function sleepMs(ms: number): Promise<void> {
+  await new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+async function waitForPidExit(pid: number): Promise<void> {
+  if (!Number.isFinite(pid) || pid <= 1) {
+    return;
+  }
+  const deadline = Date.now() + RESTART_PID_WAIT_TIMEOUT_MS;
+  while (Date.now() < deadline) {
+    try {
+      process.kill(pid, 0);
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code === "ESRCH" || code === "EPERM") {
+        return;
+      }
+      return;
+    }
+    await sleepMs(RESTART_PID_WAIT_INTERVAL_MS);
+  }
+}
+
 export async function stopLaunchAgent({ stdout, env }: GatewayServiceControlArgs): Promise<void> {
   const domain = resolveGuiDomain();
   const label = resolveLaunchAgentLabel({ env });
@@ -418,11 +446,45 @@ export async function restartLaunchAgent({
   stdout,
   env,
 }: GatewayServiceControlArgs): Promise<void> {
+  const serviceEnv = env ?? (process.env as GatewayServiceEnv);
   const domain = resolveGuiDomain();
-  const label = resolveLaunchAgentLabel({ env });
-  const res = await execLaunchctl(["kickstart", "-k", `${domain}/${label}`]);
-  if (res.code !== 0) {
-    throw new Error(`launchctl kickstart failed: ${res.stderr || res.stdout}`.trim());
+  const label = resolveLaunchAgentLabel({ env: serviceEnv });
+  const plistPath = resolveLaunchAgentPlistPath(serviceEnv);
+
+  const runtime = await execLaunchctl(["print", `${domain}/${label}`]);
+  const previousPid =
+    runtime.code === 0
+      ? parseLaunchctlPrint(runtime.stdout || runtime.stderr || "").pid
+      : undefined;
+
+  const stop = await execLaunchctl(["bootout", `${domain}/${label}`]);
+  if (stop.code !== 0 && !isLaunchctlNotLoaded(stop)) {
+    throw new Error(`launchctl bootout failed: ${stop.stderr || stop.stdout}`.trim());
+  }
+  if (typeof previousPid === "number") {
+    await waitForPidExit(previousPid);
+  }
+
+  const boot = await execLaunchctl(["bootstrap", domain, plistPath]);
+  if (boot.code !== 0) {
+    const detail = (boot.stderr || boot.stdout).trim();
+    if (isUnsupportedGuiDomain(detail)) {
+      throw new Error(
+        [
+          `launchctl bootstrap failed: ${detail}`,
+          `LaunchAgent restart requires a logged-in macOS GUI session for this user (${domain}).`,
+          "This usually means you are running from SSH/headless context or as the wrong user (including sudo).",
+          "Fix: sign in to the macOS desktop as the target user and rerun `openclaw gateway restart`.",
+          "Headless deployments should use a dedicated logged-in user session or a custom LaunchDaemon (not shipped): https://docs.openclaw.ai/gateway",
+        ].join("\n"),
+      );
+    }
+    throw new Error(`launchctl bootstrap failed: ${detail}`);
+  }
+
+  const start = await execLaunchctl(["kickstart", "-k", `${domain}/${label}`]);
+  if (start.code !== 0) {
+    throw new Error(`launchctl kickstart failed: ${start.stderr || start.stdout}`.trim());
   }
   try {
     stdout.write(`${formatLine("Restarted LaunchAgent", `${domain}/${label}`)}\n`);
diff --git a/src/daemon/service-env.test.ts b/src/daemon/service-env.test.ts
@@ -309,6 +309,26 @@ describe("buildServiceEnvironment", () => {
       expect(env.OPENCLAW_LAUNCHD_LABEL).toBe("ai.openclaw.work");
     }
   });
+
+  it("forwards proxy environment variables for launchd/systemd runtime", () => {
+    const env = buildServiceEnvironment({
+      env: {
+        HOME: "/home/user",
+        HTTP_PROXY: " http://proxy.local:7890 ",
+        HTTPS_PROXY: "https://proxy.local:7890",
+        NO_PROXY: "localhost,127.0.0.1",
+        http_proxy: "http://proxy.local:7890",
+        all_proxy: "socks5://proxy.local:1080",
+      },
+      port: 18789,
+    });
+
+    expect(env.HTTP_PROXY).toBe("http://proxy.local:7890");
+    expect(env.HTTPS_PROXY).toBe("https://proxy.local:7890");
+    expect(env.NO_PROXY).toBe("localhost,127.0.0.1");
+    expect(env.http_proxy).toBe("http://proxy.local:7890");
+    expect(env.all_proxy).toBe("socks5://proxy.local:1080");
+  });
 });
 
 describe("buildNodeServiceEnvironment", () => {
@@ -319,6 +339,19 @@ describe("buildNodeServiceEnvironment", () => {
     expect(env.HOME).toBe("/home/user");
   });
 
+  it("forwards proxy environment variables for node services", () => {
+    const env = buildNodeServiceEnvironment({
+      env: {
+        HOME: "/home/user",
+        HTTPS_PROXY: " https://proxy.local:7890 ",
+        no_proxy: "localhost,127.0.0.1",
+      },
+    });
+
+    expect(env.HTTPS_PROXY).toBe("https://proxy.local:7890");
+    expect(env.no_proxy).toBe("localhost,127.0.0.1");
+  });
+
   it("forwards TMPDIR for node services", () => {
     const env = buildNodeServiceEnvironment({
       env: { HOME: "/home/user", TMPDIR: "/tmp/custom" },
diff --git a/src/daemon/service-env.ts b/src/daemon/service-env.ts
