fix(gateway,mac): stop stale protocol mismatch reconnect loop, auto-remediate updater jobs, rate-limit server side

loeclos · loeclos · commit d8022cbd42d7 · 2026-05-16T20:16:20.000-07:00
- Add PROTOCOL_MISMATCH to ConnectErrorDetailCodes so clients recognize
  the error type and pause reconnect instead of looping forever
- Server now sends code: PROTOCOL_MISMATCH in protocol mismatch error
  response; GatewayClient includes it in shouldPauseReconnect list
- doctor --fix now auto-removes stale ai.openclaw.update.* launchd jobs
- Server-side per-IP rate limiter suppresses logs/error frames after 5
  protocol mismatches per 60s from the same address
diff --git a/src/commands/doctor-platform-notes.ts b/src/commands/doctor-platform-notes.ts
@@ -6,7 +6,10 @@ import { promisify } from "node:util";
 import { formatCliCommand } from "../cli/command-format.js";
 import type { OpenClawConfig } from "../config/types.openclaw.js";
 import { hasConfiguredSecretInput } from "../config/types.secrets.js";
-import { findStaleOpenClawUpdateLaunchdJobs } from "../daemon/launchd.js";
+import {
+  findStaleOpenClawUpdateLaunchdJobs,
+  removeOpenClawUpdateLaunchdJob,
+} from "../daemon/launchd.js";
 import { normalizeOptionalString } from "../shared/string-coerce.js";
 import { note } from "../terminal/note.js";
 import { shortenHomePath } from "../utils.js";
@@ -66,6 +69,54 @@ export async function noteMacStaleOpenClawUpdateLaunchdJobs(deps?: {
   (deps?.noteFn ?? note)(lines.join("\n"), "Gateway (macOS)");
 }
 
+export async function maybeRemoveStaleOpenClawUpdateLaunchdJobs(deps?: {
+  platform?: NodeJS.Platform;
+  shouldRemove?: boolean;
+  findJobs?: typeof findStaleOpenClawUpdateLaunchdJobs;
+  removeJob?: typeof removeOpenClawUpdateLaunchdJob;
+  noteFn?: typeof note;
+}): Promise<void> {
+  const platform = deps?.platform ?? process.platform;
+  if (platform !== "darwin" || !deps?.shouldRemove) {
+    return;
+  }
+  const jobs = await (deps?.findJobs ?? findStaleOpenClawUpdateLaunchdJobs)().catch(() => []);
+  if (jobs.length === 0) {
+    return;
+  }
+  const removeJob = deps?.removeJob ?? removeOpenClawUpdateLaunchdJob;
+  const noteFn = deps?.noteFn ?? note;
+  const removed: string[] = [];
+  const failed: string[] = [];
+  for (const job of jobs) {
+    const ok = await removeJob(job.label).catch(() => false);
+    if (ok) {
+      removed.push(job.label);
+    } else {
+      failed.push(job.label);
+    }
+  }
+  const lines: string[] = [];
+  if (removed.length > 0) {
+    lines.push(
+      `- Removed ${removed.length} stale OpenClaw updater launchd job(s):`,
+      ...removed.map((l) => `  - ${l}`),
+    );
+  }
+  if (failed.length > 0) {
+    lines.push(
+      `- Failed to remove ${failed.length} stale OpenClaw updater launchd job(s). Remove manually:`,
+      ...failed.map((l) => `  launchctl remove ${l}`),
+    );
+  }
+  if (removed.length > 0) {
+    lines.push(`  ${formatCliCommand("openclaw gateway restart")}`);
+  }
+  if (lines.length > 0) {
+    noteFn(lines.join("\n"), "Gateway (macOS)");
+  }
+}
+
 async function launchctlGetenv(name: string): Promise<string | undefined> {
   try {
     const result = await execFileAsync("/bin/launchctl", ["getenv", name], { encoding: "utf8" });
diff --git a/src/commands/doctor.fast-path-mocks.ts b/src/commands/doctor.fast-path-mocks.ts
@@ -37,6 +37,7 @@ vi.mock("./doctor-platform-notes.js", () => ({
   noteStartupOptimizationHints: vi.fn(),
   noteMacLaunchAgentOverrides: vi.fn().mockResolvedValue(undefined),
   noteMacStaleOpenClawUpdateLaunchdJobs: vi.fn().mockResolvedValue(undefined),
+  maybeRemoveStaleOpenClawUpdateLaunchdJobs: vi.fn().mockResolvedValue(undefined),
   noteMacLaunchctlGatewayEnvOverrides: vi.fn().mockResolvedValue(undefined),
 }));
 
diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts
@@ -373,6 +373,7 @@ async function runGatewayServicesHealth(ctx: DoctorHealthFlowContext): Promise<v
   const { maybeRepairGatewayServiceConfig, maybeScanExtraGatewayServices } =
     await import("../commands/doctor-gateway-services.js");
   const {
+    maybeRemoveStaleOpenClawUpdateLaunchdJobs,
     noteMacLaunchAgentOverrides,
     noteMacLaunchctlGatewayEnvOverrides,
     noteMacStaleOpenClawUpdateLaunchdJobs,
@@ -386,6 +387,9 @@ async function runGatewayServicesHealth(ctx: DoctorHealthFlowContext): Promise<v
   );
   await noteMacLaunchAgentOverrides();
   await noteMacStaleOpenClawUpdateLaunchdJobs();
+  await maybeRemoveStaleOpenClawUpdateLaunchdJobs({
+    shouldRemove: ctx.prompter.shouldRepair,
+  });
   await noteMacLaunchctlGatewayEnvOverrides(ctx.cfg);
 }
 
diff --git a/src/gateway/client.ts b/src/gateway/client.ts
@@ -758,7 +758,8 @@ export class GatewayClient {
       detailCode === ConnectErrorDetailCodes.AUTH_SCOPE_MISMATCH ||
       detailCode === ConnectErrorDetailCodes.PAIRING_REQUIRED ||
       detailCode === ConnectErrorDetailCodes.CONTROL_UI_DEVICE_IDENTITY_REQUIRED ||
-      detailCode === ConnectErrorDetailCodes.DEVICE_IDENTITY_REQUIRED
+      detailCode === ConnectErrorDetailCodes.DEVICE_IDENTITY_REQUIRED ||
+      detailCode === ConnectErrorDetailCodes.PROTOCOL_MISMATCH
     ) {
       return true;
     }
diff --git a/src/gateway/protocol/connect-error-details.ts b/src/gateway/protocol/connect-error-details.ts
@@ -28,6 +28,7 @@ export const ConnectErrorDetailCodes = {
   DEVICE_AUTH_SIGNATURE_INVALID: "DEVICE_AUTH_SIGNATURE_INVALID",
   DEVICE_AUTH_PUBLIC_KEY_INVALID: "DEVICE_AUTH_PUBLIC_KEY_INVALID",
   PAIRING_REQUIRED: "PAIRING_REQUIRED",
+  PROTOCOL_MISMATCH: "PROTOCOL_MISMATCH",
 } as const;
 
 export type ConnectErrorDetailCode =
diff --git a/src/gateway/server/ws-connection/message-handler.ts b/src/gateway/server/ws-connection/message-handler.ts
@@ -146,6 +146,58 @@ type SubsystemLogger = ReturnType<typeof createSubsystemLogger>;
 
 const DEVICE_SIGNATURE_SKEW_MS = 2 * 60 * 1000;
 
+const PROTOCOL_MISMATCH_RATE_LIMIT = 5;
+const PROTOCOL_MISMATCH_RATE_WINDOW_MS = 60_000;
+const PROTOCOL_MISMATCH_PRUNE_INTERVAL_MS = 5 * 60_000;
+
+type ProtocolMismatchEntry = {
+  count: number;
+  windowStart: number;
+};
+
+const protocolMismatchByAddr = new Map<string, ProtocolMismatchEntry>();
+
+let protocolMismatchPruneTimer: ReturnType<typeof setTimeout> | null = null;
+
+function scheduleProtocolMismatchPrune(): void {
+  if (protocolMismatchPruneTimer) {
+    return;
+  }
+  protocolMismatchPruneTimer = setTimeout(() => {
+    protocolMismatchPruneTimer = null;
+    const now = Date.now();
+    for (const [addr, entry] of protocolMismatchByAddr) {
+      if (now - entry.windowStart > PROTOCOL_MISMATCH_RATE_WINDOW_MS * 2) {
+        protocolMismatchByAddr.delete(addr);
+      }
+    }
+    if (protocolMismatchByAddr.size > 0) {
+      scheduleProtocolMismatchPrune();
+    }
+  }, PROTOCOL_MISMATCH_PRUNE_INTERVAL_MS).unref();
+}
+
+function isProtocolMismatchRateLimited(remoteAddr: string | undefined): boolean {
+  if (!remoteAddr) {
+    return false;
+  }
+  const now = Date.now();
+  let entry = protocolMismatchByAddr.get(remoteAddr);
+  if (!entry) {
+    entry = { count: 0, windowStart: now };
+    protocolMismatchByAddr.set(remoteAddr, entry);
+  }
+  if (now - entry.windowStart > PROTOCOL_MISMATCH_RATE_WINDOW_MS) {
+    entry.count = 0;
+    entry.windowStart = now;
+  }
+  entry.count++;
+  if (protocolMismatchByAddr.size === 1) {
+    scheduleProtocolMismatchPrune();
+  }
+  return entry.count > PROTOCOL_MISMATCH_RATE_LIMIT;
+}
+
 export type WsOriginCheckMetrics = {
   hostHeaderFallbackAccepted: number;
 };
@@ -547,11 +599,16 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
             expectedProtocol: PROTOCOL_VERSION,
             minimumProbeProtocol: MIN_PROBE_PROTOCOL_VERSION,
           });
+          if (isProtocolMismatchRateLimited(remoteAddr)) {
+            close(1002, "protocol mismatch");
+            return;
+          }
           logWsControl.warn(
             `protocol mismatch conn=${connId} remote=${remoteAddr ?? "?"} client=${clientLabel} ${connectParams.client.mode} v${connectParams.client.version}`,
           );
           sendHandshakeErrorResponse(ErrorCodes.INVALID_REQUEST, "protocol mismatch", {
             details: {
+              code: ConnectErrorDetailCodes.PROTOCOL_MISMATCH,
               expectedProtocol: PROTOCOL_VERSION,
               minimumProbeProtocol: MIN_PROBE_PROTOCOL_VERSION,
             },
