fix(discord): add circuit breaker for WebSocket resume loop (#13180)

funmerlin · funmerlin · commit 3cc597464785 · 2026-02-13T22:11:24.000+01:00
When the Discord WebSocket connection enters a stall loop (connects but never receives HELLO), the existing zombie timeout handler would disconnect and reconnect indefinitely. The library's reconnectAttempts counter resets on every WebSocket open event, so the maxAttempts circuit breaker is never reached. This adds an application-level circuit breaker: - Track consecutive stalls (WS opens but no HELLO within 30s) - After 5 consecutive stalls, invalidate the session state (sessionId, resumeGatewayUrl) and force a fresh IDENTIFY instead of resume - Log stall count on each attempt for observability - Reset counter on successful HELLO receipt This breaks the infinite resume loop observed in production where 1400+ reconnect attempts occurred over 12+ hours with a stale session token. Fixes: #13180
diff --git a/src/discord/monitor/provider.ts b/src/discord/monitor/provider.ts
@@ -634,10 +634,24 @@ export async function monitorDiscordProvider(opts: MonitorDiscordOpts = {}) {
     abortSignal?.addEventListener("abort", onAbort, { once: true });
   }
   // Timeout to detect zombie connections where HELLO is never received.
+  // Includes a circuit breaker: after MAX_STALL_RETRIES consecutive stalls without
+  // a successful HELLO, invalidate the session and do a full re-identify instead of
+  // resuming with a stale session token forever. See openclaw/openclaw#13180.
   const HELLO_TIMEOUT_MS = 30000;
+  const MAX_STALL_RETRIES = 5;
   let helloTimeoutId: ReturnType<typeof setTimeout> | undefined;
+  let consecutiveStalls = 0;
   const onGatewayDebug = (msg: unknown) => {
     const message = String(msg);
+    // Reset stall counter on successful HELLO (dispatched as "Received HELLO" or opcode 10).
+    if (message.includes("heartbeat_interval") || message.includes("Received HELLO")) {
+      consecutiveStalls = 0;
+      if (helloTimeoutId) {
+        clearTimeout(helloTimeoutId);
+        helloTimeoutId = undefined;
+      }
+      return;
+    }
     if (!message.includes("WebSocket connection opened")) {
       return;
     }
@@ -646,13 +660,41 @@ export async function monitorDiscordProvider(opts: MonitorDiscordOpts = {}) {
     }
     helloTimeoutId = setTimeout(() => {
       if (!gateway?.isConnected) {
-        runtime.log?.(
-          danger(
-            `connection stalled: no HELLO received within ${HELLO_TIMEOUT_MS}ms, forcing reconnect`,
-          ),
-        );
-        gateway?.disconnect();
-        gateway?.connect(false);
+        consecutiveStalls++;
+        if (consecutiveStalls >= MAX_STALL_RETRIES) {
+          runtime.log?.(
+            danger(
+              `connection stalled ${consecutiveStalls} consecutive times — circuit breaker tripped, forcing fresh identify (see #13180)`,
+            ),
+          );
+          // Invalidate the stale session so the library does a fresh IDENTIFY
+          // instead of trying to resume with a dead session token.
+          if (gateway) {
+            (
+              gateway as unknown as {
+                state: {
+                  sessionId: string | null;
+                  resumeGatewayUrl: string | null;
+                  sequence: number | null;
+                };
+              }
+            ).state.sessionId = null;
+            (
+              gateway as unknown as { state: { resumeGatewayUrl: string | null } }
+            ).state.resumeGatewayUrl = null;
+          }
+          consecutiveStalls = 0;
+          gateway?.disconnect();
+          gateway?.connect(false);
+        } else {
+          runtime.log?.(
+            danger(
+              `connection stalled: no HELLO received within ${HELLO_TIMEOUT_MS}ms, forcing reconnect (${consecutiveStalls}/${MAX_STALL_RETRIES} before circuit breaker)`,
+            ),
+          );
+          gateway?.disconnect();
+          gateway?.connect(false);
+        }
       }
       helloTimeoutId = undefined;
     }, HELLO_TIMEOUT_MS);
