fix(feishu): recover WebSocket after SDK retry exhaustion

vincentkoc · vincentkoc · commit 5dd7729dead4 · 2026-04-28T18:36:25.000Z
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,6 +36,7 @@ Docs: https://docs.openclaw.ai
 - Channels/Telegram: persist native command metadata on target sessions so topic, helper, and ACP-bound slash commands keep their session metadata attached to the routed conversation. (#57548) Thanks @GaosCode.
 - Channels/native commands: keep validated native slash command replies visible in group chats while preserving explicit owner allowlists for command authorization. (#73672) Thanks @obviyus.
 - Auto-reply/session: carry the tail of user/assistant turns into the freshly-rotated transcript on silent in-reply session resets (compaction failure, role-ordering conflict) so direct-chat continuity survives the rebind. Fixes #70853. (#70898) Thanks @neeravmakwana.
+- Feishu: recreate WebSocket clients with monitor-owned backoff after SDK reconnect exhaustion, preserving heartbeat defaults and shutdown cleanup so persistent connections recover without manual gateway restart. Fixes #52618; duplicate evidence #59753; related #55532, #68766, and #72411. Thanks @schumilin, @alex-xuweilong, @120106835, @sirfengyu, and @tianhaocui.
 
 ## 2026.4.27
 
diff --git a/extensions/feishu/src/client.test.ts b/extensions/feishu/src/client.test.ts
@@ -119,9 +119,13 @@ function readCallOptions(
   return isRecord(call) ? call : {};
 }
 
-function firstWsClientOptions(): { agent?: unknown; wsConfig?: unknown } {
+function firstWsClientOptions(): {
+  agent?: unknown;
+  wsConfig?: unknown;
+  onError?: unknown;
+} {
   const options = readCallOptions(wsClientCtorMock, 0);
-  return { agent: options.agent, wsConfig: options.wsConfig };
+  return { agent: options.agent, wsConfig: options.wsConfig, onError: options.onError };
 }
 
 beforeAll(async () => {
@@ -355,6 +359,19 @@ describe("createFeishuWSClient proxy handling", () => {
     });
   });
 
+  it("passes lifecycle callbacks while preserving heartbeat wsConfig defaults", async () => {
+    const onError = vi.fn();
+
+    await createFeishuWSClient(baseAccount, { onError });
+
+    const options = firstWsClientOptions();
+    expect(options.onError).toBe(onError);
+    expect(options.wsConfig).toEqual({
+      PingInterval: 30,
+      PingTimeout: 3,
+    });
+  });
+
   it("does not set a ws proxy agent when proxy env is absent", async () => {
     await createFeishuWSClient(baseAccount);
 
diff --git a/extensions/feishu/src/client.ts b/extensions/feishu/src/client.ts
@@ -220,11 +220,19 @@ export function createFeishuClient(creds: FeishuClientCredentials): Lark.Client
   return client;
 }
 
+export type FeishuWsClientCallbacks = Pick<
+  ConstructorParameters<typeof feishuClientSdk.WSClient>[0],
+  "onError" | "onReady" | "onReconnected" | "onReconnecting"
+>;
+
 /**
  * Create a Feishu WebSocket client for an account.
  * Note: WSClient is not cached since each call creates a new connection.
  */
-export async function createFeishuWSClient(account: ResolvedFeishuAccount): Promise<Lark.WSClient> {
+export async function createFeishuWSClient(
+  account: ResolvedFeishuAccount,
+  callbacks: FeishuWsClientCallbacks = {},
+): Promise<Lark.WSClient> {
   const { accountId, appId, appSecret, domain } = account;
 
   if (!appId || !appSecret) {
@@ -236,6 +244,7 @@ export async function createFeishuWSClient(account: ResolvedFeishuAccount): Prom
     appId,
     appSecret,
     domain: resolveDomain(domain),
+    ...callbacks,
     loggerLevel: feishuClientSdk.LoggerLevel.info,
     wsConfig: FEISHU_WS_CONFIG,
     ...(agent ? { agent } : {}),
diff --git a/extensions/feishu/src/monitor.cleanup.test.ts b/extensions/feishu/src/monitor.cleanup.test.ts
@@ -136,6 +136,116 @@ describe("feishu websocket cleanup", () => {
     expect(errorMessage).toContain("appSecret=[redacted]");
   });
 
+  it("recreates the websocket client after sdk reconnect exhaustion", async () => {
+    vi.useFakeTimers();
+    const exhaustedClient = createWsClient();
+    const recoveredClient = createWsClient();
+    createFeishuWSClientMock
+      .mockResolvedValueOnce(exhaustedClient)
+      .mockResolvedValueOnce(recoveredClient);
+
+    const abortController = new AbortController();
+    const runtime = {
+      log: vi.fn(),
+      error: vi.fn(),
+      exit: vi.fn(),
+    };
+    const accountId = "exhausted";
+    botOpenIds.set(accountId, "ou_exhausted");
+    botNames.set(accountId, "Exhausted");
+
+    const monitorPromise = monitorWebSocket({
+      account: createAccount(accountId),
+      accountId,
+      runtime,
+      abortSignal: abortController.signal,
+      eventDispatcher: {} as never,
+    });
+
+    await vi.waitFor(() => {
+      expect(exhaustedClient.start).toHaveBeenCalledTimes(1);
+      expect(wsClients.get(accountId)).toBe(exhaustedClient);
+    });
+
+    const callbacks = createFeishuWSClientMock.mock.calls[0]?.[1] as
+      | { onError?: (err: Error) => void }
+      | undefined;
+    callbacks?.onError?.(
+      new Error("WebSocket reconnect exhausted after 3 attempts\nBearer token_abc"),
+    );
+
+    await vi.waitFor(() => {
+      expect(exhaustedClient.close).toHaveBeenCalledTimes(1);
+      expect(wsClients.has(accountId)).toBe(false);
+    });
+    expect(botOpenIds.get(accountId)).toBe("ou_exhausted");
+    expect(botNames.get(accountId)).toBe("Exhausted");
+
+    await vi.advanceTimersByTimeAsync(1_000);
+
+    await vi.waitFor(() => {
+      expect(recoveredClient.start).toHaveBeenCalledTimes(1);
+      expect(wsClients.get(accountId)).toBe(recoveredClient);
+    });
+
+    abortController.abort();
+    await monitorPromise;
+
+    expect(createFeishuWSClientMock).toHaveBeenCalledTimes(2);
+    expect(recoveredClient.close).toHaveBeenCalledTimes(1);
+    expect(botOpenIds.has(accountId)).toBe(false);
+    expect(botNames.has(accountId)).toBe(false);
+    const errorMessage = String(runtime.error.mock.calls[0]?.[0] ?? "");
+    expect(errorMessage).toContain("WebSocket connection ended, recreating client in 1000ms");
+    expect(errorMessage).toContain("Bearer [redacted]");
+    expect(errorMessage).not.toContain("\n");
+    expect(errorMessage).not.toContain("token_abc");
+  });
+
+  it("clears identity without recreating a websocket when aborted during reconnect backoff", async () => {
+    vi.useFakeTimers();
+    const exhaustedClient = createWsClient();
+    createFeishuWSClientMock.mockResolvedValueOnce(exhaustedClient);
+
+    const abortController = new AbortController();
+    const accountId = "abort-backoff";
+    botOpenIds.set(accountId, "ou_abort");
+    botNames.set(accountId, "Abort");
+
+    const monitorPromise = monitorWebSocket({
+      account: createAccount(accountId),
+      accountId,
+      runtime: {
+        log: vi.fn(),
+        error: vi.fn(),
+        exit: vi.fn(),
+      },
+      abortSignal: abortController.signal,
+      eventDispatcher: {} as never,
+    });
+
+    await vi.waitFor(() => {
+      expect(exhaustedClient.start).toHaveBeenCalledTimes(1);
+    });
+
+    const callbacks = createFeishuWSClientMock.mock.calls[0]?.[1] as
+      | { onError?: (err: Error) => void }
+      | undefined;
+    callbacks?.onError?.(new Error("WebSocket reconnect exhausted after 3 attempts"));
+
+    await vi.waitFor(() => {
+      expect(exhaustedClient.close).toHaveBeenCalledTimes(1);
+    });
+
+    abortController.abort();
+    await monitorPromise;
+
+    expect(createFeishuWSClientMock).toHaveBeenCalledTimes(1);
+    expect(wsClients.has(accountId)).toBe(false);
+    expect(botOpenIds.has(accountId)).toBe(false);
+    expect(botNames.has(accountId)).toBe(false);
+  });
+
   it("redacts websocket close errors during abort cleanup", async () => {
     const wsClient = createWsClient();
     wsClient.close.mockImplementationOnce(() => {
diff --git a/extensions/feishu/src/monitor.transport.ts b/extensions/feishu/src/monitor.transport.ts
@@ -124,8 +124,9 @@ function cleanupFeishuWsClient(params: {
   accountId: string;
   wsClient?: Lark.WSClient;
   error: (message: string) => void;
+  clearIdentity: boolean;
 }): void {
-  const { accountId, wsClient, error } = params;
+  const { accountId, wsClient, error, clearIdentity } = params;
   if (wsClient) {
     try {
       wsClient.close();
@@ -136,27 +137,43 @@ function cleanupFeishuWsClient(params: {
     }
   }
   wsClients.delete(accountId);
-  botOpenIds.delete(accountId);
-  botNames.delete(accountId);
+  if (clearIdentity) {
+    botOpenIds.delete(accountId);
+    botNames.delete(accountId);
+  }
 }
 
-function waitForFeishuWsAbort(abortSignal?: AbortSignal): Promise<void> {
-  if (abortSignal?.aborted) {
-    return Promise.resolve();
+function waitForFeishuWsCycleEnd(params: {
+  abortSignal?: AbortSignal;
+  terminalError: Promise<Error>;
+}): Promise<"abort" | Error> {
+  if (params.abortSignal?.aborted) {
+    return Promise.resolve("abort");
   }
+
   return new Promise((resolve) => {
-    if (!abortSignal) {
-      // No external lifecycle owner was provided, so keep the SDK-managed connection alive.
-      return;
-    }
-    const handleAbort = () => {
-      abortSignal.removeEventListener("abort", handleAbort);
-      resolve();
+    let settled = false;
+    let handleAbort: (() => void) | undefined;
+
+    const finish = (result: "abort" | Error) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      if (handleAbort) {
+        params.abortSignal?.removeEventListener("abort", handleAbort);
+      }
+      resolve(result);
     };
-    abortSignal.addEventListener("abort", handleAbort, { once: true });
-    if (abortSignal.aborted) {
-      handleAbort();
+
+    handleAbort = () => finish("abort");
+    params.abortSignal?.addEventListener("abort", handleAbort, { once: true });
+    if (params.abortSignal?.aborted) {
+      finish("abort");
+      return;
     }
+
+    void params.terminalError.then(finish);
   });
 }
 
@@ -178,22 +195,45 @@ export async function monitorWebSocket({
 
     let wsClient: Lark.WSClient | undefined;
     try {
+      let reportTerminalError: (err: Error) => void = () => {};
+      const terminalError = new Promise<Error>((resolve) => {
+        reportTerminalError = resolve;
+      });
       log(`feishu[${accountId}]: starting WebSocket connection...`);
-      wsClient = await createFeishuWSClient(account);
+      wsClient = await createFeishuWSClient(account, {
+        onError: reportTerminalError,
+      });
       if (abortSignal?.aborted) {
-        cleanupFeishuWsClient({ accountId, wsClient, error });
+        cleanupFeishuWsClient({ accountId, wsClient, error, clearIdentity: true });
         break;
       }
       wsClients.set(accountId, wsClient);
       await wsClient.start({ eventDispatcher });
       attempt = 0;
       log(`feishu[${accountId}]: WebSocket client started`);
-      await waitForFeishuWsAbort(abortSignal);
-      log(`feishu[${accountId}]: abort signal received, stopping`);
-      cleanupFeishuWsClient({ accountId, wsClient, error });
-      return;
+      const cycleEnd = await waitForFeishuWsCycleEnd({ abortSignal, terminalError });
+      if (cycleEnd === "abort") {
+        log(`feishu[${accountId}]: abort signal received, stopping`);
+        cleanupFeishuWsClient({ accountId, wsClient, error, clearIdentity: true });
+        return;
+      }
+
+      cleanupFeishuWsClient({ accountId, wsClient, error, clearIdentity: false });
+      if (abortSignal?.aborted) {
+        break;
+      }
+
+      attempt += 1;
+      const delayMs = getFeishuWsReconnectDelayMs(attempt);
+      error(
+        `feishu[${accountId}]: WebSocket connection ended, recreating client in ${delayMs}ms: ${formatFeishuWsErrorForLog(cycleEnd)}`,
+      );
+      const shouldRetry = await waitForAbortableDelay(delayMs, abortSignal);
+      if (!shouldRetry) {
+        break;
+      }
     } catch (err) {
-      cleanupFeishuWsClient({ accountId, wsClient, error });
+      cleanupFeishuWsClient({ accountId, wsClient, error, clearIdentity: false });
       if (abortSignal?.aborted) {
         break;
       }
@@ -209,6 +249,7 @@ export async function monitorWebSocket({
       }
     }
   }
+  cleanupFeishuWsClient({ accountId, wsClient: undefined, error, clearIdentity: true });
 }
 
 export async function monitorWebhook({
