fix(codex): backoff startup connection-close retries and surface exhaustion (#83959)

ml12580 · claude · ml12580 · commit 40c0d9f03815 · 2026-07-09T23:45:50.000+08:00
The Codex app-server startup retry loop re-acquired the shared client
immediately after a connection-close, before the replacement app-server
process had time to bind, and exhausted a 3-attempt budget in the same
failing window. Scheduled/background turns then failed even though the
route became viable moments later.

Add a bounded, abortable backoff between startup retries, raise the
budget from 3 to 5, and throw a distinct CodexAppServerStartupExhausted
Error (original error preserved as cause) when the window is exhausted,
so callers can distinguish startup lifecycle exhaustion from a mid-turn
client close.

Co-Authored-By: Claude Opus 4.8 &lt;noreply@anthropic.com&gt;
diff --git a/extensions/codex/src/app-server/attempt-startup.test.ts b/extensions/codex/src/app-server/attempt-startup.test.ts
@@ -11,6 +11,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { startCodexAttemptThread } from "./attempt-startup.js";
 import { CodexAppServerClient } from "./client.js";
 import { type CodexPluginConfig, resolveCodexAppServerRuntimeOptions } from "./config.js";
+import { threadStartResult } from "./run-attempt-test-harness.js";
 import { testCodexAppServerBindingStore } from "./session-binding.test-helpers.js";
 import {
   clearSharedCodexAppServerClient,
@@ -391,4 +392,88 @@ describe("startCodexAttemptThread", () => {
     expect((error as Error).message).toBe("plugin/list timed out");
     expect(harness.process.stdin.destroyed).toBe(true);
   });
+
+  it("retries startup across transient connection-close failures with a backoff (#83959)", async () => {
+    // The app-server connection closes during startup on the first few attempts
+    // while the replacement process is still warming up. Startup must survive
+    // those transient closes (with a bounded backoff between attempts) instead
+    // of exhausting the retry budget before the server is ready.
+    const closeFailuresBeforeSuccess = 4;
+    let factoryInvocations = 0;
+    const backoffDelays: number[] = [];
+    const backoffModule = await import("openclaw/plugin-sdk/runtime-env");
+    const sleepSpy = vi
+      .spyOn(backoffModule, "sleepWithAbort")
+      .mockImplementation(async (ms: number) => {
+        backoffDelays.push(ms);
+      });
+
+    const { harness, run } = startThreadWithHarness(30_000, new AbortController().signal, {
+      attemptClientFactory:
+        () =>
+        async (...args) => {
+          factoryInvocations += 1;
+          if (factoryInvocations <= closeFailuresBeforeSuccess) {
+            throw new Error("codex app-server client is closed");
+          }
+          // On the recovering attempt, delegate to the default leased factory so
+          // the shared-client + initialize handshake path matches a real startup.
+          return getLeasedSharedCodexAppServerClient(...args);
+        },
+    });
+
+    const settled = run.then(
+      (result) => result,
+      (error: unknown) => error,
+    );
+    // The successful attempt still needs to answer the initialize handshake.
+    await answerInitialize(harness);
+    const threadStart = await waitForThreadStart(harness);
+    harness.send({ id: threadStart.id, result: threadStartResult("recovered-thread") });
+
+    const result = await settled;
+    expect(result).not.toBeInstanceOf(Error);
+    expect(factoryInvocations).toBe(closeFailuresBeforeSuccess + 1);
+    // A backoff was awaited before every retry (one less than the number of
+    // failed attempts, since the last failure is followed by the successful attempt).
+    expect(backoffDelays.length).toBe(closeFailuresBeforeSuccess);
+    // Backoff is bounded and non-negative.
+    for (const delay of backoffDelays) {
+      expect(delay).toBeGreaterThanOrEqual(0);
+      expect(delay).toBeLessThanOrEqual(4_000);
+    }
+    sleepSpy.mockRestore();
+  });
+
+  it("surfaces a distinct startup-exhausted error when connection-close persists (#83959)", async () => {
+    // When the app-server connection keeps closing through the entire bounded
+    // retry window, startup must fail with a distinguishable exhaustion error
+    // (not the raw "client is closed" message) so callers can tell a startup
+    // lifecycle failure apart from a mid-turn client close.
+    const backoffModule = await import("openclaw/plugin-sdk/runtime-env");
+    const sleepSpy = vi.spyOn(backoffModule, "sleepWithAbort").mockImplementation(async () => {});
+
+    let factoryInvocations = 0;
+    const { run } = startThreadWithHarness(30_000, new AbortController().signal, {
+      attemptClientFactory: () => async () => {
+        factoryInvocations += 1;
+        throw new Error("codex app-server client is closed");
+      },
+    });
+
+    const error = await run.then(
+      () => undefined,
+      (err: unknown) => err as Error,
+    );
+    expect(error).toBeInstanceOf(Error);
+    // Distinct exhaustion marker, not the raw connection-closed message.
+    expect(error?.message).not.toBe("codex app-server client is closed");
+    expect(error?.message.toLowerCase()).toContain("startup");
+    expect(error?.message.toLowerCase()).toContain("exhaust");
+    // The original connection-closed cause is preserved for diagnostics.
+    expect((error as { cause?: unknown })?.cause).toBeInstanceOf(Error);
+    // Bounded: the loop did not retry forever.
+    expect(factoryInvocations).toBeLessThanOrEqual(8);
+    sleepSpy.mockRestore();
+  });
 });
diff --git a/extensions/codex/src/app-server/attempt-startup.ts b/extensions/codex/src/app-server/attempt-startup.ts
@@ -10,6 +10,11 @@ import {
   type EmbeddedRunAttemptParams,
   type resolveSandboxContext,
 } from "openclaw/plugin-sdk/agent-harness-runtime";
+import {
+  computeBackoff,
+  sleepWithAbort,
+  type BackoffPolicy,
+} from "openclaw/plugin-sdk/runtime-env";
 import { defaultCodexAppInventoryCache } from "./app-inventory-cache.js";
 import {
   CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS,
@@ -75,7 +80,40 @@ import {
 } from "./turn-router.js";
 import type { CodexNativeWebSearchSupport } from "./web-search.js";
 
-const CODEX_APP_SERVER_STARTUP_CONNECTION_CLOSE_MAX_ATTEMPTS = 3;
+const CODEX_APP_SERVER_STARTUP_CONNECTION_CLOSE_MAX_ATTEMPTS = 5;
+
+/**
+ * Bounded backoff between startup connection-close retries. The replacement
+ * app-server process needs a short warm-up window to bind and become ready
+ * before the next startup attempt re-acquires the shared client; retrying
+ * immediately re-enters the same failing window and exhausts the budget
+ * before the server is viable. See issue #83959.
+ */
+const CODEX_APP_SERVER_STARTUP_RETRY_BACKOFF: BackoffPolicy = {
+  initialMs: 500,
+  maxMs: 4_000,
+  factor: 2,
+  jitter: 0.2,
+};
+
+/**
+ * Terminal error raised when the Codex app-server connection keeps closing
+ * throughout the bounded startup retry window. Carries the original
+ * connection-closed error as `cause` so callers can distinguish a startup
+ * lifecycle exhaustion from a mid-turn client close. See issue #83959.
+ */
+export class CodexAppServerStartupExhaustedError extends Error {
+  constructor(
+    public readonly attempts: number,
+    cause: unknown,
+  ) {
+    super(
+      `codex app-server startup exhausted after ${attempts} connection-close retries`,
+      cause !== undefined ? { cause } : undefined,
+    );
+    this.name = "CodexAppServerStartupExhaustedError";
+  }
+}
 
 type CodexSandboxContext = Awaited<ReturnType<typeof resolveSandboxContext>>;
 
@@ -509,18 +547,25 @@ export async function startCodexAttemptThread(params: {
                   error: formatErrorMessage(error),
                 },
               );
-              throw error;
+              throw new CodexAppServerStartupExhaustedError(attempt, error);
             }
+            const backoffMs = computeBackoff(CODEX_APP_SERVER_STARTUP_RETRY_BACKOFF, attempt);
             embeddedAgentLog.warn(
               "codex app-server connection closed during startup; restarting app-server and retrying",
               {
                 attempt,
                 nextAttempt: attempt + 1,
                 maxAttempts: CODEX_APP_SERVER_STARTUP_CONNECTION_CLOSE_MAX_ATTEMPTS,
+                backoffMs,
                 clearedSharedClient,
                 error: formatErrorMessage(error),
               },
             );
+            // Give the replacement app-server process a bounded warm-up window
+            // before re-acquiring the shared client, so the next attempt does
+            // not re-enter the same failing window. Abortable so a cancelled
+            // run does not stall on the backoff. See issue #83959.
+            await sleepWithAbort(backoffMs, params.signal);
           }
         }
         throw new Error("codex app-server startup retry loop exited unexpectedly");
