fix(crabbox): retry cold metadata probes so a slow run --help does not block validation (#102159)

steipete · web-flow · commit 83ebbcb3acf7 · 2026-07-08T12:53:48.000+01:00
The wrapper probes `crabbox --version` and `crabbox run --help` once each with a
snappy 5s timeout to enumerate providers before delegating. A cold Crabbox — the
first call right after a version bump, or one on a loaded machine — can exceed
that timeout while rendering `run --help` (52KB, emitted on stderr in 0.36) or
doing first-run init, and can emit nothing on that first call. When that happens
both guards fire and hard-exit the wrapper ("selected binary failed basic
--version/--help sanity checks" or "could not parse provider list from --help;
refusing to run"), which blocks ALL remote validation (`pnpm check:changed`,
remote `pnpm test` lanes) even though the binary is fine.

Retry each metadata probe once with a generous 20s timeout when the first attempt
is killed or returns empty. The warm path is unchanged (one ~instant probe); only
a slow/empty first probe pays for the retry, after which the sanity/provider-list
guards judge the recovered result. Add a regression test: a fake Crabbox whose
`run --help` is slower than the default probe timeout (and, like 0.36, writes help
to stderr) is now recovered by the retry instead of hard-failing.
diff --git a/scripts/crabbox-wrapper.mjs b/scripts/crabbox-wrapper.mjs
@@ -28,6 +28,11 @@ import { resolvePathEnvKey, resolveWindowsCmdExePath } from "./windows-cmd-helpe
 
 const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const CRABBOX_METADATA_PROBE_TIMEOUT_MS = 5_000;
+// A cold Crabbox (first call after an upgrade, or one on a loaded machine) can
+// exceed the snappy default probe timeout while it renders `run --help` or does
+// first-run init. Retry the metadata probes once with this generous timeout so a
+// single slow probe does not hard-fail the wrapper and block all remote validation.
+const CRABBOX_METADATA_PROBE_RETRY_TIMEOUT_MS = 20_000;
 const ignoreRepoBinary = process.env.OPENCLAW_CRABBOX_WRAPPER_IGNORE_REPO_BINARY === "1";
 const repoLocal = ignoreRepoBinary ? null : resolveCrabboxBinary(process.env, process.platform);
 const pathLocal = resolvePathBinary("crabbox", process.env, process.platform);
@@ -360,14 +365,14 @@ function buildBatchCommandLine(command, commandArgs) {
   return `"${[escapedCommand, ...escapedArgs].join(" ")}"`;
 }
 
-function checkedOutput(command, commandArgs) {
+function checkedOutput(command, commandArgs, timeoutMs = resolveMetadataProbeTimeoutMs(process.env)) {
   const invocation = spawnInvocation(command, commandArgs, process.env, process.platform);
   const result = spawnSync(invocation.command, invocation.args, {
     cwd: repoRoot,
     encoding: "utf8",
     stdio: ["ignore", "pipe", "pipe"],
     windowsVerbatimArguments: invocation.windowsVerbatimArguments,
-    timeout: resolveMetadataProbeTimeoutMs(process.env),
+    timeout: timeoutMs,
     killSignal: "SIGKILL",
   });
   const timedOut = result.error?.name === "Error" && result.signal === "SIGKILL";
@@ -378,6 +383,19 @@ function checkedOutput(command, commandArgs) {
   };
 }
 
+// Probe Crabbox metadata (`--version` / `run --help`) with one generous retry.
+// A cold Crabbox can be SIGKILLed by the snappy default timeout or emit nothing
+// on the first call, then be instant and clean on the next. Retrying keeps the
+// warm path fast (one ~instant probe) while stopping a single slow probe from
+// tripping the sanity/provider-list guards and blocking all remote validation.
+function probeCrabboxMetadata(command, commandArgs) {
+  const first = checkedOutput(command, commandArgs);
+  if (first.status === 0 && first.text.length > 0) {
+    return first;
+  }
+  return checkedOutput(command, commandArgs, CRABBOX_METADATA_PROBE_RETRY_TIMEOUT_MS);
+}
+
 function parseCrabboxVersion(value) {
   const match = `${value}`.match(/\bv?(\d+)\.(\d+)\.(\d+)(?:-([^\s+]+))?(?:\+[^\s]+)?\b/u);
   if (!match) {
@@ -3155,8 +3173,8 @@ function injectFullCheckoutLeaseReclaim(commandArgs) {
   return normalizedArgs;
 }
 
-const version = checkedOutput(binary, ["--version"]);
-const help = checkedOutput(binary, ["run", "--help"]);
+const version = probeCrabboxMetadata(binary, ["--version"]);
+const help = probeCrabboxMetadata(binary, ["run", "--help"]);
 const providers = parseProvidersFromHelp(help.text);
 const displayBinary = binary === "crabbox" ? "crabbox" : relative(repoRoot, binary);
 const provider = selectedProvider(args, providers);
diff --git a/test/scripts/crabbox-wrapper.test.ts b/test/scripts/crabbox-wrapper.test.ts
@@ -227,6 +227,32 @@ function makeSlowVersionCrabbox(helpText: string): string {
   return binDir;
 }
 
+// Fake Crabbox whose `run --help` is slow on every call and, like real Crabbox
+// 0.36, renders the provider help to stderr. Used to prove the wrapper retries a
+// cold/slow metadata probe instead of hard-failing.
+function makeSlowHelpCrabbox(helpText: string, delayMs: number): string {
+  const binDir = mkdtempSync(path.join(tmpdir(), "openclaw-slow-help-crabbox-"));
+  tempDirs.push(binDir);
+  const crabboxPath = path.join(binDir, "crabbox");
+
+  const script = [
+    "#!/usr/bin/env node",
+    "const args = process.argv.slice(2);",
+    "if (args[0] === '--version') {",
+    "  console.log(process.env.OPENCLAW_FAKE_CRABBOX_VERSION || 'crabbox 0.22.1');",
+    "  process.exit(0);",
+    "} else if (args[0] === 'run' && args[1] === '--help') {",
+    `  setTimeout(() => { process.stderr.write(${JSON.stringify(helpText)}); process.exit(0); }, ${delayMs});`,
+    "} else {",
+    "  process.exit(0);",
+    "}",
+  ].join("\n");
+  writeFileSync(crabboxPath, `${script}\n`, "utf8");
+  writeFileSync(`${crabboxPath}.cmd`, windowsNodeCmdShim("crabbox"), "utf8");
+  chmodSync(crabboxPath, 0o755);
+  return binDir;
+}
+
 function testTimingPreload(options: { clockScale?: number; spawnTimeoutMs?: number }): string {
   const key = JSON.stringify(options);
   let preloadPath = timingPreloads.get(key);
@@ -3128,6 +3154,24 @@ describe("scripts/crabbox-wrapper", () => {
     expect(result.stderr).toContain("selected binary failed basic --version/--help sanity checks");
   });
 
+  it("retries a cold Crabbox whose run --help is slower than the default probe timeout", () => {
+    const helpText = "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n";
+    // First probe is SIGKILLed at 150ms; the retry gets the full generous timeout
+    // and reads the (600ms) stderr help, so the wrapper must not hard-fail.
+    const result = runWrapper(helpText, ["--version"], {
+      env: { OPENCLAW_TEST_CRABBOX_METADATA_PROBE_TIMEOUT_MS: "150" },
+      extraPathEntries: [makeSlowHelpCrabbox(helpText, 600)],
+    });
+
+    expect(result.error).toBeUndefined();
+    expect(result.status).toBe(0);
+    expect(result.stderr).not.toContain("could not parse provider list");
+    expect(result.stderr).not.toContain("selected binary failed basic --version/--help sanity checks");
+    expect(result.stderr).toContain(
+      "providers=hetzner,aws,local-container,blacksmith-testbox,cloudflare",
+    );
+  });
+
   it("parses provider choices from the --provider flag help format", () => {
     const helpText =
       "Usage: crabbox run [options]\n  --provider hetzner|aws|local-container|blacksmith-testbox|cloudflare\n";
