fix(scripts): preserve secret proof timeout grace

vincentkoc · vincentkoc · commit 68790eb4b9a8 · 2026-06-20T10:16:59.000+02:00
diff --git a/scripts/e2e/secret-provider-integrations.mjs b/scripts/e2e/secret-provider-integrations.mjs
@@ -25,6 +25,7 @@ const COMMAND_TIMEOUT_MS = readPositiveInt(
   120000,
   "OPENCLAW_SECRET_PROOF_COMMAND_MS",
 );
+const COMMAND_TIMEOUT_KILL_GRACE_MS = 1000;
 const READY_TIMEOUT_MS = readPositiveInt(
   process.env.OPENCLAW_SECRET_PROOF_READY_MS,
   120000,
@@ -297,20 +298,27 @@ function runCommand(command, args, options = {}) {
     let timedOut = false;
     let aborted = false;
     let killTimer;
+    let forceKillAt;
+    const armForceKill = () => {
+      forceKillAt ??= Date.now() + COMMAND_TIMEOUT_KILL_GRACE_MS;
+      killTimer ??= setTimeout(
+        () => terminateProcessTree(child, "SIGKILL"),
+        COMMAND_TIMEOUT_KILL_GRACE_MS,
+      );
+      killTimer.unref();
+    };
     const abort = () => {
       if (usesProcessGroup ? !processTreeIsAlive(child) : childHasExited(child)) {
         return;
       }
       aborted = true;
       terminateProcessTree(child, "SIGTERM");
-      killTimer ??= setTimeout(() => terminateProcessTree(child, "SIGKILL"), 1000);
-      killTimer.unref();
+      armForceKill();
     };
     const timer = setTimeout(() => {
       timedOut = true;
       terminateProcessTree(child, "SIGTERM");
-      killTimer = setTimeout(() => terminateProcessTree(child, "SIGKILL"), 1000);
-      killTimer.unref();
+      armForceKill();
     }, timeoutMs);
     const abortSignal = options.signal;
     if (abortSignal?.aborted) {
@@ -347,27 +355,42 @@ function runCommand(command, args, options = {}) {
       if (killTimer) {
         clearTimeout(killTimer);
       }
+      forceKillAt = undefined;
       abortSignal?.removeEventListener("abort", abort);
       removeParentSignalHandlers();
       reject(error instanceof Error ? error : new Error(formatErrorMessage(error)));
     });
     child.on("close", (code, signal) => {
       clearTimeout(timer);
-      if (killTimer) {
-        clearTimeout(killTimer);
-      }
       abortSignal?.removeEventListener("abort", abort);
       removeParentSignalHandlers();
       const result = { code, signal, stdout: stdout.text(), stderr: stderr.text() };
+      const finishTerminatedTree = async () => {
+        await finishTimedOutCommandProcessTree(child, {
+          forceKillAt,
+          timeoutKillGraceMs: COMMAND_TIMEOUT_KILL_GRACE_MS,
+        });
+        if (killTimer) {
+          clearTimeout(killTimer);
+        }
+        forceKillAt = undefined;
+      };
       if (aborted) {
-        reject(new Error(scrub(`command aborted: ${command} ${args.join(" ")}`)));
+        void finishTerminatedTree().finally(() =>
+          reject(new Error(scrub(`command aborted: ${command} ${args.join(" ")}`))),
+        );
         return;
       }
       if (timedOut) {
-        terminateProcessTree(child, "SIGKILL");
-        reject(new Error(scrub(`command timed out: ${command} ${args.join(" ")}`)));
+        void finishTerminatedTree().finally(() =>
+          reject(new Error(scrub(`command timed out: ${command} ${args.join(" ")}`))),
+        );
         return;
       }
+      if (killTimer) {
+        clearTimeout(killTimer);
+      }
+      forceKillAt = undefined;
       if (result.signal && options.allowFailure !== true) {
         reject(
           new Error(
@@ -880,6 +903,21 @@ async function stopGateway(child) {
   await waitForProcessTreeExit(child, 1000);
 }
 
+async function finishTimedOutCommandProcessTree(child, { forceKillAt, timeoutKillGraceMs }) {
+  if (!processTreeIsAlive(child)) {
+    return;
+  }
+  const graceRemainingMs =
+    forceKillAt === undefined ? timeoutKillGraceMs : Math.max(0, forceKillAt - Date.now());
+  if (graceRemainingMs > 0) {
+    await waitForProcessTreeExit(child, graceRemainingMs);
+  }
+  if (processTreeIsAlive(child)) {
+    terminateProcessTree(child, "SIGKILL");
+  }
+  await waitForProcessTreeExit(child, timeoutKillGraceMs);
+}
+
 function childHasExited(child) {
   return child.exitCode !== null || child.signalCode !== null;
 }
diff --git a/test/scripts/secret-provider-integrations.test.ts b/test/scripts/secret-provider-integrations.test.ts
@@ -514,6 +514,62 @@ describe("secret provider integration proof harness", () => {
     expect(sizeAfterWait).toBe(sizeAfterReturn);
   });
 
+  it.runIf(process.platform !== "win32")(
+    "preserves timeout kill grace for descendants after the leader exits",
+    async () => {
+      const root = makeTempDir();
+      const cleanupPath = path.join(root, "command-descendant-cleanup.txt");
+      const descendantPidPath = path.join(root, "command-descendant.pid");
+      const scriptPath = path.join(root, "spawn-cleaning-descendant.mjs");
+      const descendantScript = [
+        "import fs from 'node:fs';",
+        `fs.writeFileSync(${JSON.stringify(descendantPidPath)}, String(process.pid));`,
+        "process.on('SIGTERM', () => {",
+        `  setTimeout(() => { fs.writeFileSync(${JSON.stringify(
+          cleanupPath,
+        )}, "clean"); process.exit(0); }, 75);`,
+        "});",
+        "setInterval(() => {}, 1000);",
+      ].join("\n");
+      fs.writeFileSync(
+        scriptPath,
+        [
+          "import childProcess from 'node:child_process';",
+          "import { setTimeout as delay } from 'node:timers/promises';",
+          `childProcess.spawn(process.execPath, ["--input-type=module", "--eval", ${JSON.stringify(
+            descendantScript,
+          )}], { stdio: "ignore" });`,
+          "process.on('SIGTERM', () => process.exit(0));",
+          "await delay(60_000);",
+          "",
+        ].join("\n"),
+      );
+      const proof = await import(
+        `${pathToFileURL(proofScriptPath).href}?case=timeout-grace-${Date.now()}`
+      );
+      let descendantPid = 0;
+
+      try {
+        const command = proof.runCommand(process.execPath, [scriptPath], {
+          timeoutMs: 150,
+        });
+
+        await waitFor(() => fs.existsSync(descendantPidPath));
+        descendantPid = Number.parseInt(fs.readFileSync(descendantPidPath, "utf8"), 10);
+        expect(Number.isInteger(descendantPid)).toBe(true);
+        expect(isProcessAlive(descendantPid)).toBe(true);
+
+        await expect(command).rejects.toThrow(/command timed out/u);
+        expect(fs.readFileSync(cleanupPath, "utf8")).toBe("clean");
+        expect(isProcessAlive(descendantPid)).toBe(false);
+      } finally {
+        if (descendantPid && isProcessAlive(descendantPid)) {
+          process.kill(descendantPid, "SIGKILL");
+        }
+      }
+    },
+  );
+
   it.runIf(process.platform !== "win32")(
     "aborts command process groups after the leader exits before stdio closes",
     async () => {
