fix(crabbox): preserve telegram proof kill grace

vincentkoc · vincentkoc · commit a1d278b174f0 · 2026-06-20T12:25:03.000+02:00
diff --git a/scripts/e2e/telegram-user-crabbox-proof.ts b/scripts/e2e/telegram-user-crabbox-proof.ts
@@ -148,6 +148,7 @@ export const COMMAND_STDERR_TAIL_CHARS = 256 * 1024;
 export const COMMAND_FAILURE_STDOUT_TAIL_CHARS = 64 * 1024;
 export const COMMAND_TIMEOUT_MS = 30 * 60 * 1000;
 export const COMMAND_TIMEOUT_KILL_GRACE_MS = 5_000;
+const COMMAND_PROCESS_TREE_EXIT_POLL_MS = 25;
 export const REMOTE_SETUP_COMMAND_TIMEOUT_MS = 90 * 60 * 1000;
 const REMOTE_ROOT = "/tmp/openclaw-telegram-user-crabbox";
 const CREDENTIAL_SCRIPT = fileURLToPath(new URL("./telegram-user-credential.ts", import.meta.url));
@@ -608,6 +609,44 @@ function commandProcessTreeAlive(child: ChildProcess) {
   }
 }
 
+async function waitForCommandProcessTreeExit(child: ChildProcess, timeoutMs: number) {
+  const deadlineAt = Date.now() + timeoutMs;
+  while (Date.now() < deadlineAt) {
+    if (!commandProcessTreeAlive(child)) {
+      return true;
+    }
+    await new Promise((resolvePoll) => {
+      setTimeout(resolvePoll, COMMAND_PROCESS_TREE_EXIT_POLL_MS);
+    });
+  }
+  return !commandProcessTreeAlive(child);
+}
+
+async function finishTimedOutCommandProcessTree(
+  child: ChildProcess,
+  options: {
+    forceKillAt: number | undefined;
+    timeoutKillGraceMs: number;
+  },
+) {
+  if (!commandProcessTreeAlive(child)) {
+    activeCommandChildren.delete(child);
+    return;
+  }
+  const graceRemainingMs =
+    options.forceKillAt === undefined
+      ? options.timeoutKillGraceMs
+      : Math.max(0, options.forceKillAt - Date.now());
+  if (graceRemainingMs > 0) {
+    await waitForCommandProcessTreeExit(child, graceRemainingMs);
+  }
+  if (commandProcessTreeAlive(child)) {
+    signalCommandTree(child, "SIGKILL");
+    await waitForCommandProcessTreeExit(child, options.timeoutKillGraceMs);
+  }
+  activeCommandChildren.delete(child);
+}
+
 function untrackCommandChild(child: ChildProcess) {
   if (!commandProcessTreeAlive(child)) {
     activeCommandChildren.delete(child);
@@ -664,6 +703,7 @@ export function runCommand(params: {
     let settled = false;
     let stdoutLimitError: string | null = null;
     let timeoutError: Error | null = null;
+    let forceKillAt: number | undefined;
     let killTimer: NodeJS.Timeout | undefined;
     const timeoutMs = params.timeoutMs ?? COMMAND_TIMEOUT_MS;
     const timeoutKillGraceMs = params.timeoutKillGraceMs ?? COMMAND_TIMEOUT_KILL_GRACE_MS;
@@ -684,6 +724,7 @@ export function runCommand(params: {
         )}`,
       );
       signalCommandTree(child, "SIGTERM");
+      forceKillAt = Date.now() + timeoutKillGraceMs;
       killTimer = setTimeout(() => {
         signalCommandTree(child, "SIGKILL");
       }, timeoutKillGraceMs);
@@ -736,9 +777,16 @@ export function runCommand(params: {
       settled = true;
       untrackCommandChild(child);
       if (timeoutError) {
-        signalCommandTree(child, "SIGKILL");
+        const error = timeoutError;
         clearTimers();
-        reject(timeoutError);
+        void finishTimedOutCommandProcessTree(child, {
+          forceKillAt,
+          timeoutKillGraceMs,
+        }).then(
+          () => reject(error),
+          (cleanupError: unknown) =>
+            reject(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))),
+        );
         return;
       }
       clearTimers();
diff --git a/test/scripts/telegram-user-crabbox-proof.test.ts b/test/scripts/telegram-user-crabbox-proof.test.ts
@@ -311,6 +311,55 @@ setInterval(() => {}, 1000);
     }
   });
 
+  posixIt("lets timed-out command descendants exit during kill grace", async () => {
+    const root = makeTempDir();
+    const scriptPath = path.join(root, "trap-term-grace.mjs");
+    const readyPath = path.join(root, "descendant.ready");
+    const donePath = path.join(root, "descendant.done");
+
+    fs.writeFileSync(
+      scriptPath,
+      `
+import { spawn } from "node:child_process";
+
+const descendant = spawn(process.execPath, [
+  "--input-type=module",
+  "--eval",
+  ${JSON.stringify(
+    `import { writeFileSync } from "node:fs";
+writeFileSync(${JSON.stringify(readyPath)}, "ready");
+process.on("SIGTERM", () => {
+  setTimeout(() => {
+    writeFileSync(${JSON.stringify(donePath)}, "done");
+    process.exit(0);
+  }, 75);
+});
+setInterval(() => {}, 1000);`,
+  )},
+], { stdio: "ignore" });
+descendant.unref();
+process.on("SIGTERM", () => process.exit(0));
+setInterval(() => {}, 1000);
+`,
+      "utf8",
+    );
+
+    const runPromise = runCommand({
+      args: [scriptPath],
+      command: process.execPath,
+      cwd: root,
+      timeoutKillGraceMs: 500,
+      timeoutMs: 500,
+    });
+
+    await waitFor(() => fs.existsSync(readyPath));
+    await expect(runPromise).rejects.toMatchObject({
+      code: "ETIMEDOUT",
+      message: expect.stringContaining("timed out after 500ms"),
+    });
+    expect(fs.readFileSync(donePath, "utf8")).toBe("done");
+  });
+
   posixIt("keeps closed command groups tracked for parent cleanup", async () => {
     const root = makeTempDir();
     const commandPath = path.join(root, "closed-command.mjs");
