fix(lint): reap timed-out oxlint shard groups

vincentkoc · vincentkoc · commit f26f45c050cc · 2026-06-20T09:54:36.000+02:00
diff --git a/scripts/run-oxlint-shards.mjs b/scripts/run-oxlint-shards.mjs
@@ -13,6 +13,8 @@ const DEFAULT_WINDOWS_EXTENSION_CHUNK_SIZE = 8;
 const DEFAULT_SHARD_HEARTBEAT_MS = 30_000;
 const DEFAULT_SHARD_TIMEOUT_MS = 15 * 60_000;
 const DEFAULT_SHARD_KILL_GRACE_MS = 5_000;
+const POST_FORCE_KILL_WAIT_MS = 1_000;
+const PROCESS_GROUP_EXIT_POLL_MS = 25;
 const DEFAULT_SPLIT_CORE_SHARD_CONCURRENCY = 4;
 const FAST_LOCAL_CHECK_MIN_CPUS = 12;
 const FAST_LOCAL_CHECK_MIN_MEMORY_BYTES = 48 * 1024 ** 3;
@@ -443,6 +445,7 @@ export async function runShard({ env, extraArgs, runner, shard }) {
     let finished = false;
     let timedOut = false;
     let forceKill = null;
+    let forceKillAt = null;
     const heartbeat =
       heartbeatMs > 0
         ? setInterval(() => {
@@ -461,6 +464,7 @@ export async function runShard({ env, extraArgs, runner, shard }) {
             );
             signalChildProcess({ child, signal: "SIGTERM", useProcessGroup });
             if (killGraceMs > 0) {
+              forceKillAt = Date.now() + killGraceMs;
               forceKill = setTimeout(() => {
                 console.error(`[oxlint:${shard.name}] did not exit cleanly; killing shard`);
                 signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
@@ -486,22 +490,49 @@ export async function runShard({ env, extraArgs, runner, shard }) {
       if (forceKill) {
         clearTimeout(forceKill);
       }
+      forceKillAt = null;
       unregisterShardChild();
       console.error(`[oxlint:${shard.name}] finished`);
       resolve(status);
     };
+    const finishAfterForcedTeardown = async (status) => {
+      const graceRemainingMs =
+        forceKillAt === null ? killGraceMs : Math.max(0, forceKillAt - Date.now());
+      if (graceRemainingMs > 0) {
+        await waitForChildProcessGroupExit({
+          child,
+          timeoutMs: graceRemainingMs,
+          useProcessGroup,
+        });
+      }
+      if (isChildProcessGroupAlive({ child, useProcessGroup })) {
+        signalChildProcess({ child, signal: "SIGKILL", useProcessGroup });
+      }
+      await waitForChildProcessGroupExit({
+        child,
+        timeoutMs: POST_FORCE_KILL_WAIT_MS,
+        useProcessGroup,
+      });
+      finish(status);
+    };
     child.once("error", (error) => {
       console.error(error);
       finish(1);
     });
     child.once("close", (status) => {
-      finish(
-        parentTerminationSignal
-          ? getSignalExitCode(parentTerminationSignal)
-          : timedOut
-            ? 124
-            : (status ?? 1),
-      );
+      const exitStatus = parentTerminationSignal
+        ? getSignalExitCode(parentTerminationSignal)
+        : timedOut
+          ? 124
+          : (status ?? 1);
+      if (
+        (timedOut || parentTerminationSignal) &&
+        isChildProcessGroupAlive({ child, useProcessGroup })
+      ) {
+        void finishAfterForcedTeardown(exitStatus);
+        return;
+      }
+      finish(exitStatus);
     });
   });
 }
@@ -604,6 +635,31 @@ function signalChildProcess({ child, signal, useProcessGroup }) {
   }
 }
 
+function isChildProcessGroupAlive({ child, useProcessGroup }) {
+  if (!useProcessGroup || !child.pid) {
+    return false;
+  }
+  try {
+    process.kill(-child.pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === "EPERM";
+  }
+}
+
+async function waitForChildProcessGroupExit({ child, timeoutMs, useProcessGroup }) {
+  const deadlineAt = Date.now() + timeoutMs;
+  while (Date.now() < deadlineAt) {
+    if (!isChildProcessGroupAlive({ child, useProcessGroup })) {
+      return true;
+    }
+    await new Promise((resolvePoll) => {
+      setTimeout(resolvePoll, PROCESS_GROUP_EXIT_POLL_MS);
+    });
+  }
+  return !isChildProcessGroupAlive({ child, useProcessGroup });
+}
+
 function registerShardChild(entry) {
   installParentSignalForwarding();
   ACTIVE_SHARD_CHILDREN.add(entry);
diff --git a/test/scripts/run-oxlint.test.ts b/test/scripts/run-oxlint.test.ts
@@ -1,6 +1,6 @@
 // Run Oxlint tests cover run oxlint script behavior.
 import { spawnSync } from "node:child_process";
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { pathToFileURL } from "node:url";
@@ -22,6 +22,31 @@ import {
   filterSparseMissingOxlintTargets,
   shouldPrepareExtensionPackageBoundaryArtifacts,
 } from "../../scripts/run-oxlint.mjs";
+import { createScriptTestHarness } from "./test-helpers.js";
+
+const { createTempDir } = createScriptTestHarness();
+
+async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
+  const deadlineAt = Date.now() + timeoutMs;
+  while (Date.now() < deadlineAt) {
+    if (predicate()) {
+      return;
+    }
+    await new Promise((resolvePoll) => {
+      setTimeout(resolvePoll, 25);
+    });
+  }
+  throw new Error("condition was not met before timeout");
+}
+
+function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
 
 describe("run-oxlint", () => {
   it("prepares extension package boundary artifacts for normal lint runs", () => {
@@ -256,6 +281,57 @@ describe("run-oxlint", () => {
     }
   });
 
+  it.runIf(process.platform !== "win32")(
+    "kills timed-out shard process groups when the leader exits first",
+    async () => {
+      const tempDir = createTempDir("openclaw-oxlint-timeout-group-");
+      const runner = join(tempDir, "timeout-runner.mjs");
+      const childPidPath = join(tempDir, "child.pid");
+      let childPid = 0;
+      const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
+      try {
+        writeFileSync(
+          runner,
+          [
+            "import { spawn } from 'node:child_process';",
+            "import { writeFileSync } from 'node:fs';",
+            `const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
+            "writeFileSync(process.env.CHILD_PID_PATH, String(child.pid));",
+            "process.on('SIGTERM', () => process.exit(0));",
+            "setInterval(() => {}, 1000);",
+            "",
+          ].join("\n"),
+          "utf8",
+        );
+
+        const command = runShard({
+          env: {
+            ...process.env,
+            CHILD_PID_PATH: childPidPath,
+            OPENCLAW_OXLINT_SHARD_HEARTBEAT_MS: "0",
+            OPENCLAW_OXLINT_SHARD_KILL_GRACE_MS: "25",
+            OPENCLAW_OXLINT_SHARD_TIMEOUT_MS: "1000",
+          },
+          extraArgs: [],
+          runner,
+          shard: { name: "timeout-group-test", args: [] },
+        });
+
+        await waitFor(() => existsSync(childPidPath), 2_000);
+        childPid = Number(readFileSync(childPidPath, "utf8"));
+        expect(isProcessAlive(childPid)).toBe(true);
+
+        await expect(command).resolves.toBe(124);
+        await waitFor(() => !isProcessAlive(childPid), 2_000);
+      } finally {
+        if (childPid && isProcessAlive(childPid)) {
+          process.kill(childPid, "SIGKILL");
+        }
+        rmSync(tempDir, { force: true, recursive: true });
+      }
+    },
+  );
+
   it.runIf(process.platform !== "win32")(
     "forwards parent termination to detached oxlint shard processes",
     () => {
@@ -406,6 +482,92 @@ describe("run-oxlint", () => {
     },
   );
 
+  it.runIf(process.platform !== "win32")(
+    "kills parent-terminated shard process groups when the leader exits first",
+    () => {
+      const tempDir = createTempDir("openclaw-oxlint-parent-group-");
+      const runner = join(tempDir, "signal-runner.mjs");
+      const harness = join(tempDir, "signal-harness.mjs");
+      const childPidPath = join(tempDir, "child.pid");
+      const readyFile = join(tempDir, "ready");
+      const childScript = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);";
+      try {
+        writeFileSync(
+          runner,
+          [
+            "import { spawn } from 'node:child_process';",
+            "import { writeFileSync } from 'node:fs';",
+            `const child = spawn(process.execPath, ['-e', ${JSON.stringify(childScript)}], { stdio: 'ignore' });`,
+            "writeFileSync(process.env.CHILD_PID_PATH, String(child.pid));",
+            "writeFileSync(process.env.READY_FILE, String(process.pid));",
+            "process.on('SIGTERM', () => process.exit(0));",
+            "setInterval(() => {}, 1000);",
+            "",
+          ].join("\n"),
+          "utf8",
+        );
+        writeFileSync(
+          harness,
+          [
+            "import { existsSync, readFileSync } from 'node:fs';",
+            `import { runShard } from ${JSON.stringify(pathToFileURL(join(process.cwd(), "scripts/run-oxlint-shards.mjs")).href)};`,
+            "const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));",
+            "const isAlive = (pid) => {",
+            "  try { process.kill(pid, 0); return true; } catch { return false; }",
+            "};",
+            "const waitFor = async (predicate) => {",
+            "  for (let attempt = 0; attempt < 100; attempt += 1) {",
+            "    if (predicate()) return true;",
+            "    await sleep(25);",
+            "  }",
+            "  return false;",
+            "};",
+            "const promise = runShard({",
+            "  env: {",
+            "    ...process.env,",
+            "    OPENCLAW_OXLINT_SHARD_HEARTBEAT_MS: '0',",
+            "    OPENCLAW_OXLINT_SHARD_KILL_GRACE_MS: '25',",
+            "    OPENCLAW_OXLINT_SHARD_TIMEOUT_MS: '0',",
+            "  },",
+            "  extraArgs: [],",
+            "  runner: process.env.RUNNER_FILE,",
+            "  shard: { name: 'signal-group-test', args: [] },",
+            "});",
+            "if (!(await waitFor(() => existsSync(process.env.CHILD_PID_PATH)))) {",
+            "  process.exit(2);",
+            "}",
+            "const childPid = Number(readFileSync(process.env.CHILD_PID_PATH, 'utf8'));",
+            "process.kill(process.pid, 'SIGTERM');",
+            "const status = await promise;",
+            "if (await waitFor(() => !isAlive(childPid))) {",
+            "  process.exit(status === 143 ? 0 : 4);",
+            "}",
+            "process.kill(childPid, 'SIGKILL');",
+            "process.exit(5);",
+            "",
+          ].join("\n"),
+          "utf8",
+        );
+
+        const result = spawnSync(process.execPath, [harness], {
+          encoding: "utf8",
+          env: {
+            ...process.env,
+            CHILD_PID_PATH: childPidPath,
+            READY_FILE: readyFile,
+            RUNNER_FILE: runner,
+          },
+          timeout: 5_000,
+        });
+
+        expect(result.status).toBe(0);
+        expect(result.signal).toBeNull();
+      } finally {
+        rmSync(tempDir, { force: true, recursive: true });
+      }
+    },
+  );
+
   it("chunks extension oxlint shards on Windows", () => {
     const shards = createOxlintShards({
       cwd: "/repo",
