Skip to content

Commit 3be74ca

Browse files
vincentkocvincentkoc
committed
fix(e2e): wait for kitchen rpc process groups
1 parent 2c71e71 commit 3be74ca

2 files changed

Lines changed: 127 additions & 36 deletions

File tree

‎scripts/e2e/kitchen-sink-rpc-walk.mjs‎

Lines changed: 82 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const DEFAULT_MAX_COMMAND_RSS_MIB = 8192;
3131
const DEFAULT_OUTPUT_CAPTURE_CHARS = 1024 * 1024;
3232
const GATEWAY_TEARDOWN_GRACE_MS = 10000;
3333
const GATEWAY_TEARDOWN_KILL_GRACE_MS = 2000;
34+
const COMMAND_PROCESS_TREE_EXIT_POLL_MS = 50;
3435
const LOG_SCAN_CHUNK_BYTES = 64 * 1024;
3536
const LOG_SCAN_MAX_LINE_CHARS = 16 * 1024;
3637
const LOG_TAIL_BYTES = 256 * 1024;
@@ -380,49 +381,94 @@ export function runCommand(command, args, options = {}) {
380381
});
381382
child.on("close", (status, signal) => {
382383
clearTimeout(timer);
383-
clearTimeout(forceKillTimer);
384-
void stopResourceSampling().then((resourceSampleFailure) => {
385-
if (!timedOut && status === 0) {
386-
if (resourceSampleFailure) {
387-
reject(resourceSampleFailure);
384+
const finish = () => {
385+
clearTimeout(forceKillTimer);
386+
void stopResourceSampling().then((resourceSampleFailure) => {
387+
if (!timedOut && status === 0) {
388+
if (resourceSampleFailure) {
389+
reject(resourceSampleFailure);
390+
return;
391+
}
392+
resolve({
393+
stdout: stdout.text,
394+
stderr: stderr.text,
395+
stdoutTruncatedChars: stdout.truncatedChars,
396+
stderrTruncatedChars: stderr.truncatedChars,
397+
});
388398
return;
389399
}
390-
resolve({
391-
stdout: stdout.text,
392-
stderr: stderr.text,
393-
stdoutTruncatedChars: stdout.truncatedChars,
394-
stderrTruncatedChars: stderr.truncatedChars,
395-
});
396-
return;
397-
}
398-
const detail = [
399-
formatCapturedOutput("stdout", stdout),
400-
formatCapturedOutput("stderr", stderr),
401-
]
402-
.filter(Boolean)
403-
.join("\n")
404-
.trim();
405-
const failure = timedOut
406-
? `timed out after ${timeoutMs}ms`
407-
: `failed with ${signal || status}`;
408-
reject(
409-
Object.assign(
410-
new Error(
411-
`${command} ${args.join(" ")} ${failure}${detail ? `\n${tailText(detail)}` : ""}`,
400+
const detail = [
401+
formatCapturedOutput("stdout", stdout),
402+
formatCapturedOutput("stderr", stderr),
403+
]
404+
.filter(Boolean)
405+
.join("\n")
406+
.trim();
407+
const failure = timedOut
408+
? `timed out after ${timeoutMs}ms`
409+
: `failed with ${signal || status}`;
410+
reject(
411+
Object.assign(
412+
new Error(
413+
`${command} ${args.join(" ")} ${failure}${detail ? `\n${tailText(detail)}` : ""}`,
414+
),
415+
{
416+
signal,
417+
status,
418+
stderr: stderr.text,
419+
stdout: stdout.text,
420+
},
412421
),
413-
{
414-
signal,
415-
status,
416-
stderr: stderr.text,
417-
stdout: stdout.text,
418-
},
419-
),
420-
);
421-
});
422+
);
423+
});
424+
};
425+
426+
if (timedOut) {
427+
void finishTimedOutCommandProcessTree(child, timeoutKillGraceMs).then(finish, finish);
428+
return;
429+
}
430+
431+
finish();
422432
});
423433
});
424434
}
425435

436+
async function finishTimedOutCommandProcessTree(child, timeoutKillGraceMs) {
437+
if (!commandProcessTreeIsAlive(child)) {
438+
return;
439+
}
440+
signalProcessGroup(child, "SIGKILL");
441+
await waitForCommandProcessTreeExit(child, timeoutKillGraceMs);
442+
}
443+
444+
async function waitForCommandProcessTreeExit(child, timeoutMs) {
445+
const deadlineAt = Date.now() + timeoutMs;
446+
while (Date.now() < deadlineAt) {
447+
if (!commandProcessTreeIsAlive(child)) {
448+
return true;
449+
}
450+
await new Promise((resolvePoll) => {
451+
setTimeout(resolvePoll, COMMAND_PROCESS_TREE_EXIT_POLL_MS);
452+
});
453+
}
454+
return !commandProcessTreeIsAlive(child);
455+
}
456+
457+
function commandProcessTreeIsAlive(child) {
458+
if (process.platform === "win32" || typeof child.pid !== "number") {
459+
return !hasChildExited(child);
460+
}
461+
try {
462+
process.kill(-child.pid, 0);
463+
return true;
464+
} catch (error) {
465+
if (error?.code === "EPERM") {
466+
return true;
467+
}
468+
return false;
469+
}
470+
}
471+
426472
function signalProcessGroup(child, signal) {
427473
if (process.platform !== "win32" && typeof child.pid === "number") {
428474
try {

‎test/scripts/kitchen-sink-rpc-walk.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -761,6 +761,51 @@ describe("kitchen-sink RPC caller loading", () => {
761761
rmSync(root, { recursive: true, force: true });
762762
}
763763
});
764+
765+
posixIt("kills descendants when timed commands exit cleanly after SIGTERM", async () => {
766+
const root = mkdtempSync(path.join(tmpdir(), "openclaw-kitchen-rpc-timeout-clean-parent-"));
767+
const scriptPath = path.join(root, "term-zero-grandchild.mjs");
768+
const grandchildPidPath = path.join(root, "grandchild.pid");
769+
let grandchildPid = 0;
770+
771+
writeFileSync(
772+
scriptPath,
773+
`
774+
import { spawn } from "node:child_process";
775+
import fs from "node:fs";
776+
777+
const grandchild = spawn(process.execPath, [
778+
"-e",
779+
"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);",
780+
], { stdio: "ignore" });
781+
fs.writeFileSync(process.argv[2], String(grandchild.pid));
782+
process.on("SIGTERM", () => process.exit(0));
783+
setInterval(() => {}, 1000);
784+
`,
785+
"utf8",
786+
);
787+
788+
const runPromise = runCommand(process.execPath, [scriptPath, grandchildPidPath], {
789+
timeoutKillGraceMs: 2_000,
790+
timeoutMs: 100,
791+
});
792+
793+
try {
794+
await waitFor(() => existsSync(grandchildPidPath));
795+
grandchildPid = Number.parseInt(readText(grandchildPidPath), 10);
796+
expect(Number.isInteger(grandchildPid)).toBe(true);
797+
expect(isProcessAlive(grandchildPid)).toBe(true);
798+
799+
await expect(runPromise).rejects.toThrow("timed out after 100ms");
800+
await waitFor(() => !isProcessAlive(grandchildPid), 5_000);
801+
} finally {
802+
await runPromise.catch(() => {});
803+
if (grandchildPid && isProcessAlive(grandchildPid)) {
804+
process.kill(grandchildPid, "SIGKILL");
805+
}
806+
rmSync(root, { recursive: true, force: true });
807+
}
808+
});
764809
});
765810

766811
describe("kitchen-sink RPC payload unwrapping", () => {

0 commit comments

Comments
 (0)