Skip to content

Commit b34c5fc

Browse files
kklouzalkklouzal
committed
test(gateway): harden watch regression diagnostics
1 parent 654544b commit b34c5fc

2 files changed

Lines changed: 411 additions & 13 deletions

File tree

‎scripts/check-gateway-watch-regression.mjs‎

Lines changed: 124 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,15 @@ const WATCH_GATEWAY_SKIP_ENV = {
4949
* Maximum retained stdout/stderr text for gateway watch diagnostics.
5050
*/
5151
export const WATCH_LOG_CAPTURE_MAX_CHARS = 2 * 1024 * 1024;
52+
export const WATCH_LOG_FAILURE_TAIL_CHARS = 12_000;
5253
const WATCH_BUILD_DETECTION_MAX_CHARS = 4096;
5354
const NON_NEGATIVE_INTEGER_PATTERN = /^(0|[1-9]\d*)$/u;
5455
const ANSI_ESCAPE_PATTERN = new RegExp(`${String.fromCharCode(27)}\\[[0-?]*[ -/]*[@-~]`, "g");
5556

57+
function shellQuote(value) {
58+
return `'${String(value).replaceAll("'", "'\\''")}'`;
59+
}
60+
5661
/**
5762
* Appends watch output while preserving only the diagnostic tail.
5863
*/
@@ -70,6 +75,28 @@ function formatCapturedWatchLog(text, truncated) {
7075
: text;
7176
}
7277

78+
function formatWatchExit(exit) {
79+
if (!exit) {
80+
return "unknown exit";
81+
}
82+
const parts = [];
83+
if (exit.code !== undefined) {
84+
parts.push(`code ${exit.code === null ? "null" : exit.code}`);
85+
}
86+
if (exit.signal !== undefined) {
87+
parts.push(`signal ${exit.signal === null ? "null" : exit.signal}`);
88+
}
89+
if (exit.error) {
90+
parts.push(`error ${exit.error}`);
91+
}
92+
return parts.length > 0 ? parts.join(", ") : "unknown exit";
93+
}
94+
95+
function readTextTail(filePath, maxChars = WATCH_LOG_FAILURE_TAIL_CHARS) {
96+
const text = fs.readFileSync(filePath, "utf8");
97+
return text.length <= maxChars ? text : text.slice(-maxChars);
98+
}
99+
73100
/**
74101
* Updates bounded watch-build detection state from new output.
75102
*/
@@ -454,36 +481,58 @@ async function allocateLoopbackPort() {
454481
});
455482
}
456483

457-
function buildTimedWatchCommand(pidFilePath, timeFilePath, isolatedHomeDir, port) {
484+
export function resolveTimedWatchShell(deps = {}) {
485+
const existsSync = deps.existsSync ?? fs.existsSync;
486+
for (const shellPath of ["/bin/bash", "/usr/bin/bash", "/bin/sh"]) {
487+
if (existsSync(shellPath)) {
488+
return shellPath;
489+
}
490+
}
491+
return "/bin/sh";
492+
}
493+
494+
export function buildTimedWatchCommand(
495+
pidFilePath,
496+
timeFilePath,
497+
isolatedHomeDir,
498+
port,
499+
deps = {},
500+
) {
458501
const isolatedStateDir = path.join(isolatedHomeDir, ".openclaw");
459502
const isolatedConfigPath = path.join(isolatedStateDir, "openclaw.json");
503+
// CI env hooks can contain bash-only `declare` lines; running the watch shell
504+
// under sh delays gateway readiness behind stderr noise before the idle window.
505+
const shellPath = deps.shellPath ?? resolveTimedWatchShell(deps);
506+
const nodeExecPath = deps.nodeExecPath ?? process.execPath;
460507
const shellSource = [
461508
'echo "$$" > "$OPENCLAW_WATCH_PID_FILE"',
462509
'mkdir -p "$OPENCLAW_STATE_DIR"',
463510
`printf '%s\n' '{"gateway":{"controlUi":{"enabled":false}},"plugins":{"enabled":false}}' > "$OPENCLAW_CONFIG_PATH"`,
464-
`exec node scripts/watch-node.mjs gateway --force --allow-unconfigured --port ${String(port)} --token watch-regression-token`,
511+
`exec ${shellQuote(nodeExecPath)} scripts/watch-node.mjs gateway --force --allow-unconfigured --port ${String(port)} --token watch-regression-token`,
465512
].join("\n");
513+
const nodeBinDir = path.dirname(nodeExecPath);
466514
const env = {
467515
OPENCLAW_WATCH_PID_FILE: pidFilePath,
468516
HOME: isolatedHomeDir,
469517
OPENCLAW_HOME: isolatedHomeDir,
470518
OPENCLAW_CONFIG_PATH: isolatedConfigPath,
471519
OPENCLAW_STATE_DIR: isolatedStateDir,
520+
PATH: `${nodeBinDir}${path.delimiter}${process.env.PATH ?? ""}`,
472521
XDG_CONFIG_HOME: path.join(isolatedHomeDir, ".config"),
473522
...WATCH_GATEWAY_SKIP_ENV,
474523
};
475524

476525
if (process.platform === "darwin") {
477526
return {
478527
command: "/usr/bin/time",
479-
args: ["-lp", "-o", timeFilePath, "/bin/sh", "-lc", shellSource],
528+
args: ["-lp", "-o", timeFilePath, shellPath, "-lc", shellSource],
480529
env,
481530
};
482531
}
483532

484533
if (!fs.existsSync("/usr/bin/time")) {
485534
return {
486-
command: "/bin/sh",
535+
command: shellPath,
487536
args: ["-lc", shellSource],
488537
env,
489538
};
@@ -496,7 +545,7 @@ function buildTimedWatchCommand(pidFilePath, timeFilePath, isolatedHomeDir, port
496545
"__TIMING__ user=%U sys=%S elapsed=%e",
497546
"-o",
498547
timeFilePath,
499-
"/bin/sh",
548+
shellPath,
500549
"-lc",
501550
shellSource,
502551
],
@@ -585,56 +634,78 @@ export async function runTimedWatch(options, outputDir, deps = {}) {
585634
resolve({ code: null, signal: null, error: error.message });
586635
});
587636
});
588-
const raceSpawnError = async (operation) =>
637+
const childExit = new Promise((resolve) => {
638+
child.once("exit", (code, signal) => resolve({ code, signal }));
639+
});
640+
const raceChildLifecycle = async (operation) =>
589641
await Promise.race([
590642
Promise.resolve(operation).then((value) => ({ type: "value", value })),
591643
spawnErrorExit.then((value) => ({ type: "spawn-error", value })),
644+
childExit.then((value) => ({ type: "child-exit", value })),
592645
]);
593646

594647
let watchPid = null;
595648
let exit = null;
649+
let exitedBeforeReady = false;
650+
let exitedBeforeStop = false;
596651
for (let attempt = 0; attempt < 50; attempt += 1) {
597652
if (fs.existsSync(pidFilePath)) {
598653
watchPid = Number(fs.readFileSync(pidFilePath, "utf8").trim());
599654
break;
600655
}
601-
const waitResult = await raceSpawnError(sleepMs(100));
656+
const waitResult = await raceChildLifecycle(sleepMs(100));
602657
if (waitResult.type === "spawn-error") {
603658
exit = waitResult.value;
604659
break;
605660
}
661+
if (waitResult.type === "child-exit") {
662+
exit = waitResult.value;
663+
exitedBeforeReady = true;
664+
exitedBeforeStop = true;
665+
break;
666+
}
606667
}
607668

608669
let readyBeforeWindow = false;
609670
let idleCpuStartMs = null;
610671
let idleCpuEndMs = null;
611672
if (!exit) {
612-
const readyResult = await raceSpawnError(
673+
const readyResult = await raceChildLifecycle(
613674
waitReady(() => `${stdout}\n${stderr}`, options.readyTimeoutMs),
614675
);
615676
if (readyResult.type === "spawn-error") {
616677
exit = readyResult.value;
678+
} else if (readyResult.type === "child-exit") {
679+
exit = readyResult.value;
680+
exitedBeforeReady = true;
681+
exitedBeforeStop = true;
617682
} else {
618683
readyBeforeWindow = readyResult.value;
619684
}
620685
}
621686
if (!exit && readyBeforeWindow && options.readySettleMs > 0) {
622-
const settleResult = await raceSpawnError(sleepMs(options.readySettleMs));
687+
const settleResult = await raceChildLifecycle(sleepMs(options.readySettleMs));
623688
if (settleResult.type === "spawn-error") {
624689
exit = settleResult.value;
690+
} else if (settleResult.type === "child-exit") {
691+
exit = settleResult.value;
692+
exitedBeforeStop = true;
625693
}
626694
}
627695
if (!exit) {
628696
idleCpuStartMs = watchPid ? readCpuMs(watchPid) : null;
629-
const windowResult = await raceSpawnError(sleepMs(options.windowMs));
697+
const windowResult = await raceChildLifecycle(sleepMs(options.windowMs));
630698
if (windowResult.type === "spawn-error") {
631699
exit = windowResult.value;
700+
} else if (windowResult.type === "child-exit") {
701+
exit = windowResult.value;
702+
exitedBeforeStop = true;
632703
} else {
633704
idleCpuEndMs = watchPid ? readCpuMs(watchPid) : null;
634705
}
635706
}
636707
if (!exit) {
637-
const stopResult = await raceSpawnError(stopChild(child, watchPid, options));
708+
const stopResult = await raceChildLifecycle(stopChild(child, watchPid, options));
638709
exit = stopResult.value;
639710
}
640711

@@ -651,6 +722,8 @@ export async function runTimedWatch(options, outputDir, deps = {}) {
651722
timingFileMissing,
652723
timing,
653724
readyBeforeWindow,
725+
exitedBeforeReady,
726+
exitedBeforeStop,
654727
idleCpuMs:
655728
idleCpuStartMs == null || idleCpuEndMs == null
656729
? null
@@ -814,9 +887,16 @@ export function collectGatewayWatchFindings(params) {
814887
if (watchResult.spawnError) {
815888
failures.push(`gateway:watch failed to start: ${watchResult.spawnError}`);
816889
}
817-
if (!watchResult.readyBeforeWindow) {
890+
if (!watchResult.readyBeforeWindow && watchResult.exitedBeforeReady) {
891+
failures.push(`gateway:watch exited before ready (${formatWatchExit(watchResult.exit)})`);
892+
} else if (!watchResult.readyBeforeWindow) {
818893
failures.push("gateway:watch did not report ready before the idle CPU window");
819894
}
895+
if (watchResult.readyBeforeWindow && watchResult.exitedBeforeStop) {
896+
failures.push(
897+
`gateway:watch exited before the idle CPU window completed (${formatWatchExit(watchResult.exit)})`,
898+
);
899+
}
820900
if (watchResult.timingFileMissing && !Number.isFinite(watchResult.idleCpuMs)) {
821901
failures.push(
822902
"failed to collect CPU timing from the bounded gateway:watch run; timing artifact is missing",
@@ -855,6 +935,32 @@ export function collectGatewayWatchFindings(params) {
855935
return { failures, warnings };
856936
}
857937

938+
export function shouldReportDuplicateDistRuntimeRegression(failures) {
939+
return failures.some((message) => message.startsWith("dist-runtime "));
940+
}
941+
942+
function printWatchLogDiagnostics(watchResult) {
943+
const artifacts = [
944+
["stderr", watchResult.stderrPath],
945+
["stdout", watchResult.stdoutPath],
946+
];
947+
for (const [label, filePath] of artifacts) {
948+
if (!filePath || !fs.existsSync(filePath)) {
949+
continue;
950+
}
951+
const tail = readTextTail(filePath).trimEnd();
952+
if (!tail) {
953+
warn(`gateway:watch ${label} artifact is empty (${filePath})`);
954+
continue;
955+
}
956+
console.error(
957+
`--- gateway:watch ${label} tail (${filePath}, last ${WATCH_LOG_FAILURE_TAIL_CHARS} chars) ---`,
958+
);
959+
console.error(tail);
960+
console.error(`--- end gateway:watch ${label} tail ---`);
961+
}
962+
}
963+
858964
async function main() {
859965
const options = parseArgs(process.argv.slice(2));
860966
ensureDir(options.outputDir);
@@ -938,6 +1044,8 @@ async function main() {
9381044
cpuMs,
9391045
totalCpuMs,
9401046
readyBeforeWindow: watchResult.readyBeforeWindow,
1047+
exitedBeforeReady: watchResult.exitedBeforeReady,
1048+
exitedBeforeStop: watchResult.exitedBeforeStop,
9411049
cpuWarnMs: options.cpuWarnMs,
9421050
cpuFailMs: options.cpuFailMs,
9431051
distRuntimeFileGrowth,
@@ -949,6 +1057,8 @@ async function main() {
9491057
removedPaths: diff.removed.length,
9501058
watchExit: watchResult.exit,
9511059
spawnError: watchResult.spawnError,
1060+
stdoutPath: watchResult.stdoutPath,
1061+
stderrPath: watchResult.stderrPath,
9521062
timingFileMissing: watchResult.timingFileMissing,
9531063
timing: watchResult.timing,
9541064
};
@@ -977,7 +1087,8 @@ async function main() {
9771087
for (const message of failures) {
9781088
fail(message);
9791089
}
980-
if (!failures.every((message) => message.includes("dirty watched source tree"))) {
1090+
printWatchLogDiagnostics(watchResult);
1091+
if (shouldReportDuplicateDistRuntimeRegression(failures)) {
9811092
fail(
9821093
"Possible duplicate dist-runtime graph regression: this can reintroduce split runtime personalities where plugins and core observe different global state, including Telegram missing /voice, /phone, or /pair.",
9831094
);

0 commit comments

Comments
 (0)