Skip to content

Commit 98bd8c0

Browse files
authored
test(cli): await sibling startup signal instead of polling deadline (#8062)
Co-authored-by: JUN <[email protected]>
1 parent d5708c3 commit 98bd8c0

3 files changed

Lines changed: 85 additions & 106 deletions

File tree

‎tests/cli/cli-start-journal-order.test.ts‎

Lines changed: 28 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { join, resolve } from "node:path";
1010
import { watchdogMs } from "../helpers/ci-watchdog";
1111
import { removeTreeWithRetry } from "../helpers/remove-tree";
1212
import { repoPath } from "../helpers/repo-root";
13+
import { captureStartupChildOutput } from "../helpers/startup-child-output";
1314
import {
1415
inspectServiceStateRecords,
1516
selectAuthoritativeServiceState,
@@ -33,6 +34,7 @@ const JOURNAL_OWNERSHIP_BUDGET_MS = Math.max(30_000, OWNER_WAIT_MS * 4);
3334
const cliPath = repoPath("src/cli/index.ts");
3435
const roots: string[] = [];
3536
const children: Array<ReturnType<typeof Bun.spawn>> = [];
37+
const startupOutputs: Array<ReturnType<typeof captureStartupChildOutput>> = [];
3638

3739
type Fixture = {
3840
root: string;
@@ -162,6 +164,7 @@ afterEach(async () => {
162164
const child = children.pop()!;
163165
if (child.exitCode === null) await child.exited;
164166
}
167+
for (const output of startupOutputs.splice(0)) await output.finish();
165168
while (roots.length) removeTreeWithRetry(roots.pop()!);
166169
});
167170

@@ -398,124 +401,46 @@ function freeLoopbackPort(): number {
398401
}
399402

400403
/**
401-
* Who holds a pinned port when a sibling missed its deadline: the child itself (bound, record not
402-
* yet written), another process (the port was taken between probe and bind) or nobody (the child
403-
* never reached its bind). The bare deadline cannot tell these apart.
404-
*
405-
* Best effort and read-only while the child lives: socket owners are listed before the child is
406-
* killed, the bind probe runs after. Nothing here may throw over the original timeout.
407-
*/
408-
function pinnedPortSockets(port: number, childPid: number): string {
409-
if (process.platform !== "win32") return "";
410-
try {
411-
const listing = Bun.spawnSync(["netstat", "-ano", "-p", "TCP"], { stdout: "pipe", stderr: "ignore", timeout: 5_000 });
412-
if (listing.exitCode !== 0) return `; netstat exit ${listing.exitCode}`;
413-
const rows = [...new Set(listing.stdout.toString().split(/\r?\n/)
414-
.map(line => line.trim().split(/\s+/))
415-
.filter(cols => cols.length >= 5 && cols[1]!.endsWith(`:${port}`))
416-
.map(cols => `${cols[3]} pid ${cols[4]}${cols[4] === String(childPid) ? " (child)" : ""}`))];
417-
return rows.length ? `; ${rows.join(", ")}` : "; no socket rows";
418-
} catch (error) {
419-
return `; netstat unavailable (${(error as NodeJS.ErrnoException).code ?? "error"})`;
420-
}
421-
}
422-
423-
function pinnedPortBindable(port: number): string {
424-
try { Bun.serve({ hostname: "127.0.0.1", port, fetch: () => new Response(null) }).stop(true); return "free after kill"; }
425-
catch { return "still bound after kill"; }
426-
}
427-
428-
/**
429-
* Spawn `ocx start` and wait for its runtime record and settled readiness.
430-
*
431-
* `port` pins the listener with `--port`. A pinned port comes from freeLoopbackPort(), which
432-
* releases its probe before the child binds, so another process on the runner can take it in
433-
* between; a hard-pinned start then waits out port reclaim (up to 60 s) before it binds and
434-
* publishes, which outlasts OWNER_WAIT_MS with the child alive and no record. Pass `undefined`
435-
* when the case does not need a predetermined port: the child keeps config.port and binds its
436-
* own, and the runtime record supplies the port it chose.
404+
* Drain both child pipes immediately and await the CLI's post-sync startup signal or exit.
405+
* Record publication and /readyz are then assertions, with no independent wall-clock deadline.
406+
* The existing test budget remains the watchdog for a child that never signals or exits.
407+
* A supplied port still pins --port; undefined uses the configured port.
437408
*/
438409
async function startSibling(
439410
env: Record<string, string>,
440411
siblingHome: string,
441412
port: number | undefined,
442413
cwd: string,
443-
): Promise<{ child: ReturnType<typeof Bun.spawn>; runtime: { pid: number; port: number; siblingOfPort?: number } }> {
414+
) {
444415
const child = Bun.spawn([process.execPath, cliPath, "start", ...(port === undefined ? [] : ["--port", String(port)])], {
445416
cwd,
446417
env,
447418
stdout: "pipe",
448419
stderr: "pipe",
449420
});
450421
children.push(child);
422+
const output = captureStartupChildOutput(child.stdout, child.stderr);
423+
startupOutputs.push(output);
424+
await output.waitForStartup(child);
451425
const runtimePath = join(siblingHome, "runtime-port.json");
452-
// Allowlisted fields only: the record also carries the per-process attestationSecret.
453-
let lastRecord: string | null = null;
454-
let exitedEarly = false;
455426
let runtime: { pid: number; port: number; siblingOfPort?: number };
456427
try {
457-
runtime = await waitFor(async () => {
458-
if (child.exitCode !== null) {
459-
exitedEarly = true;
460-
throw new Error(`sibling exited ${child.exitCode}: ${await new Response(child.stderr).text()}`);
461-
}
462-
if (!existsSync(runtimePath)) return null;
463-
try {
464-
const value = JSON.parse(readFileSync(runtimePath, "utf8")) as { pid: number; port: number; hostname?: string; siblingOfPort?: number };
465-
lastRecord = JSON.stringify({ pid: value.pid, port: value.port, hostname: value.hostname, siblingOfPort: value.siblingOfPort });
466-
const portOk = Number.isInteger(value.port) && value.port >= 1 && value.port <= 65535
467-
&& (port === undefined || value.port === port);
468-
return value.pid === child.pid && portOk ? value : null;
469-
} catch {
470-
return null;
471-
}
472-
}, "sibling runtime record");
473-
} catch (error) {
474-
// An early exit already reported its stderr and consumed the stream.
475-
if (exitedEarly) throw error;
476-
// Report what the child was doing, which the bare deadline message cannot.
477-
const sockets = port === undefined ? "" : pinnedPortSockets(port, child.pid);
478-
if (child.exitCode === null) child.kill("SIGKILL");
479-
// Bounded like tail() below: the diagnostic must never outlast the original failure.
480-
const exited = port === undefined
481-
|| await Promise.race([child.exited.then(() => true), Bun.sleep(5_000).then(() => false)]);
482-
// A bind probe while the child may still be alive would only find the child itself.
483-
const holders = port === undefined ? "not pinned"
484-
: `${exited ? pinnedPortBindable(port) : "child alive 5s after kill; bind probe skipped"}${sockets}`;
485-
const tail = async (stream: unknown): Promise<string> => {
486-
try {
487-
return await Promise.race([
488-
new Response(stream as ReadableStream<Uint8Array>).text().then(text => text.slice(-4000)),
489-
Bun.sleep(2000).then(() => "<no EOF within 2s>"),
490-
]);
491-
} catch {
492-
return "<unreadable>";
493-
}
494-
};
495-
const [stdout, stderr] = await Promise.all([tail(child.stdout), tail(child.stderr)]);
496-
throw new Error(
497-
`${(error as Error).message} (requested port ${port ?? "config"}, pid ${child.pid}, last record ${lastRecord ?? "none"}, port holders: ${holders})\n`
498-
+ `--- stdout ---\n${stdout}\n--- stderr ---\n${stderr}`,
499-
);
428+
const record = JSON.parse(readFileSync(runtimePath, "utf8"));
429+
// Allowlisted fields only: never put the runtime's attestationSecret in assertion output.
430+
runtime = { pid: record.pid, port: record.port, siblingOfPort: record.siblingOfPort };
431+
} catch {
432+
throw new Error(`sibling signalled startup without a readable runtime record: ${JSON.stringify(output.snapshot())}`);
500433
}
501-
const readyPort = runtime.port;
502-
// /readyz settles after the startup sync. Without the sibling gate that sync is exactly
503-
// what rewrote the shared config.toml, so the bytes are compared only once it has settled,
504-
// and it has to settle as "ready": a sibling that wrote nothing has nothing to fail.
505-
let settled: string | undefined;
506-
await waitFor(async () => {
507-
try {
508-
const response = await fetch(`http://127.0.0.1:${readyPort}/readyz`, { signal: AbortSignal.timeout(500) });
509-
const body = await response.json() as { status?: string; pid?: number };
510-
if (body.pid !== child.pid || body.status === undefined || body.status === "pending") return null;
511-
settled = body.status;
512-
return true;
513-
} catch {
514-
return null;
515-
}
516-
}, "sibling readiness");
517-
expect(settled).toBe("ready");
518-
return { child, runtime };
434+
expect(runtime.pid).toBe(child.pid);
435+
expect(Number.isInteger(runtime.port) && runtime.port >= 1 && runtime.port <= 65535).toBe(true);
436+
if (port !== undefined) expect(runtime.port).toBe(port);
437+
// Keep the independent identity/readiness proof before comparing shared routing bytes.
438+
const response = await fetch(`http://127.0.0.1:${runtime.port}/readyz`);
439+
const ready = await response.json() as { status?: string; pid?: number };
440+
expect(response.ok).toBe(true);
441+
expect(ready.pid).toBe(child.pid);
442+
expect(ready.status).toBe("ready");
443+
return { child, runtime, output };
519444
}
520445

521446
/**
@@ -594,7 +519,7 @@ describe("a sibling instance leaves the live owner's client routing alone", () =
594519
expect(stop.exitCode, stop.stderr).toBe(0);
595520
expect(stop.stdout).toContain(`Client routing stays on the proxy at port ${ownerRuntime.port}`);
596521
await sibling.child.exited;
597-
expect(await new Response(sibling.child.stdout).text()).toContain(
522+
expect((await sibling.output.finish()).stdout.tail).toContain(
598523
`Client routing stays on the proxy at port ${ownerRuntime.port}; this instance serves direct requests on port ${siblingPort} only.`,
599524
);
600525
expect(snapshot()).toEqual(before);
@@ -720,7 +645,7 @@ describe("a sibling instance leaves the live owner's client routing alone", () =
720645
const stop = await runCli({ ...fx, env: { ...fx.env, OPENCODEX_HOME: siblingHome } }, ["stop"]);
721646
expect(stop.exitCode, stop.stderr).toBe(0);
722647
await replacement.child.exited;
723-
expect(await new Response(replacement.child.stdout).text()).toContain(
648+
expect((await replacement.output.finish()).stdout.tail).toContain(
724649
`Client routing stays on the proxy at port ${ownerPort}; this instance serves direct requests on port ${siblingPort} only.`,
725650
);
726651
expect(snapshot()).toEqual(before);

‎tests/cli/sibling-home-client-sync.test.ts‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { afterEach, expect, test } from "bun:test";
1+
import { afterEach, expect, spyOn, test } from "bun:test";
22
import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import { join } from "node:path";
@@ -760,6 +760,28 @@ test("a registry-only discovered owner vetoes shared writes end to end", async (
760760
expect([grokPath, codexPath, claudePath].map(path => readFileSync(path))).toEqual(before);
761761
}, 30_000);
762762

763+
test("startup readiness follows the child signal across a wall-clock jump", async () => {
764+
let stdout!: ReadableStreamDefaultController<Uint8Array>;
765+
const out = new ReadableStream<Uint8Array>({ start(controller) { stdout = controller; } });
766+
const err = new ReadableStream<Uint8Array>({ start(controller) { controller.close(); } });
767+
const capture = captureStartupChildOutput(out, err);
768+
const child = { pid: 123, exitCode: null, exited: new Promise<number>(() => {}) };
769+
const startedAt = Date.now();
770+
const clock = spyOn(Date, "now").mockReturnValue(startedAt);
771+
try {
772+
const ready = capture.waitForStartup(child);
773+
clock.mockReturnValue(startedAt + 90_000);
774+
await Promise.resolve();
775+
stdout.enqueue(new TextEncoder().encode("Client startup work complete.\n"));
776+
stdout.close();
777+
await ready;
778+
expect(capture.snapshot().startupComplete).toBe(true);
779+
} finally {
780+
clock.mockRestore();
781+
await capture.finish();
782+
}
783+
});
784+
763785
test("startup diagnostics drain both pipes before runtime publication", async () => {
764786
const fx = fixture();
765787
const runtimePath = join(fx.ocx, "runtime-port.json");
@@ -806,8 +828,25 @@ test("startup diagnostics include both child tails and exit state on publication
806828
expect(failure).toContain('"exitCode":2');
807829
expect(failure).toContain(`"pid":${child.pid}`);
808830
expect(failure).toContain("lastOutputAgoMs");
831+
await expect(capturedChildren.get(child)!.waitForStartup(child)).rejects.toThrow('"exitCode":2');
809832
});
810833

834+
for (const failure of ["stdout EOF", "stdout read failure"] as const) {
835+
test(`startup readiness rejects ${failure} without a completion signal`, async () => {
836+
const stdout = new ReadableStream<Uint8Array>({ start(controller) {
837+
controller.enqueue(new TextEncoder().encode("before-startup"));
838+
if (failure === "stdout EOF") controller.close();
839+
else controller.error(new Error("fixture pipe failure"));
840+
} });
841+
const stderr = new ReadableStream<Uint8Array>({ start(controller) { controller.close(); } });
842+
const output = captureStartupChildOutput(stdout, stderr);
843+
const child = { pid: 123, exitCode: null, exited: new Promise<number>(() => {}) };
844+
await expect(output.waitForStartup(child)).rejects.toThrow("child ended before live client startup");
845+
expect(output.snapshot().startupComplete).toBe(false);
846+
expect(output.snapshot().stdout.readFailed).toBe(failure === "stdout read failure");
847+
});
848+
}
849+
811850
test("startup diagnostics preserve split UTF-8 and startup markers before tail eviction", async () => {
812851
const bytes = new TextEncoder().encode("한글 Client startup work complete.");
813852
const stdout = new ReadableStream<Uint8Array>({
@@ -823,7 +862,9 @@ test("startup diagnostics preserve split UTF-8 and startup markers before tail e
823862
controller.close();
824863
},
825864
});
826-
const output = await captureStartupChildOutput(stdout, stderr).finish();
865+
const capture = captureStartupChildOutput(stdout, stderr);
866+
await capture.waitForStartup({ pid: 123, exitCode: null, exited: new Promise<number>(() => {}) });
867+
const output = await capture.finish();
827868
expect(output.startupComplete).toBe(true);
828869
expect(output.stdout.tail).toBe("z".repeat(8192));
829870
expect(output.stderr.tail).toBe("오류");

‎tests/helpers/startup-child-output.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ export function captureStartupChildOutput(stdout: ReadableStream<Uint8Array>, st
66
const startedAt = Date.now();
77
let lastOutputAt: number | null = null;
88
let startupComplete = false;
9+
let signalStartup!: (complete: boolean) => void;
10+
const startup = new Promise<boolean>(resolve => { signalStartup = resolve; });
911
const collect = (stream: ReadableStream<Uint8Array>, isStdout: boolean) => {
1012
const reader = stream.getReader();
1113
const decoder = new TextDecoder();
@@ -20,6 +22,7 @@ export function captureStartupChildOutput(stdout: ReadableStream<Uint8Array>, st
2022
if (isStdout && !startupComplete) {
2123
const search = markerPrefix + text;
2224
startupComplete = search.includes(STARTUP_MARKER);
25+
if (startupComplete) signalStartup(true);
2326
markerPrefix = search.slice(-(STARTUP_MARKER.length - 1));
2427
}
2528
tail = (tail + text).slice(-STARTUP_OUTPUT_TAIL_CHARS);
@@ -33,7 +36,10 @@ export function captureStartupChildOutput(stdout: ReadableStream<Uint8Array>, st
3336
append(decoder.decode(next.value, { stream: true }));
3437
}
3538
} catch { readFailed = true; }
36-
finally { reader.releaseLock(); }
39+
finally {
40+
if (isStdout) signalStartup(false);
41+
reader.releaseLock();
42+
}
3743
})();
3844
return {
3945
done,
@@ -55,6 +61,13 @@ export function captureStartupChildOutput(stdout: ReadableStream<Uint8Array>, st
5561
const stop = () => { out.stop(); err.stop(); };
5662
return {
5763
snapshot, stop,
64+
async waitForStartup(child: { pid: number; exitCode: number | null; exited: Promise<number> }) {
65+
// The test runner owns the hang watchdog. Startup is a child event, not a latency claim.
66+
const complete = await Promise.race([startup, child.exited.then(() => false)]);
67+
if (complete && child.exitCode === null) return;
68+
await this.finish();
69+
throw new Error(`child ended before live client startup: ${JSON.stringify({ pid: child.pid, exitCode: child.exitCode, ...snapshot() })}`);
70+
},
5871
async finish(timeoutMs = 1000) {
5972
let timer: ReturnType<typeof setTimeout> | undefined;
6073
try {

0 commit comments

Comments
 (0)