Skip to content

Commit 5211a84

Browse files
axisrowclaude
andcommitted
test: scale broker/app-server spawn timeouts under CPU contention (#53)
Concurrent `npm test` runs made unrelated tests fail at almost exactly 5000ms: the broker-smoke IPC-ready wait and the withAppServer fallback watchdogs budget real process spawns, so under contention they lost the race against the machine rather than catching a logic defect. Add scaleTimeout() in tests/helpers.mjs and apply it to those budgets. The multiplier is a flat base rather than an os.loadavg() derivation: loadavg is a 1-minute average that still reads near-idle during the first seconds of a burst, which is exactly when these deadlines are computed — a load-derived multiplier measured 1x while the work needed 4x, and the suite still failed at a flat 5000ms. loadavg is kept only as a floor for the case where it has caught up. Handshake deadlines that are themselves under test (300/500ms) stay unscaled. Also clear the losing watchdog timer via a raceHang() helper. A bare setTimeout in a Promise.race keeps the event loop alive for the full budget even when the real promise wins; scaling the budget exposed this, inflating the file from 6.7s to 31.5s wall-clock. Clearing it brings the two files to 2.98s, faster than before this change. Verified on a 10-core box: 10 concurrent runs of the two files went 10/10 red before, 10/10 green after. Beyond ~10 (1x cores) the failures change character — they hit the production SPAWNED_INITIALIZE_TIMEOUT_MS (10s) in app-server.mjs, which is a genuine signal and deliberately left alone. Full suite 194/194 green; tsc build clean. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_0185ypzB1bqu9CLxTVyWdrzx
1 parent e470027 commit 5211a84

3 files changed

Lines changed: 96 additions & 13 deletions

File tree

‎tests/broker-client-initialize-timeout.test.mjs‎

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,31 @@ import assert from "node:assert/strict";
44
import { CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs";
55
import { withAppServer } from "../plugins/codex/scripts/lib/codex.mjs";
66
import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs";
7-
import { makeTempDir } from "./helpers.mjs";
7+
import { makeTempDir, scaleTimeout } from "./helpers.mjs";
88
import { spawnBusyBroker, spawnWedgedServer } from "./broker-controller-helpers.mjs";
99

10+
/**
11+
* Race `promise` against a watchdog that resolves to `hangValue`.
12+
*
13+
* The watchdog timer is cleared once the race settles. A bare
14+
* `setTimeout` in a Promise.race keeps the event loop alive for the full
15+
* budget even when the real promise won, which held the test process open
16+
* long after the assertions passed.
17+
*/
18+
async function raceHang(promise, hangValue, budgetMs) {
19+
let timer;
20+
try {
21+
return await Promise.race([
22+
promise,
23+
new Promise((resolve) => {
24+
timer = setTimeout(() => resolve(hangValue), scaleTimeout(budgetMs));
25+
})
26+
]);
27+
} finally {
28+
clearTimeout(timer);
29+
}
30+
}
31+
1032
// Reproduces fork #29 / upstream openai#509: when the shared broker is wedged
1133
// (busy with a long-running request and never reaching the second socket), the
1234
// client must not hang forever on connect + initialize. This is the RED test —
@@ -17,6 +39,10 @@ test("client does not hang on initialize when broker is wedged (#29/#509)", asyn
1739
// Short handshake budgets so the test does not wait on the production
1840
// defaults (2s connect / 5s initialize). The wedged broker connects but
1941
// never answers initialize, so the initialize deadline must fire.
42+
//
43+
// These two stay unscaled on purpose: they are the deadlines under test, and
44+
// the assertion below is that the client honors them. Only the HANG watchdog
45+
// racing them is scaled, since that one budgets real connect work.
2046
const connectP = CodexAppServerClient.connect(cwd, {
2147
brokerEndpoint: `unix:${wedged.socketPath}`,
2248
brokerConnectTimeoutMs: 300,
@@ -25,14 +51,14 @@ test("client does not hang on initialize when broker is wedged (#29/#509)", asyn
2551

2652
let outcome = "HANG";
2753
try {
28-
const winner = await Promise.race([
54+
outcome = await raceHang(
2955
connectP.then(
3056
() => "connected",
3157
(error) => ({ errored: error.code ?? error.rpcCode ?? error.message })
3258
),
33-
new Promise((resolve) => setTimeout(() => resolve("HANG"), 2000))
34-
]);
35-
outcome = winner;
59+
"HANG",
60+
2000
61+
);
3662
} finally {
3763
wedged.cleanup();
3864
}
@@ -58,7 +84,7 @@ test("withAppServer falls back to a direct app-server when the broker wedges (#2
5884

5985
let calls = 0;
6086
try {
61-
const result = await Promise.race([
87+
const result = await raceHang(
6288
withAppServer(
6389
cwd,
6490
async (client) => {
@@ -72,8 +98,9 @@ test("withAppServer falls back to a direct app-server when the broker wedges (#2
7298
env
7399
}
74100
),
75-
new Promise((resolve) => setTimeout(() => resolve({ HANG: true }), 5000))
76-
]);
101+
{ HANG: true },
102+
5000
103+
);
77104

78105
assert.ok(!result.HANG, "withAppServer hung instead of falling back to direct");
79106
assert.equal(calls, 1, "the operation must run exactly once via the direct fallback");
@@ -97,7 +124,7 @@ test("withAppServer falls back to a direct app-server on broker/busy after a suc
97124

98125
let calls = 0;
99126
try {
100-
const result = await Promise.race([
127+
const result = await raceHang(
101128
withAppServer(
102129
cwd,
103130
async (client) => {
@@ -113,8 +140,9 @@ test("withAppServer falls back to a direct app-server on broker/busy after a suc
113140
env
114141
}
115142
),
116-
new Promise((resolve) => setTimeout(() => resolve({ HANG: true }), 5000))
117-
]);
143+
{ HANG: true },
144+
5000
145+
);
118146

119147
assert.ok(!result.HANG, "withAppServer hung instead of falling back to direct on broker/busy");
120148
// The broker attempt hits -32001, withAppServer retries on direct — so fn

‎tests/broker-smoke.test.mjs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { fork } from "node:child_process";
77
import { fileURLToPath } from "node:url";
88

99
import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs";
10-
import { makeTempDir } from "./helpers.mjs";
10+
import { makeTempDir, scaleTimeout } from "./helpers.mjs";
1111

1212
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
1313
const BROKER_SCRIPT = path.join(ROOT, "plugins", "codex", "scripts", "app-server-broker.mjs");
@@ -29,9 +29,12 @@ function forkBroker({ argv, env, cwd }) {
2929
});
3030
}
3131

32+
// The budget covers a real broker fork plus a real fake-codex app-server spawn,
33+
// so it is scaled: on a loaded box those spawns are what blow past a flat 5s.
3234
function onceMessage(child, type, timeoutMs = 5000) {
35+
const budgetMs = scaleTimeout(timeoutMs);
3336
return new Promise((resolve, reject) => {
34-
const timer = setTimeout(() => reject(new Error(`Timed out waiting for IPC ${type}.`)), timeoutMs);
37+
const timer = setTimeout(() => reject(new Error(`Timed out waiting for IPC ${type} after ${budgetMs}ms.`)), budgetMs);
3538
const onMessage = (msg) => {
3639
if (msg?.type === type) {
3740
clearTimeout(timer);

‎tests/helpers.mjs‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,58 @@ export function makeTempDir(prefix = "codex-plugin-test-") {
2727
// ephemeral value for spawned subprocesses; this covers in-process reads.
2828
delete process.env.CLAUDE_PLUGIN_DATA;
2929

30+
// Timeouts guarding *real* work (spawning brokers and app-servers) are races
31+
// against the machine, not assertions about logic. Under CPU contention — several
32+
// `npm test` runs at once, or a busy CI box — a hard 5s budget loses to work that
33+
// takes ~0.5s when idle, and unrelated tests fail at almost exactly the deadline
34+
// (fork #53). Scale those budgets so the tests keep asserting behavior instead of
35+
// asserting machine speed.
36+
//
37+
// Deliberately NOT derived from os.loadavg(): that is a 1-minute average, so it
38+
// still reads near-idle during the first seconds of a burst — exactly when these
39+
// deadlines are computed. Measured on a 10-core box, ten concurrent test
40+
// processes produced a load-derived multiplier of 1 while the work needed ~4x,
41+
// and the suite failed at a flat 5000ms. The base multiplier below is therefore
42+
// unconditional: a too-generous budget costs nothing on a green run (the wait
43+
// resolves the moment the event arrives), while a too-tight one costs a false
44+
// failure. CODEX_TEST_TIMEOUT_SCALE overrides it — set it to 1 when debugging a
45+
// genuine hang and you want the short deadline back.
46+
const TIMEOUT_SCALE = resolveTimeoutScale();
47+
48+
function resolveTimeoutScale() {
49+
const override = Number(process.env.CODEX_TEST_TIMEOUT_SCALE);
50+
if (Number.isFinite(override) && override > 0) return override;
51+
52+
// Absorbs the common case: a developer running the suite next to an editor,
53+
// a build, or a second `npm test`. Chosen so the observed worst case under
54+
// 10-way contention (~4x the idle spawn time) still fits.
55+
const BASE = 6;
56+
57+
// loadavg lags a burst, but when it *has* caught up — a CI box already busy
58+
// when the suite starts, or a long-running run — it is a true signal, so take
59+
// it as a floor on top of BASE rather than as the whole answer. Capped so a
60+
// pathological load cannot turn a real hang into a multi-minute wait.
61+
const cpus = Math.max(1, os.availableParallelism?.() ?? os.cpus().length);
62+
const load = os.loadavg()[0];
63+
if (!Number.isFinite(load) || load <= 0) return BASE;
64+
return Math.min(12, Math.max(BASE, Math.ceil((load / cpus) * 2)));
65+
}
66+
67+
/**
68+
* Scale a timeout that budgets real work (process spawn, IPC handshake, socket
69+
* connect) so it survives CPU contention.
70+
*
71+
* Do NOT use this for deadlines that are themselves under test — a production
72+
* timeout a test asserts on must stay exact, or the assertion stops meaning
73+
* anything.
74+
*
75+
* @param {number} ms Timeout budget measured on an idle machine.
76+
* @returns {number} The budget adjusted for contention.
77+
*/
78+
export function scaleTimeout(ms) {
79+
return Math.round(ms * TIMEOUT_SCALE);
80+
}
81+
3082
export function writeExecutable(filePath, source) {
3183
fs.writeFileSync(filePath, source, { encoding: "utf8", mode: 0o755 });
3284
}

0 commit comments

Comments
 (0)