Skip to content

Commit 357639b

Browse files
fix(scripts): bound check:changed concurrency and step time
check:changed started every step at once, so on a machine with few cores, little memory, or other work running, typecheck, knip and tests competed until the host stalled. Steps now run through a limiter sized from the core count and available memory (CHECK_CHANGED_CONCURRENCY overrides), typecheck uses the same budget, and a step running past CHECK_CHANGED_TIMEOUT seconds (default 900) is killed with its process tree and reported as TIMEOUT. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent dd14848 commit 357639b

4 files changed

Lines changed: 245 additions & 29 deletions

File tree

‎docs/operations/development.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,11 @@ files to packages with [ci-changes.cjs](../../.github/scripts/ci-changes.cjs), a
170170
compares against the merge base with `origin/main`, or with a ref you pass. CI still runs the full
171171
test suites and the jobs the summary lists, so a local pass does not promise a green CI run.
172172

173-
The steps run side by side, so a run takes about as long as its slowest step, usually knip's
174-
exports pass at about 10 s. When the files are exactly those of the last passing run, it prints that
173+
The steps run side by side, as many at once as the machine allows: one per two cores and per 6 GiB
174+
of available memory. On a large machine a run takes about as long as its slowest step, usually
175+
knip's exports pass at about 10 s. Set `CHECK_CHANGED_CONCURRENCY` to change the limit (`1` runs
176+
the steps one by one). A step that runs past `CHECK_CHANGED_TIMEOUT` seconds (default 900) is
177+
killed with every process it started and reported as `TIMEOUT`, not as a failing check. When the files are exactly those of the last passing run, it prints that
175178
run's summary instead of running again. Any change to a file, the install, or a generated file runs
176179
the checks again.
177180

‎scripts/check-changed.ts‎

Lines changed: 81 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,25 @@
1212
//
1313
// Uncommitted and untracked files count. Affected packages come from
1414
// .github/scripts/ci-changes.cjs, the same mapping CI uses to gate its jobs.
15-
// The steps run side by side; each prints its output when it finishes. A
16-
// rerun on exactly the files of the last passing run prints that run's summary.
15+
// The steps run side by side, as many at once as the machine's cores and
16+
// available memory allow (CHECK_CHANGED_CONCURRENCY overrides; 1 runs them one
17+
// by one), and each prints its output when it finishes. A step running past
18+
// CHECK_CHANGED_TIMEOUT seconds (default 900) is killed with its process tree
19+
// and reported as TIMEOUT. A rerun on exactly the files of the last passing run
20+
// prints that run's summary.
1721
import * as NodeChildProcess from "node:child_process";
1822
import * as NodeCrypto from "node:crypto";
1923
import * as NodeFS from "node:fs";
2024
import * as NodeOS from "node:os";
2125
import * as NodePath from "node:path";
2226
import { decide, readWorkspace } from "../.github/scripts/ci-changes.cjs";
27+
import {
28+
awaitStep,
29+
createLimiter,
30+
defaultConcurrency,
31+
positiveInteger,
32+
type StepOutcome,
33+
} from "./lib/check-changed-steps.ts";
2334

2435
const root = NodePath.resolve(import.meta.dirname, "..");
2536
const CODE = /\.[cm]?[jt]sx?$/;
@@ -245,7 +256,7 @@ const checkPatches = (): boolean => {
245256
const { status, stdout, stderr, error } = NodeChildProcess.spawnSync(
246257
"vp",
247258
["install", "--lockfile-only", "--ignore-scripts"],
248-
{ cwd: temp, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 },
259+
{ cwd: temp, encoding: "utf8", maxBuffer: 64 * 1024 * 1024, timeout: timeoutSeconds * 1000 },
249260
);
250261
if (status === 0) return true;
251262
if (stdout) console.log(stdout);
@@ -270,6 +281,18 @@ const graphql = ["apps/server", "packages/source-control-github"].filter((dir) =
270281
files.some((file) => file.startsWith(`${dir}/`)),
271282
);
272283

284+
// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone script has no Effect runtime.
285+
const platform = process.platform;
286+
// The typecheck step gets the same budget for its own package runs. macOS
287+
// counts its file cache as used, so freemem() there would starve a large Mac.
288+
const concurrency =
289+
positiveInteger(process.env.CHECK_CHANGED_CONCURRENCY) ??
290+
defaultConcurrency(
291+
NodeOS.availableParallelism(),
292+
platform === "darwin" ? NodeOS.totalmem() : NodeOS.freemem(),
293+
);
294+
const timeoutSeconds = positiveInteger(process.env.CHECK_CHANGED_TIMEOUT) ?? 900;
295+
273296
const filters = (names: ReadonlyArray<string>) => names.flatMap((name) => ["--filter", name]);
274297
const steps = [
275298
// Runs first; it takes about a second.
@@ -293,7 +316,7 @@ const steps = [
293316
"--cache",
294317
"--parallel",
295318
"--concurrency-limit",
296-
"4",
319+
String(concurrency),
297320
...filters(affected),
298321
"typecheck",
299322
],
@@ -308,42 +331,73 @@ const steps = [
308331

309332
// Output is piped, so keep the colors a terminal would get.
310333
const env = process.stdout.isTTY ? { ...process.env, FORCE_COLOR: "1" } : process.env;
311-
const runStep = (step: (typeof steps)[number]) =>
312-
new Promise<{ name: string; ok: boolean; seconds: string }>((resolve) => {
313-
const started = performance.now();
314-
const finish = (ok: boolean, output?: Buffer) => {
315-
const seconds = ((performance.now() - started) / 1000).toFixed(1);
316-
console.log(`\n> vp ${step.name} ${ok ? "pass" : "FAIL"} ${seconds}s`);
317-
if (output) process.stdout.write(output);
318-
resolve({ name: step.name, ok, seconds });
319-
};
320-
// In-process steps print their own output. The patch check is first and
321-
// takes about a second, so the commands start right after it.
322-
if ("run" in step) return finish(step.run());
323-
const output: Array<Buffer> = [];
324-
const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env });
325-
child.stdout.on("data", (chunk: Buffer) => output.push(chunk));
326-
child.stderr.on("data", (chunk: Buffer) => output.push(chunk));
327-
// A failed spawn emits "error", then "close" with a nonzero code.
328-
child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`)));
329-
child.on("close", (status) => finish(status === 0, Buffer.concat(output)));
334+
// Each command leads its own process group so a timeout can kill the workers it
335+
// started. That also hides Ctrl-C from them, so kill them on it. Windows has no
336+
// process groups; taskkill /t walks the tree instead.
337+
const windows = platform === "win32";
338+
const running = new Set<NodeChildProcess.ChildProcess>();
339+
const killTree = (child: NodeChildProcess.ChildProcess) => {
340+
if (child.pid === undefined) return;
341+
try {
342+
if (windows) {
343+
NodeChildProcess.spawnSync("taskkill", ["/pid", String(child.pid), "/t", "/f"]);
344+
} else {
345+
process.kill(-child.pid, "SIGKILL");
346+
}
347+
} catch {
348+
// Already gone.
349+
}
350+
};
351+
for (const signal of ["SIGINT", "SIGTERM"] as const) {
352+
process.on(signal, () => {
353+
for (const child of running) killTree(child);
354+
process.exit(signal === "SIGINT" ? 130 : 143);
330355
});
356+
}
357+
const runStep = async (step: (typeof steps)[number]) => {
358+
const started = performance.now();
359+
const finish = (outcome: StepOutcome, output?: Buffer) => {
360+
const time =
361+
outcome === "TIMEOUT"
362+
? `killed after ${timeoutSeconds}s`
363+
: `${((performance.now() - started) / 1000).toFixed(1)}s`;
364+
console.log(`\n> vp ${step.name} ${outcome} ${time}`);
365+
if (output) process.stdout.write(output);
366+
return { name: step.name, outcome, time };
367+
};
368+
// In-process steps print their own output. The patch check is first and
369+
// takes about a second, so the commands start right after it.
370+
if ("run" in step) return finish(step.run() ? "pass" : "FAIL");
371+
const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env, detached: !windows });
372+
running.add(child);
373+
const { outcome, output } = await awaitStep(child, timeoutSeconds * 1000, () => killTree(child));
374+
running.delete(child);
375+
return finish(outcome, output);
376+
};
377+
378+
const limit = createLimiter(concurrency);
331379

332380
// Vitest runs started together race to rebuild node_modules/.vitest-cache after
333381
// an install (ENOTEMPTY), so they run one after another.
334382
let previousVitest: Promise<unknown> = Promise.resolve();
335383
const results = await Promise.all(
336384
steps.map((step) => {
337-
if (!("args" in step) || step.args[0] !== "test") return runStep(step);
338-
const result = previousVitest.then(() => runStep(step));
385+
if (!("args" in step) || step.args[0] !== "test") return limit(() => runStep(step));
386+
const result = previousVitest.then(() => limit(() => runStep(step)));
339387
previousVitest = result;
340388
return result;
341389
}),
342390
);
343391

344392
const summary = ["check:changed summary"];
345393
for (const result of results) {
346-
summary.push(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
394+
summary.push(` ${result.outcome} ${result.name} ${result.time}`);
395+
}
396+
if (results.some((result) => result.outcome === "TIMEOUT")) {
397+
summary.push(
398+
` TIMEOUT: ran past CHECK_CHANGED_TIMEOUT=${timeoutSeconds}s and was killed; this is not ` +
399+
"a check failure. Raise CHECK_CHANGED_TIMEOUT or lower CHECK_CHANGED_CONCURRENCY.",
400+
);
347401
}
348402
const ciOnly = Object.entries({
349403
"full test suites": outputs.test || outputs.test_web || outputs.test_server,
@@ -358,7 +412,7 @@ if (!runKnip) summary.push(" knip skipped: no code or manifest changed");
358412
console.log(`\n${summary.join("\n")}`);
359413
// exitCode, not exit(): piped stdout is asynchronous on macOS, and exit()
360414
// would cut off a failing step's output.
361-
if (results.some((result) => !result.ok)) {
415+
if (results.some((result) => result.outcome !== "pass")) {
362416
process.exitCode = 1;
363417
} else if (inputKey() !== key) {
364418
console.log(" pass not cached: files changed during the run");
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - drives the scheduler with fake child processes.
2+
import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
3+
import * as NodeEvents from "node:events";
4+
import * as NodeStream from "node:stream";
5+
6+
import {
7+
awaitStep,
8+
createLimiter,
9+
defaultConcurrency,
10+
positiveInteger,
11+
} from "./check-changed-steps.ts";
12+
13+
const GiB = 1024 ** 3;
14+
15+
describe("check:changed concurrency", () => {
16+
it("is bounded by the scarcer of cores and available memory, and at least 1", () => {
17+
expect(defaultConcurrency(32, 128 * GiB)).toBe(16);
18+
expect(defaultConcurrency(32, 20 * GiB)).toBe(3);
19+
expect(defaultConcurrency(4, 15 * GiB)).toBe(2);
20+
expect(defaultConcurrency(4, 2 * GiB)).toBe(1);
21+
expect(defaultConcurrency(1, 64 * GiB)).toBe(1);
22+
});
23+
24+
it("accepts only positive integer overrides", () => {
25+
expect(positiveInteger("3")).toBe(3);
26+
for (const value of [undefined, "", "0", "-1", "1.5", "two"]) {
27+
expect(positiveInteger(value)).toBeUndefined();
28+
}
29+
});
30+
31+
it("never runs more tasks than the limit, starts queued tasks in order, and frees a slot on failure", async () => {
32+
const limit = createLimiter(2);
33+
const gates = Array.from({ length: 4 }, () => Promise.withResolvers<void>());
34+
const started: Array<number> = [];
35+
const results = gates.map((gate, index) =>
36+
limit(async () => {
37+
started.push(index);
38+
await gate.promise;
39+
if (index === 0) throw new Error("step crashed");
40+
return index;
41+
}),
42+
);
43+
await Promise.resolve();
44+
expect(started).toEqual([0, 1]);
45+
46+
gates[0]!.resolve();
47+
await expect(results[0]).rejects.toThrow("step crashed");
48+
expect(started).toEqual([0, 1, 2]);
49+
50+
gates[2]!.resolve();
51+
await expect(results[2]).resolves.toBe(2);
52+
expect(started).toEqual([0, 1, 2, 3]);
53+
54+
gates[1]!.resolve();
55+
gates[3]!.resolve();
56+
await expect(Promise.all(results.slice(1))).resolves.toEqual([1, 2, 3]);
57+
});
58+
});
59+
60+
const fakeChild = () =>
61+
Object.assign(new NodeEvents.EventEmitter(), {
62+
stdout: new NodeStream.PassThrough(),
63+
stderr: new NodeStream.PassThrough(),
64+
});
65+
66+
describe("check:changed step timeout", () => {
67+
beforeEach(() => {
68+
vi.useFakeTimers();
69+
});
70+
afterEach(() => {
71+
vi.useRealTimers();
72+
});
73+
74+
it("reports exit status as pass or FAIL with the step's output", async () => {
75+
for (const [status, outcome] of [
76+
[0, "pass"],
77+
[2, "FAIL"],
78+
] as const) {
79+
const child = fakeChild();
80+
const result = awaitStep(child, 1000, () => {});
81+
child.stdout.emit("data", Buffer.from("out "));
82+
child.stderr.emit("data", Buffer.from("err"));
83+
child.emit("close", status);
84+
await expect(result).resolves.toEqual({ outcome, output: Buffer.from("out err") });
85+
expect(vi.getTimerCount()).toBe(0);
86+
}
87+
});
88+
89+
it("kills the step at the deadline and reports TIMEOUT, not FAIL", async () => {
90+
const child = fakeChild();
91+
const kill = vi.fn(() => child.emit("close", null, "SIGKILL"));
92+
const result = awaitStep(child, 1000, kill);
93+
vi.advanceTimersByTime(999);
94+
expect(kill).not.toHaveBeenCalled();
95+
vi.advanceTimersByTime(1);
96+
expect(kill).toHaveBeenCalledOnce();
97+
await expect(result).resolves.toMatchObject({ outcome: "TIMEOUT" });
98+
});
99+
});

‎scripts/lib/check-changed-steps.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// @effect-diagnostics globalTimers:off - scheduling for the standalone check:changed script.
2+
const GiB = 1024 ** 3;
3+
4+
export const positiveInteger = (value: string | undefined) => {
5+
const number = Number(value);
6+
return value !== undefined && Number.isInteger(number) && number > 0 ? number : undefined;
7+
};
8+
9+
// A typecheck or test step can take several cores and gigabytes of memory, so
10+
// allow one step per two cores and per 6 GiB of available memory.
11+
export const defaultConcurrency = (cores: number, availableBytes: number) =>
12+
Math.max(1, Math.min(Math.floor(cores / 2), Math.floor(availableBytes / (6 * GiB))));
13+
14+
export const createLimiter = (concurrency: number) => {
15+
let active = 0;
16+
const queue: Array<() => void> = [];
17+
return async <A>(task: () => Promise<A>): Promise<A> => {
18+
if (active >= concurrency) await new Promise<void>((resolve) => queue.push(resolve));
19+
active++;
20+
try {
21+
return await task();
22+
} finally {
23+
active--;
24+
queue.shift()?.();
25+
}
26+
};
27+
};
28+
29+
export type StepOutcome = "pass" | "FAIL" | "TIMEOUT";
30+
31+
interface Child {
32+
stdout: NodeJS.ReadableStream | null;
33+
stderr: NodeJS.ReadableStream | null;
34+
on(event: "error", listener: (error: Error) => void): unknown;
35+
on(event: "close", listener: (status: number | null) => void): unknown;
36+
}
37+
38+
// Collects a step's output and settles when it exits. Past timeoutMs it calls
39+
// kill, which must end the whole process tree, and reports TIMEOUT whatever the
40+
// exit status.
41+
export const awaitStep = (child: Child, timeoutMs: number, kill: () => void) =>
42+
new Promise<{ outcome: StepOutcome; output: Buffer }>((resolve) => {
43+
const output: Array<Buffer> = [];
44+
let timedOut = false;
45+
const timer = setTimeout(() => {
46+
timedOut = true;
47+
kill();
48+
}, timeoutMs);
49+
child.stdout?.on("data", (chunk: Buffer) => output.push(chunk));
50+
child.stderr?.on("data", (chunk: Buffer) => output.push(chunk));
51+
// A failed spawn emits "error", then "close" with a nonzero code.
52+
child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`)));
53+
child.on("close", (status) => {
54+
clearTimeout(timer);
55+
resolve({
56+
outcome: timedOut ? "TIMEOUT" : status === 0 ? "pass" : "FAIL",
57+
output: Buffer.concat(output),
58+
});
59+
});
60+
});

0 commit comments

Comments
 (0)