Skip to content

Commit 8970a32

Browse files
fix(scripts): bound check:changed concurrency and step time (#27227)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 6d12bb1 commit 8970a32

3 files changed

Lines changed: 677 additions & 337 deletions

File tree

‎docs/operations/development.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,12 @@ files to packages with [ci-changes.cjs](../../.github/scripts/ci-changes.cjs), a
170170
compares against the merge base with `origin/main`, or with a ref you pass. CI still runs the full
171171
test suites and the jobs the summary lists, so a local pass does not promise a green CI run.
172172

173-
The steps run side by side, so a run takes about as long as its slowest step, usually knip's
174-
exports pass at about 10 s. When the files are exactly those of the last passing run, it prints that
173+
The steps run side by side, as many at once as the machine allows: one per two cores and per 6 GiB
174+
of available memory. On a large machine a run takes about as long as its slowest step, usually
175+
knip's exports pass at about 10 s. Pass `--concurrency <n>` or set `CHECK_CHANGED_CONCURRENCY` to
176+
change the limit (`1` runs the steps one by one). A step that runs past `--timeout <seconds>` or
177+
`CHECK_CHANGED_TIMEOUT` (default 900) is killed with every process it started and reported as
178+
`TIMEOUT`, not as a failing check. When the files are exactly those of the last passing run, it prints that
175179
run's summary instead of running again. Any change to a file, the install, or a generated file runs
176180
the checks again.
177181

‎scripts/check-changed.test.ts‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
import { assert, describe, it } from "@effect/vitest";
2+
import * as Clock from "effect/Clock";
3+
import * as Effect from "effect/Effect";
4+
import * as Fiber from "effect/Fiber";
5+
import * as Sink from "effect/Sink";
6+
import * as Stream from "effect/Stream";
7+
import { ChildProcessSpawner } from "effect/process";
8+
import * as TestClock from "effect/testing/TestClock";
9+
10+
import { defaultConcurrency, gitLines, runSteps, type Step } from "./check-changed.ts";
11+
12+
const GiB = 1024 ** 3;
13+
14+
describe("check:changed", () => {
15+
it("bounds the default concurrency by the scarcer of cores and available memory", () => {
16+
assert.strictEqual(defaultConcurrency(32, 128 * GiB), 16);
17+
assert.strictEqual(defaultConcurrency(32, 20 * GiB), 3);
18+
assert.strictEqual(defaultConcurrency(4, 15 * GiB), 2);
19+
assert.strictEqual(defaultConcurrency(4, 2 * GiB), 1);
20+
assert.strictEqual(defaultConcurrency(1, 64 * GiB), 1);
21+
});
22+
23+
it.effect(
24+
"keeps weighted steps within the budget, in order, and serial steps one at a time",
25+
() =>
26+
Effect.gen(function* () {
27+
const startedAt: Record<string, number> = {};
28+
let active = 0;
29+
let maxActive = 0;
30+
let serialActive = 0;
31+
let maxSerial = 0;
32+
const step = (
33+
name: string,
34+
options: { weight?: number; serial?: boolean } = {},
35+
): Step<never> => {
36+
const slots = Math.min(options.weight ?? 1, 3);
37+
return {
38+
name,
39+
...options,
40+
run: Effect.gen(function* () {
41+
startedAt[name] = (yield* Clock.currentTimeMillis) / 1000;
42+
active += slots;
43+
maxActive = Math.max(maxActive, active);
44+
if (options.serial) maxSerial = Math.max(maxSerial, ++serialActive);
45+
yield* Effect.sleep("1 second");
46+
active -= slots;
47+
if (options.serial) serialActive--;
48+
return { ok: true, output: "" };
49+
}),
50+
};
51+
};
52+
const fiber = yield* runSteps(
53+
[
54+
step("light 1"),
55+
// Above the budget, so it holds every slot rather than never starting.
56+
step("heavy", { weight: 5 }),
57+
step("pair", { weight: 2 }),
58+
step("light 2"),
59+
step("test 1", { serial: true }),
60+
step("test 2", { serial: true }),
61+
],
62+
{ concurrency: 3, timeoutSeconds: 60 },
63+
).pipe(Effect.forkChild);
64+
yield* TestClock.adjust("10 seconds");
65+
const results = yield* Fiber.join(fiber);
66+
67+
assert.deepStrictEqual(
68+
results.map((result) => result.outcome),
69+
["pass", "pass", "pass", "pass", "pass", "pass"],
70+
);
71+
assert.strictEqual(maxActive, 3);
72+
// Lighter steps queued behind the heavy one never start ahead of it.
73+
assert.deepStrictEqual(startedAt, {
74+
"light 1": 0,
75+
heavy: 1,
76+
pair: 2,
77+
"light 2": 2,
78+
"test 1": 3,
79+
"test 2": 4,
80+
});
81+
assert.strictEqual(maxSerial, 1);
82+
}),
83+
);
84+
85+
it.effect("interrupts a step at its timeout and reports TIMEOUT, apart from FAIL", () =>
86+
Effect.gen(function* () {
87+
let interrupted = false;
88+
const fiber = yield* runSteps(
89+
[
90+
{
91+
name: "hangs",
92+
run: Effect.never.pipe(
93+
Effect.onInterrupt(() => Effect.sync(() => (interrupted = true))),
94+
),
95+
},
96+
{ name: "fails", run: Effect.succeed({ ok: false, output: "error" }) },
97+
{ name: "passes", run: Effect.succeed({ ok: true, output: "" }) },
98+
],
99+
{ concurrency: 3, timeoutSeconds: 10 },
100+
).pipe(Effect.forkChild);
101+
yield* TestClock.adjust("9 seconds");
102+
assert.isFalse(interrupted);
103+
yield* TestClock.adjust("1 second");
104+
const results = yield* Fiber.join(fiber);
105+
106+
assert.isTrue(interrupted);
107+
assert.deepStrictEqual(
108+
results.map(({ name, outcome, time }) => [name, outcome, time]),
109+
[
110+
["hangs", "TIMEOUT", "killed after 10s"],
111+
["fails", "FAIL", "0.0s"],
112+
["passes", "pass", "0.0s"],
113+
],
114+
);
115+
}),
116+
);
117+
118+
it.effect("fails instead of returning an empty file list when git exits nonzero", () => {
119+
const spawner = (exitCode: number) =>
120+
ChildProcessSpawner.make(() =>
121+
Effect.succeed(
122+
ChildProcessSpawner.makeHandle({
123+
pid: ChildProcessSpawner.ProcessId(1),
124+
exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)),
125+
isRunning: Effect.succeed(false),
126+
kill: () => Effect.void,
127+
unref: Effect.succeed(Effect.void),
128+
stdin: Sink.drain,
129+
stdout: Stream.encodeText(Stream.make(exitCode === 0 ? "a.ts\nb.ts\n" : "")),
130+
stderr: Stream.encodeText(
131+
Stream.make(exitCode === 0 ? "" : "fatal: index file corrupt"),
132+
),
133+
all: Stream.empty,
134+
getInputFd: () => Sink.drain,
135+
getOutputFd: () => Stream.empty,
136+
}),
137+
),
138+
);
139+
return Effect.gen(function* () {
140+
const listed = yield* gitLines("/repo", ["ls-files"]).pipe(
141+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner(0)),
142+
);
143+
assert.deepStrictEqual(listed, ["a.ts", "b.ts"]);
144+
const error = yield* gitLines("/repo", ["ls-files"]).pipe(
145+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner(128)),
146+
Effect.flip,
147+
);
148+
assert.strictEqual(
149+
error,
150+
"check:changed: git ls-files exited with 128: fatal: index file corrupt",
151+
);
152+
});
153+
});
154+
});

0 commit comments

Comments
 (0)