Skip to content

Commit 3588360

Browse files
t3dotggclaude
andcommitted
perf(scripts): check:changed finishes in about 10 s
Steps run side by side, knip's two passes split from knip:check, typecheck runs without dependency order, and a rerun on the last passing tree prints its summary. The knip schema preprocessor shares parsed files across programs, checks only the names its issues mention, and runs in three workers, which also speeds up CI's knip:check. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8d3b061 commit 3588360

3 files changed

Lines changed: 263 additions & 83 deletions

File tree

‎docs/operations/development.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,11 @@ files to packages with [ci-changes.cjs](../../.github/scripts/ci-changes.cjs), a
117117
compares against the merge base with `origin/main`, or with a ref you pass. CI still runs the full
118118
test suites and the jobs the summary lists, so a local pass does not promise a green CI run.
119119

120+
The steps run side by side, so a run takes about as long as its slowest step, usually knip's
121+
exports pass at about 10 s. When the files are exactly those of the last passing run, it prints that
122+
run's summary instead of running again. Any change to a file, the install, or a generated file runs
123+
the checks again.
124+
120125
While you iterate, run single checks:
121126

122127
```sh

‎scripts/check-changed.ts‎

Lines changed: 108 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@
99
//
1010
// Uncommitted and untracked files count. Affected packages come from
1111
// .github/scripts/ci-changes.cjs, the same mapping CI uses to gate its jobs.
12+
// The steps run side by side; each prints its output when it finishes. A
13+
// rerun on exactly the files of the last passing run prints that run's summary.
1214
import * as NodeChildProcess from "node:child_process";
15+
import * as NodeCrypto from "node:crypto";
1316
import * as NodeFS from "node:fs";
1417
import * as NodePath from "node:path";
1518
import { decide, readWorkspace } from "../.github/scripts/ci-changes.cjs";
@@ -44,6 +47,54 @@ if (files.length === 0) {
4447
console.log(`check:changed: no changes since ${base.slice(0, 12)}.`);
4548
process.exit(0);
4649
}
50+
console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`);
51+
52+
// The checked tree is the merge base plus the diff plus the untracked files.
53+
// Ignored files outside node_modules (generated sources, .astro types) and the
54+
// install state can change results too, so they are part of the key.
55+
// tsbuildinfo files are excluded: typecheck rewrites them on every run.
56+
const hash = NodeCrypto.createHash("sha256");
57+
const hashFile = (file: string) => {
58+
hash.update(`\0${file}\0`);
59+
const path = NodePath.join(root, file);
60+
if (NodeFS.existsSync(path)) hash.update(NodeFS.readFileSync(path));
61+
};
62+
hash.update(`${process.version}\0${base}\0`);
63+
hash.update(
64+
NodeChildProcess.execFileSync("git", ["diff", "--binary", "--no-renames", base, "--"], {
65+
cwd: root,
66+
maxBuffer: 1024 * 1024 * 1024,
67+
}),
68+
);
69+
const ignored = git(
70+
"ls-files",
71+
"--others",
72+
"--ignored",
73+
"--exclude-standard",
74+
"--",
75+
":!node_modules/**",
76+
":!**/node_modules/**",
77+
":!.t3/**",
78+
":!**/*.tsbuildinfo",
79+
);
80+
for (const file of [
81+
...untracked,
82+
...ignored,
83+
"node_modules/.modules.yaml",
84+
"node_modules/.pnpm/lock.yaml",
85+
]) {
86+
hashFile(file);
87+
}
88+
const key = hash.digest("hex");
89+
const cachePath = NodePath.join(root, "node_modules/.cache/check-changed.json");
90+
const cached = NodeFS.existsSync(cachePath)
91+
? (JSON.parse(NodeFS.readFileSync(cachePath, "utf8")) as { key: string; summary: Array<string> })
92+
: undefined;
93+
if (cached?.key === key) {
94+
console.log("Nothing changed since the last passing run.\n");
95+
for (const line of cached.summary) console.log(line);
96+
process.exit(0);
97+
}
4798

4899
const present = files.filter((file) => NodeFS.existsSync(NodePath.join(root, file)));
49100
const code = present.filter((file) => CODE.test(file));
@@ -64,12 +115,24 @@ for (const file of code) {
64115
}
65116
}
66117

67-
// knip:check takes about 35 s and cannot be scoped to a package. Any code
68-
// change can leave a file, export, or dependency unused (removing the last
69-
// import of a helper does), so it runs whenever code or a manifest changes.
118+
// knip:check cannot be scoped to a package. Any code change can leave a file,
119+
// export, or dependency unused (removing the last import of a helper does), so
120+
// it runs whenever code or a manifest changes.
70121
const runKnip = files.some(
71122
(file) => CODE.test(file) || NodePath.basename(file) === "package.json" || file === "knip.jsonc",
72123
);
124+
// CI runs knip:check as one script of `knip ... && knip ...` passes. The passes
125+
// do not depend on each other, so they run side by side here.
126+
const knipCheck: string = JSON.parse(
127+
NodeFS.readFileSync(NodePath.join(root, "package.json"), "utf8"),
128+
).scripts["knip:check"];
129+
const knipPasses = knipCheck.split(" && ").map((command, index, all) => {
130+
const [bin, ...args] = command.trim().split(/\s+/);
131+
if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) {
132+
throw new Error(`check:changed cannot split knip:check: ${command}`);
133+
}
134+
return { name: `knip ${index + 1}/${all.length}`, args: ["exec", "knip", ...args] };
135+
});
73136

74137
// CI's Lint job checks that generated GraphQL operations match their sources.
75138
const graphql = ["apps/server", "packages/source-control-github"].filter((dir) =>
@@ -83,14 +146,24 @@ const steps = [
83146
args: ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present],
84147
},
85148
code.length > 0 && { name: "lint", args: ["lint", ...code] },
86-
runKnip && { name: "knip", args: ["run", "knip:check"] },
149+
...(runKnip ? knipPasses : []),
87150
...graphql.map((dir) => ({
88151
name: `graphql (${dir})`,
89152
args: ["run", "--filter", `./${dir}`, "generate", "--check"],
90153
})),
91154
affected.length > 0 && {
92155
name: `typecheck (${affected.length} packages)`,
93-
args: ["run", "--cache", "--concurrency-limit", "4", ...filters(affected), "typecheck"],
156+
// `tsc --noEmit` writes nothing another package reads, so dependency order
157+
// only made dependents wait: --parallel cut a contracts change from 13 s to 9 s.
158+
args: [
159+
"run",
160+
"--cache",
161+
"--parallel",
162+
"--concurrency-limit",
163+
"4",
164+
...filters(affected),
165+
"typecheck",
166+
],
94167
},
95168
// From the repo root, the root vite.config.ts hands each file to its package's config.
96169
tests.size > 0 && {
@@ -99,18 +172,33 @@ const steps = [
99172
},
100173
].filter((step) => step !== false);
101174

102-
console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`);
103-
const results = steps.map((step) => {
104-
console.log(`\n> vp ${step.name}`);
105-
const started = performance.now();
106-
const { status } = NodeChildProcess.spawnSync("vp", step.args, { cwd: root, stdio: "inherit" });
107-
const seconds = ((performance.now() - started) / 1000).toFixed(1);
108-
return { name: step.name, ok: status === 0, seconds };
109-
});
175+
// Output is piped, so keep the colors a terminal would get.
176+
const env = process.stdout.isTTY ? { ...process.env, FORCE_COLOR: "1" } : process.env;
177+
const results = await Promise.all(
178+
steps.map(
179+
(step) =>
180+
new Promise<{ name: string; ok: boolean; seconds: string }>((resolve) => {
181+
const started = performance.now();
182+
const output: Array<Buffer> = [];
183+
const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env });
184+
child.stdout.on("data", (chunk: Buffer) => output.push(chunk));
185+
child.stderr.on("data", (chunk: Buffer) => output.push(chunk));
186+
// A failed spawn emits "error", then "close" with a nonzero code.
187+
child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`)));
188+
child.on("close", (status) => {
189+
const ok = status === 0;
190+
const seconds = ((performance.now() - started) / 1000).toFixed(1);
191+
console.log(`\n> vp ${step.name} ${ok ? "pass" : "FAIL"} ${seconds}s`);
192+
process.stdout.write(Buffer.concat(output));
193+
resolve({ name: step.name, ok, seconds });
194+
});
195+
}),
196+
),
197+
);
110198

111-
console.log("\ncheck:changed summary");
199+
const summary = ["check:changed summary"];
112200
for (const result of results) {
113-
console.log(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
201+
summary.push(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
114202
}
115203
const ciOnly = Object.entries({
116204
"full test suites": outputs.test || outputs.test_web || outputs.test_server,
@@ -119,6 +207,9 @@ const ciOnly = Object.entries({
119207
"mobile native lint": outputs.mobile_native,
120208
"release smoke": outputs.release_smoke,
121209
}).flatMap(([job, runs]) => (runs ? [job] : []));
122-
if (ciOnly.length > 0) console.log(` CI also runs: ${ciOnly.join(", ")}`);
123-
if (!runKnip) console.log(" knip skipped: no code or manifest changed");
210+
if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`);
211+
if (!runKnip) summary.push(" knip skipped: no code or manifest changed");
212+
console.log(`\n${summary.join("\n")}`);
124213
if (results.some((result) => !result.ok)) process.exit(1);
214+
NodeFS.mkdirSync(NodePath.dirname(cachePath), { recursive: true });
215+
NodeFS.writeFileSync(cachePath, JSON.stringify({ key, summary }));

0 commit comments

Comments
 (0)