Skip to content

Commit a8f1b5b

Browse files
committed
perf(scripts): list ignored check inputs without walking node_modules
check:changed hashed the ignored files that checks read with one `git ls-files --others --ignored` pathspec command. A pathspec only filters git's output, so on Windows it walked every ignored directory, following pnpm's node_modules junctions until MAX_PATH: about 12 minutes for a one-file change. List ignored directories collapsed with --directory and expand the ones that can hold inputs by literal name, in batches bounded by command-line length. Directories that replace a tracked path (including paths marked assume-unchanged or skip-worktree) and ignored nested repositories keep contributing as before.
1 parent e68e19d commit a8f1b5b

2 files changed

Lines changed: 400 additions & 12 deletions

File tree

‎scripts/check-changed.test.ts‎

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process";
1616
import * as TestClock from "effect/testing/TestClock";
1717

1818
import {
19+
chunkByCommandLength,
1920
comparisonBase,
2021
defaultConcurrency,
2122
docReaderTests,
2223
gitLines,
24+
ignoredCheckInputs,
2325
runSteps,
2426
splitTests,
2527
type Step,
@@ -446,3 +448,299 @@ it.each(["install", "lint"])("reports a failing %s step with its output", (step)
446448
NodeFS.existsSync(NodePath.join(project.root, "node_modules/.cache/check-changed.json")),
447449
).toBe(false);
448450
});
451+
452+
// The command that listed ignored inputs before ignoredCheckInputs: a pathspec-filtered walk
453+
// of every ignored directory. `-z` only changes how git delimits and quotes paths, not
454+
// which paths it selects.
455+
const walkedIgnoredInputs = (git: (...args: Array<string>) => string) =>
456+
git(
457+
"-c",
458+
"core.quotePath=false",
459+
"ls-files",
460+
"-z",
461+
"--others",
462+
"--ignored",
463+
"--exclude-standard",
464+
"--",
465+
":(glob)**/.generated/**",
466+
":(glob)**/.astro/**",
467+
":(glob)**/*.d.ts",
468+
":(glob)**/.env*",
469+
":(exclude,glob)**/node_modules/**",
470+
)
471+
.split("\0")
472+
.filter(Boolean)
473+
.sort();
474+
475+
// Runs ignoredCheckInputs on a repository, recording the git commands it starts.
476+
const listIgnoredInputs = (root: string, gitEnv: Record<string, string>, budget?: number) => {
477+
const calls: Array<ReadonlyArray<string>> = [];
478+
return Effect.gen(function* () {
479+
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
480+
const listed = yield* ignoredCheckInputs(root, budget).pipe(
481+
Effect.provideService(
482+
ChildProcessSpawner.ChildProcessSpawner,
483+
ChildProcessSpawner.make((command) => {
484+
if (command._tag === "StandardCommand") calls.push(command.args.map(String));
485+
return spawner.spawn(ChildProcess.setEnv(command, gitEnv));
486+
}),
487+
),
488+
);
489+
return { listed, calls };
490+
}).pipe(Effect.provide(NodeServices.layer), Effect.runPromise);
491+
};
492+
493+
// An empty global config: git for Windows rejects the NUL device as a config path.
494+
const withIgnoredFixture = async (
495+
body: (fixture: {
496+
root: string;
497+
gitEnv: Record<string, string>;
498+
git: (...args: Array<string>) => string;
499+
write: (file: string, content?: string) => void;
500+
}) => Promise<void>,
501+
) => {
502+
const dir = NodeFS.realpathSync(
503+
NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-ignored-inputs-")),
504+
);
505+
const root = NodePath.join(dir, "repo");
506+
const globalConfig = NodePath.join(dir, "gitconfig");
507+
const gitEnv = { ...env, GIT_CONFIG_GLOBAL: globalConfig };
508+
try {
509+
NodeFS.writeFileSync(globalConfig, "");
510+
const write = (file: string, content = file) => {
511+
const path = NodePath.join(root, file);
512+
NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true });
513+
NodeFS.writeFileSync(path, content);
514+
};
515+
const git = (...args: Array<string>) =>
516+
NodeChildProcess.execFileSync("git", args, { cwd: root, encoding: "utf8", env: gitEnv });
517+
await body({ root, gitEnv, git, write });
518+
} finally {
519+
NodeFS.rmSync(dir, { recursive: true, force: true });
520+
}
521+
};
522+
523+
it("lists ignored generated, declaration, and env files without entering node_modules", async () => {
524+
await withIgnoredFixture(async ({ root, gitEnv, git, write }) => {
525+
write(
526+
".gitignore",
527+
"node_modules/\ndist/\ndist_old/\n.astro\n.env*\n!.env.example\n/.generated/\napps/*/.generated/\n*.log\n",
528+
);
529+
// Enough tracked packages, each with an ignored dist/, for several expansion calls.
530+
const packages = Array.from({ length: 60 }, (_, i) => `many/p${i}`);
531+
for (const file of [
532+
"src/a.ts",
533+
"src/tracked.d.ts",
534+
"apps/web/.env.example",
535+
"packages/replaced/dist",
536+
"sib/tracked.ts",
537+
"vendor/lib/a.ts",
538+
...packages.map((pkg) => `${pkg}/src.ts`),
539+
])
540+
write(file);
541+
git("init");
542+
git("add", "-f", ".");
543+
git("commit", "-m", "init");
544+
// A tracked file replaced by an ignored directory: ls-files hides the directory.
545+
NodeFS.rmSync(NodePath.join(root, "packages/replaced/dist"));
546+
// A nested repository in the tracked vendor/lib: git mishandles it as a pathspec prefix.
547+
git("init", "vendor/lib");
548+
for (const file of [
549+
"packages/replaced/dist/index.d.ts",
550+
"vendor/lib/dist/x.d.ts",
551+
// dist/ sorts right before dist_old/: a prefix test without the trailing
552+
// slash would take dist/ for its parent.
553+
"sib/dist/a.d.ts",
554+
"sib/dist_old/a.d.ts",
555+
".generated/x.ts",
556+
".generated/sub/y.d.ts",
557+
"apps/mobile/.generated/z.ts",
558+
"apps/web/.astro/types.d.ts",
559+
"apps/web/.astro/data.json",
560+
"packages/shared/dist/index.d.ts",
561+
"packages/shared/dist/deep/b.d.ts",
562+
"packages/shared/dist/readme.txt",
563+
".env",
564+
".env.local",
565+
"apps/x/.env.production",
566+
"src/debug.log",
567+
"src/new.d.ts",
568+
"apps/\u00fcn\u00ef/dist/\u00fc.d.ts",
569+
// Never listed: node_modules at any depth.
570+
"node_modules/pkg/index.d.ts",
571+
"node_modules/.generated/q.ts",
572+
"apps/web/node_modules/p/.env",
573+
"packages/shared/dist/node_modules/m/t.d.ts",
574+
"apps/web/node_modules/.astro/a.d.ts",
575+
// A package folder left behind by another branch: only ignored files, among them
576+
// its own node_modules. Its contents are listed with it, so it must not be walked.
577+
"stale/dist/x.d.ts",
578+
"stale/node_modules/pkg/index.d.ts",
579+
"stale/.env",
580+
// Listed: only a directory named node_modules is excluded, not a file.
581+
"apps/web/.astro/node_modules",
582+
...packages.map((pkg) => `${pkg}/dist/index.d.ts`),
583+
])
584+
write(file);
585+
// git quotes these in plain output; NTFS does not allow them.
586+
if (NodePath.sep === "/") {
587+
for (const file of ['dist/q"uote.d.ts', ".generated/ta\tb.ts", "back\\slash/dist/b.d.ts"])
588+
write(file);
589+
}
590+
591+
const budget = 600;
592+
const { listed, calls } = await listIgnoredInputs(root, gitEnv, budget);
593+
594+
// git is only asked to enter a directory by name, in batches that fit the budget, and
595+
// never one under node_modules or one whose contents the first listing already gave.
596+
const listings = calls.filter((args) => args.includes("ls-files") && !args.includes("-v"));
597+
const expansions = listings.filter((args) => !args.includes("--directory"));
598+
assert.strictEqual(listings.length, 1 + expansions.length);
599+
assert.isTrue(listings[0]!.includes("--directory"));
600+
assert.isAtLeast(expansions.length, 4);
601+
for (const args of expansions) {
602+
// The budget leaves room for `git -c core.quotePath=false`, the first two arguments.
603+
assert.isAtMost(
604+
args
605+
.slice(2)
606+
.map((arg) => `"${arg}"`)
607+
.join(" ").length,
608+
budget,
609+
);
610+
const pathspecs = args.slice(args.indexOf("--") + 1);
611+
assert.strictEqual(pathspecs[0], ":(literal).git/");
612+
assert.isFalse(pathspecs.some((spec) => spec.includes("node_modules")));
613+
assert.isFalse(pathspecs.includes(":(literal)stale/"));
614+
// A name is never read as a pattern, so git cannot be sent into a sibling by a `[1]`.
615+
assert.isTrue(pathspecs.every((spec) => spec.startsWith(":(literal)")));
616+
}
617+
618+
const walked = walkedIgnoredInputs(git);
619+
assert.isAbove(walked.length, 10);
620+
for (const file of [
621+
"stale/dist/x.d.ts",
622+
"stale/.env",
623+
"apps/web/.astro/node_modules",
624+
"many/p59/dist/index.d.ts",
625+
"packages/replaced/dist/index.d.ts",
626+
"vendor/lib/dist/x.d.ts",
627+
"sib/dist/a.d.ts",
628+
])
629+
assert.include(listed, file);
630+
assert.isFalse(listed.includes("stale/node_modules/pkg/index.d.ts"));
631+
assert.deepStrictEqual(listed, walked);
632+
});
633+
});
634+
635+
it("lists what the old command listed for marked paths and ignored nested repositories", async () => {
636+
await withIgnoredFixture(async ({ root, gitEnv, git, write }) => {
637+
write(".gitignore", "node_modules/\ndist/\nvendored/\n.env*\n");
638+
for (const file of [
639+
"src/a.ts",
640+
"assume/dist",
641+
"skip/dist",
642+
"both/dist",
643+
"sparse/dist",
644+
"vendor/lib/a.ts",
645+
])
646+
write(file);
647+
git("init");
648+
git("add", "-f", ".");
649+
git("commit", "-m", "init");
650+
git("update-index", "--assume-unchanged", "assume/dist");
651+
git("update-index", "--skip-worktree", "skip/dist");
652+
git("update-index", "--assume-unchanged", "--skip-worktree", "both/dist");
653+
// A tracked file that was never replaced, marked as a sparse checkout would.
654+
git("update-index", "--skip-worktree", "sparse/dist");
655+
const check = async () => {
656+
const { listed } = await listIgnoredInputs(root, gitEnv);
657+
assert.deepStrictEqual(listed, walkedIgnoredInputs(git));
658+
return listed;
659+
};
660+
// The only ignored directory, in a nested repository: one call, whose pathspecs would
661+
// share a prefix through that repository without the .git/ sentinel.
662+
git("init", "vendor/lib");
663+
write("vendor/lib/dist/x.d.ts");
664+
assert.deepStrictEqual(await check(), ["vendor/lib/dist/x.d.ts"]);
665+
666+
// The old command lists a marked path's files, because it walks the directory.
667+
for (const file of ["assume/dist", "skip/dist", "both/dist"]) {
668+
NodeFS.rmSync(NodePath.join(root, file));
669+
write(`${file}/index.d.ts`, "export type A = string;");
670+
}
671+
const first = await check();
672+
for (const file of ["assume", "skip", "both"]) assert.include(first, `${file}/dist/index.d.ts`);
673+
assert.include(first, "vendor/lib/dist/x.d.ts");
674+
675+
// An ignored nested repository changes the list when it is created or deleted: the old
676+
// command printed its directory, as the repository itself or inside an ignored folder.
677+
write("dist/x.d.ts");
678+
git("init", "dist");
679+
write("vendored/lib/y.d.ts");
680+
git("init", "vendored/lib");
681+
const withRepositories = await check();
682+
assert.include(withRepositories, "dist/");
683+
assert.include(withRepositories, "vendored/lib/");
684+
NodeFS.rmSync(NodePath.join(root, "dist/.git"), { recursive: true, force: true });
685+
NodeFS.rmSync(NodePath.join(root, "vendored/lib/.git"), { recursive: true, force: true });
686+
const without = await check();
687+
assert.isFalse(without.includes("dist/"));
688+
assert.isFalse(without.includes("vendored/lib/"));
689+
assert.include(without, "dist/x.d.ts");
690+
assert.include(without, "vendored/lib/y.d.ts");
691+
});
692+
});
693+
694+
describe("chunkByCommandLength", () => {
695+
// The quoted command line, as the longest form Windows would see it.
696+
const commandLine = (args: ReadonlyArray<string>) =>
697+
["git", "-c", "core.quotePath=false", ...args]
698+
.map((arg) => `"${arg.replace(/(["\\])/g, "\\$1")}"`)
699+
.join(" ");
700+
const fixed = [
701+
"ls-files",
702+
"-z",
703+
"--others",
704+
"--ignored",
705+
"--exclude-standard",
706+
"--",
707+
":(literal).git/",
708+
];
709+
710+
it("keeps long paths under the 32,767-unit command line limit, whatever their count", () => {
711+
// 80-character components, four deep: 340 characters a path, so 100 of them fill 35,000.
712+
const prefix = Array.from({ length: 4 }, (_, i) => String.fromCharCode(97 + i).repeat(80));
713+
const items = Array.from(
714+
{ length: 100 },
715+
(_, i) => `:(literal)${prefix.join("/")}/p${String(i).padStart(3, "0")}/.generated/`,
716+
);
717+
assert.isAbove(commandLine([...fixed, ...items]).length, 32_767);
718+
const chunks = chunkByCommandLength(items, fixed);
719+
assert.isAbove(chunks.length, 1);
720+
assert.deepStrictEqual(chunks.flat(), items);
721+
for (const chunk of chunks)
722+
assert.isAtMost(commandLine([...fixed, ...chunk]).length, 32_767 - 2_000);
723+
});
724+
725+
it("counts UTF-16 units, quotes and backslashes, and keeps short paths in one call", () => {
726+
const short = Array.from({ length: 300 }, (_, i) => `:(literal)many/p${i}/dist/`);
727+
assert.strictEqual(chunkByCommandLength(short, fixed).length, 1);
728+
// Three of each cost about 3,000 units apiece on the command line, so two fit in 7,000: an
729+
// emoji is two UTF-16 units, and a backslash or quote is written with an escape.
730+
for (const name of ["😀".repeat(1_495), "\\".repeat(1_495), '"'.repeat(1_495)]) {
731+
const wide = [1, 2, 3].map((n) => `:(literal)${name}${n}/`);
732+
const chunks = chunkByCommandLength(wide, fixed, 7_000);
733+
assert.deepStrictEqual(
734+
chunks.map((chunk) => chunk.length),
735+
[2, 1],
736+
);
737+
for (const chunk of chunks) assert.isAtMost(commandLine([...fixed, ...chunk]).length, 7_000);
738+
}
739+
// A path longer than the budget still goes out, alone, instead of being dropped.
740+
assert.deepStrictEqual(chunkByCommandLength(["x".repeat(500), "y"], fixed, 100), [
741+
["x".repeat(500)],
742+
["y"],
743+
]);
744+
assert.deepStrictEqual(chunkByCommandLength([], fixed), []);
745+
});
746+
});

0 commit comments

Comments
 (0)