Skip to content

Commit 78704ad

Browse files
committed
perf(server): expanding a wide ignored folder no longer classifies every child
Listing a folder asks git check-ignore about each child, and git consults the index for every path, so a node_modules-sized folder took about 270 ms in a 20,000-file repository. When the listed folder is itself ignored and nothing under it is tracked, every child is ignored, so the listing now answers that with two git calls instead. Output is unchanged; folders that are not ignored, or that hold tracked files, take the existing path.
1 parent 4d73e1d commit 78704ad

2 files changed

Lines changed: 141 additions & 15 deletions

File tree

‎apps/server/src/workspace/WorkspaceEntries.test.ts‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,91 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
134134
}),
135135
);
136136

137+
describe("wide ignored directories", () => {
138+
// More children than one `git check-ignore` chunk, which is where listing skips asking git
139+
// about each child when the whole directory is ignored.
140+
const WIDE = 1001;
141+
const writeWideDirectory = (cwd: string, directory: string) =>
142+
Effect.forEach(
143+
Array.from({ length: WIDE }, (_, index) => `${directory}/pkg-${index}.js`),
144+
(relativePath) => writeTextFile(cwd, relativePath),
145+
{ concurrency: 16, discard: true },
146+
);
147+
148+
it.effect("marks every child of an ignored directory with nothing tracked as ignored", () =>
149+
Effect.gen(function* () {
150+
const cwd = yield* makeTempDir({ git: true });
151+
yield* writeTextFile(cwd, ".gitignore", "node_modules/\n");
152+
yield* writeWideDirectory(cwd, "node_modules");
153+
154+
const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
155+
const listing = yield* workspaceEntries.list({ cwd, directoryPath: "node_modules" });
156+
expect(listing.entries).toHaveLength(WIDE);
157+
expect(listing.entries.every((entry) => entry.ignored === true)).toBe(true);
158+
}),
159+
);
160+
161+
it.effect("keeps a force-tracked file and a directory with tracked files unignored", () =>
162+
Effect.gen(function* () {
163+
const cwd = yield* makeTempDir({ git: true });
164+
yield* writeTextFile(cwd, ".gitignore", "node_modules/\n");
165+
yield* writeWideDirectory(cwd, "node_modules");
166+
yield* writeTextFile(cwd, "node_modules/kept.js");
167+
yield* writeTextFile(cwd, "node_modules/vendored/index.js");
168+
yield* git(cwd, ["add", "-f", "node_modules/kept.js", "node_modules/vendored/index.js"]);
169+
170+
const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
171+
const listing = yield* workspaceEntries.list({ cwd, directoryPath: "node_modules" });
172+
const ignoredPaths = new Set(
173+
listing.entries.filter((entry) => entry.ignored).map((entry) => entry.path),
174+
);
175+
expect(listing.entries).toHaveLength(WIDE + 2);
176+
expect(ignoredPaths.has("node_modules/pkg-0.js")).toBe(true);
177+
expect(ignoredPaths.has("node_modules/kept.js")).toBe(false);
178+
expect(ignoredPaths.has("node_modules/vendored")).toBe(false);
179+
}),
180+
);
181+
182+
it.effect(
183+
"leaves children of a symlinked directory unlabelled, as git cannot classify them",
184+
() =>
185+
Effect.gen(function* () {
186+
const platform = yield* HostProcess.Platform;
187+
if (platform === "win32") return;
188+
const cwd = yield* makeTempDir({ git: true });
189+
const fileSystem = yield* FileSystem.FileSystem;
190+
const path = yield* Path.Path;
191+
yield* writeTextFile(cwd, ".gitignore", "node_modules/\n");
192+
yield* writeWideDirectory(cwd, "store");
193+
yield* fileSystem.makeDirectory(path.join(cwd, "node_modules"), { recursive: true });
194+
yield* fileSystem.symlink(path.join(cwd, "store"), path.join(cwd, "node_modules/link"));
195+
196+
const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
197+
const listing = yield* workspaceEntries.list({
198+
cwd,
199+
directoryPath: "node_modules/link",
200+
});
201+
expect(listing.entries).toHaveLength(WIDE);
202+
expect(listing.entries.some((entry) => entry.ignored)).toBe(false);
203+
}),
204+
);
205+
206+
it.effect("does not mark a wide directory that is not ignored", () =>
207+
Effect.gen(function* () {
208+
const cwd = yield* makeTempDir({ git: true });
209+
yield* writeTextFile(cwd, ".gitignore", "*.map\n");
210+
yield* writeWideDirectory(cwd, "src");
211+
yield* writeTextFile(cwd, "src/bundle.map");
212+
213+
const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
214+
const listing = yield* workspaceEntries.list({ cwd, directoryPath: "src" });
215+
expect(
216+
listing.entries.filter((entry) => entry.ignored).map((entry) => entry.path),
217+
).toEqual(["src/bundle.map"]);
218+
}),
219+
);
220+
});
221+
137222
it.effect(
138223
"rejects directory traversal, git internals, and symlinks outside the workspace",
139224
() =>

‎apps/server/src/workspace/WorkspaceEntries.ts‎

Lines changed: 56 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ export class WorkspaceEntries extends Context.Service<
106106
}
107107
>()("t3/workspace/WorkspaceEntries") {}
108108

109+
/** Paths sent to one `git check-ignore` call when labelling a folder's children. */
110+
const IGNORE_CHECK_CHUNK_SIZE = 1000;
111+
109112
const resolveBrowseTarget = Effect.fn("WorkspaceEntries.resolveBrowseTarget")(function* (
110113
input: FilesystemBrowseInput,
111114
path: Path.Path,
@@ -291,7 +294,7 @@ export const make = Effect.gen(function* () {
291294
relativePath: directoryPath,
292295
})
293296
.pipe(Effect.mapError(toError));
294-
const entries = yield* Effect.tryPromise({
297+
const { entries, throughSymlink } = yield* Effect.tryPromise({
295298
try: async () => {
296299
const root = await NodeFSP.realpath(normalizedCwd);
297300
const directory = await NodeFSP.realpath(target.absolutePath);
@@ -306,35 +309,73 @@ export const make = Effect.gen(function* () {
306309
throw new Error("Directory must be inside the workspace and outside .git.");
307310
}
308311
const children = await NodeFSP.readdir(directory, { withFileTypes: true });
309-
return children.flatMap((child): ProjectEntry[] => {
310-
if (child.name === ".git" || (!child.isDirectory() && !child.isFile())) return [];
311-
return [
312-
{
313-
path: target.relativePath ? `${target.relativePath}/${child.name}` : child.name,
314-
kind: child.isDirectory() ? "directory" : "file",
315-
},
316-
];
317-
});
312+
return {
313+
entries: children.flatMap((child): ProjectEntry[] => {
314+
if (child.name === ".git" || (!child.isDirectory() && !child.isFile())) return [];
315+
return [
316+
{
317+
path: target.relativePath ? `${target.relativePath}/${child.name}` : child.name,
318+
kind: child.isDirectory() ? "directory" : "file",
319+
},
320+
];
321+
}),
322+
// Git refuses to classify paths beyond a symbolic link.
323+
throughSymlink: relative !== target.relativePath.split("/").join(path.sep),
324+
};
318325
},
319326
catch: toError,
320327
});
321328
// Use stdin so large directories cannot exceed the command-line argument limit.
322329
// Ignore classification is optional in non-git workspaces or when git is unavailable.
323330
const ignored = new Set<string>();
324-
for (let offset = 0; offset < entries.length; offset += 1000) {
325-
const chunk = entries.slice(offset, offset + 1000);
326-
const result = yield* vcsProcess
331+
const runGit = (args: ReadonlyArray<string>, stdin?: string) =>
332+
vcsProcess
327333
.run({
328334
operation: "WorkspaceEntries.list",
329335
command: "git",
330-
args: ["-c", "core.fsmonitor=false", "check-ignore", "-z", "--stdin"],
336+
args: ["-c", "core.fsmonitor=false", ...args],
331337
cwd: normalizedCwd,
332-
stdin: `${chunk.map((entry) => entry.path).join("\0")}\0`,
338+
...(stdin === undefined ? {} : { stdin }),
333339
allowNonZeroExit: true,
334340
timeoutMs: 10_000,
335341
maxOutputBytes: 16 * 1024 * 1024,
336342
})
337343
.pipe(Effect.orElseSucceed(() => undefined));
344+
// Git consults the index for every path it classifies, which costs hundreds of
345+
// milliseconds for a `node_modules`-sized folder. An ignored directory hides all of its
346+
// children, so when nothing under it is tracked they are all ignored without asking.
347+
const wholeDirectoryIgnored =
348+
entries.length > IGNORE_CHECK_CHUNK_SIZE &&
349+
target.relativePath !== "" &&
350+
!throughSymlink &&
351+
(yield* Effect.gen(function* () {
352+
const directory = yield* runGit(
353+
["check-ignore", "-z", "--stdin"],
354+
`${target.relativePath}\0`,
355+
);
356+
if (directory?.exitCode !== 0) return false;
357+
const tracked = yield* runGit([
358+
"--literal-pathspecs",
359+
"ls-files",
360+
"--cached",
361+
"-z",
362+
"--",
363+
`${target.relativePath}/`,
364+
]);
365+
return tracked?.exitCode === 0 && tracked.stdout === "";
366+
}));
367+
if (wholeDirectoryIgnored) {
368+
return {
369+
entries: entries.map((entry) => ({ ...entry, ignored: true })),
370+
truncated: false,
371+
};
372+
}
373+
for (let offset = 0; offset < entries.length; offset += IGNORE_CHECK_CHUNK_SIZE) {
374+
const chunk = entries.slice(offset, offset + IGNORE_CHECK_CHUNK_SIZE);
375+
const result = yield* runGit(
376+
["check-ignore", "-z", "--stdin"],
377+
`${chunk.map((entry) => entry.path).join("\0")}\0`,
378+
);
338379
if (!result || (result.exitCode !== 0 && result.exitCode !== 1)) break;
339380
for (const ignoredPath of result.stdout.split("\0")) ignored.add(ignoredPath);
340381
}

0 commit comments

Comments
 (0)