Skip to content

Commit da39114

Browse files
fix(server): workspace search leaves out files Git ignores outside .gitignore
The native search index applies .gitignore files only, so files ignored through .git/info/exclude, core.excludesFile or the default global ignore file showed up in path search, the flat listing and content search. The index now lists Git's ignored paths with `git ls-files --others --ignored --exclude-standard --directory` when it is built and on each refresh, and drops matching entries and grep hits. Path searches widen their first page when hidden entries fill it. AI-assisted; verified with the tests and tools listed above.
1 parent 117d792 commit da39114

2 files changed

Lines changed: 160 additions & 17 deletions

File tree

‎apps/server/src/workspace/WorkspaceSearchIndex.test.ts‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,15 @@ import { afterEach, expect, it } from "@effect/vitest";
1010
import * as HostProcess from "@t3tools/shared/HostProcess";
1111
import * as Effect from "effect/Effect";
1212
import * as FileSystem from "effect/FileSystem";
13+
import * as Layer from "effect/Layer";
1314
import * as Path from "effect/Path";
1415
import { vi } from "vite-plus/test";
1516

17+
import * as VcsProcess from "../vcs/VcsProcess.ts";
1618
import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts";
1719

20+
const testLayer = VcsProcess.layer.pipe(Layer.provideMerge(NodeServices.layer));
21+
1822
afterEach(() => {
1923
vi.restoreAllMocks();
2024
});
@@ -64,7 +68,49 @@ it.effect("filters image searches before applying the result limit", () =>
6468
expect(fileSearch).toHaveBeenCalledTimes(2);
6569
expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 });
6670
}),
67-
),
71+
).pipe(Effect.provide(testLayer)),
72+
);
73+
74+
it.effect("fetches past a first page of Git-ignored results", () =>
75+
Effect.scoped(
76+
Effect.gen(function* () {
77+
const fs = yield* FileSystem.FileSystem;
78+
const path = yield* Path.Path;
79+
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-search-ignored-" });
80+
const vcsProcess = yield* VcsProcess.VcsProcess;
81+
yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd });
82+
yield* fs.writeFileString(path.join(cwd, ".git", "info", "exclude"), "hidden/\n");
83+
yield* fs.makeDirectory(path.join(cwd, "hidden"));
84+
yield* fs.writeFileString(path.join(cwd, "hidden", "a.ts"), "");
85+
86+
const items = [
87+
fileItem("hidden/a.ts"),
88+
fileItem("hidden/b.ts"),
89+
fileItem("hidden/c.ts"),
90+
fileItem("src/a.ts"),
91+
];
92+
const fileSearch = vi.fn((_query: string, options?: { pageSize?: number }) => ({
93+
ok: true as const,
94+
value: {
95+
items: items.slice(0, options?.pageSize),
96+
scores: [],
97+
totalMatched: items.length,
98+
totalFiles: items.length,
99+
},
100+
}));
101+
const finder = {
102+
destroy: vi.fn(),
103+
waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
104+
fileSearch,
105+
} as unknown as FileFinder;
106+
vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
107+
108+
const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
109+
const result = yield* searchIndex.search("a", 1, "file", false);
110+
111+
expect(result).toEqual({ entries: [{ kind: "file", path: "src/a.ts" }], truncated: false });
112+
}),
113+
).pipe(Effect.provide(testLayer)),
68114
);
69115

70116
it.effect("continues whole-word searches after a filtered grep page", () =>
@@ -141,7 +187,7 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
141187
expect(grep).toHaveBeenCalledTimes(2);
142188
expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor);
143189
}),
144-
),
190+
).pipe(Effect.provide(testLayer)),
145191
);
146192

147193
// fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there.
@@ -172,5 +218,5 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
172218

173219
expect(result).toBe("WorkspaceSearchIndexCreateFailed");
174220
}),
175-
).pipe(Effect.provide(NodeServices.layer)),
221+
).pipe(Effect.provide(testLayer)),
176222
);

‎apps/server/src/workspace/WorkspaceSearchIndex.ts‎

Lines changed: 111 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,11 @@ import * as NodeModule from "node:module";
22

33
import type {
44
DirItem,
5-
DirSearchResult,
65
FileItem,
76
FileFinder as FileFinderType,
87
GrepCursor,
98
MixedItem,
10-
MixedSearchResult,
119
Result,
12-
SearchResult,
1310
} from "@ff-labs/fff-node";
1411
import * as Context from "effect/Context";
1512
import * as Effect from "effect/Effect";
@@ -27,6 +24,8 @@ import type {
2724
} from "@t3tools/contracts";
2825
import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
2926

27+
import * as VcsProcess from "../vcs/VcsProcess.ts";
28+
3029
// fff-node's native library stays external; single-executables require disk packages instead of importing them.
3130
const requireForFff = NodeModule.createRequire(import.meta.url);
3231
const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node");
@@ -156,8 +155,10 @@ function toDirectoryEntry(item: DirItem): ProjectEntry | null {
156155
return normalizedPath ? { path: normalizedPath, kind: "directory" } : null;
157156
}
158157

158+
type SearchPage<I> = { readonly items: ReadonlyArray<I>; readonly totalMatched: number };
159+
159160
function mapFileSearchResult(
160-
result: SearchResult,
161+
result: SearchPage<FileItem>,
161162
limit: number,
162163
imageOnly = false,
163164
): ProjectSearchEntriesResult {
@@ -172,7 +173,7 @@ function mapFileSearchResult(
172173
}
173174

174175
function mapDirectorySearchResult(
175-
result: DirSearchResult,
176+
result: SearchPage<DirItem>,
176177
limit: number,
177178
): ProjectSearchEntriesResult {
178179
const entries = result.items.flatMap((item) => {
@@ -187,7 +188,7 @@ function mapDirectorySearchResult(
187188
}
188189

189190
function mapMixedSearchResult(
190-
result: MixedSearchResult,
191+
result: SearchPage<MixedItem>,
191192
limit: number,
192193
): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
193194
const entries: ProjectEntry[] = [];
@@ -284,6 +285,54 @@ function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEn
284285
return [...entryByPath.values()];
285286
}
286287

288+
// fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file.
289+
const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* (
290+
vcsProcess: VcsProcess.VcsProcess["Service"],
291+
cwd: string,
292+
) {
293+
const result = yield* vcsProcess
294+
.run({
295+
operation: "WorkspaceSearchIndex.listGitIgnoredPaths",
296+
command: "git",
297+
args: [
298+
"-c",
299+
"core.fsmonitor=false",
300+
"ls-files",
301+
"--others",
302+
"--ignored",
303+
"--exclude-standard",
304+
"--directory",
305+
"-z",
306+
],
307+
cwd,
308+
allowNonZeroExit: true,
309+
timeoutMs: 10_000,
310+
maxOutputBytes: 16 * 1024 * 1024,
311+
})
312+
.pipe(Effect.orElseSucceed(() => undefined));
313+
if (!result || result.exitCode !== 0 || result.stdoutTruncated) return new Set<string>();
314+
return new Set(
315+
result.stdout
316+
.split("\0")
317+
.filter((entry) => entry.length > 0)
318+
.map((entry) => trimDirectorySeparator(toPosixPath(entry))),
319+
);
320+
});
321+
322+
function mixedItemPath(item: MixedItem): string {
323+
return item.item.relativePath;
324+
}
325+
326+
function isIgnoredPath(ignoredPaths: ReadonlySet<string>, relativePath: string): boolean {
327+
if (ignoredPaths.size === 0) return false;
328+
let candidate: string | undefined = trimDirectorySeparator(toPosixPath(relativePath));
329+
while (candidate) {
330+
if (ignoredPaths.has(candidate)) return true;
331+
candidate = parentPathOf(candidate);
332+
}
333+
return false;
334+
}
335+
287336
const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (
288337
cwd: string,
289338
variant: WorkspaceSearchIndexVariant,
@@ -346,6 +395,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
346395
}).pipe(Effect.orDie),
347396
);
348397
let initialScanTimedOut = false;
398+
const vcsProcess = yield* VcsProcess.VcsProcess;
399+
let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd);
349400
yield* waitForIndexReady(
350401
cwd,
351402
finder,
@@ -396,6 +447,34 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
396447
return result.value;
397448
});
398449

450+
// Grows the first page until hidden entries can no longer crowd visible ones out of it.
451+
const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* <I>(
452+
query: string,
453+
pageSize: number,
454+
operation: "directorySearch" | "fileSearch" | "mixedSearch",
455+
execute: (pageSize: number) => Result<SearchPage<I>>,
456+
pathOf: (item: I) => string,
457+
): Effect.fn.Return<SearchPage<I>, WorkspaceSearchIndexSearchFailed> {
458+
let size = pageSize;
459+
while (true) {
460+
const fetchSize = size;
461+
const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize));
462+
if (ignoredPaths.size === 0) return result;
463+
const items = result.items.filter((item) => !isIgnoredPath(ignoredPaths, pathOf(item)));
464+
if (
465+
items.length >= pageSize ||
466+
result.items.length >= result.totalMatched ||
467+
fetchSize >= WORKSPACE_INDEX_PAGE_SIZE
468+
) {
469+
return {
470+
items,
471+
totalMatched: result.totalMatched - (result.items.length - items.length),
472+
};
473+
}
474+
size = Math.min(WORKSPACE_INDEX_PAGE_SIZE, fetchSize * 4);
475+
}
476+
});
477+
399478
const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
400479
"WorkspaceSearchIndex.refresh",
401480
)(function* () {
@@ -427,13 +506,18 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
427506
cause,
428507
}),
429508
).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void }));
509+
ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd);
430510
});
431511

432512
const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
433513
function* () {
434514
const incompleteBeforeQuery = hasIncompleteInitialScan();
435-
const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () =>
436-
finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }),
515+
const result = yield* searchVisible(
516+
"",
517+
WORKSPACE_INDEX_PAGE_SIZE,
518+
"mixedSearch",
519+
(pageSize) => finder.mixedSearch("", { pageSize }),
520+
mixedItemPath,
437521
);
438522
const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES);
439523
const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
@@ -456,19 +540,31 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
456540
) {
457541
const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1);
458542
if (kind === "file" || imageOnly) {
459-
const result = yield* runSearch(query, pageSize, "fileSearch", () =>
460-
finder.fileSearch(query, { pageSize }),
543+
const result = yield* searchVisible(
544+
query,
545+
pageSize,
546+
"fileSearch",
547+
(size) => finder.fileSearch(query, { pageSize: size }),
548+
(item) => item.relativePath,
461549
);
462550
return mapFileSearchResult(result, limit, imageOnly);
463551
}
464552
if (kind === "directory") {
465-
const result = yield* runSearch(query, pageSize, "directorySearch", () =>
466-
finder.directorySearch(query, { pageSize }),
553+
const result = yield* searchVisible(
554+
query,
555+
pageSize,
556+
"directorySearch",
557+
(size) => finder.directorySearch(query, { pageSize: size }),
558+
(item) => item.relativePath,
467559
);
468560
return mapDirectorySearchResult(result, limit);
469561
}
470-
const result = yield* runSearch(query, pageSize, "mixedSearch", () =>
471-
finder.mixedSearch(query, { pageSize }),
562+
const result = yield* searchVisible(
563+
query,
564+
pageSize,
565+
"mixedSearch",
566+
(size) => finder.mixedSearch(query, { pageSize: size }),
567+
mixedItemPath,
472568
);
473569
return mapMixedSearchResult(result, limit);
474570
});
@@ -523,6 +619,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
523619
}
524620

525621
for (const match of result.items) {
622+
if (isIgnoredPath(ignoredPaths, match.relativePath)) continue;
526623
const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter(
527624
(range) => !input.wholeWord || isWholeWordRange(match.lineContent, range),
528625
);

0 commit comments

Comments
 (0)