Skip to content

Commit f694a6c

Browse files
committed
refactor(code-index): simplify file preparation stages
1 parent a68d518 commit f694a6c

2 files changed

Lines changed: 76 additions & 73 deletions

File tree

‎src/services/code-index/processors/__tests__/file-preparation.spec.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,12 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo
7676
reason: "File is in an ignored directory",
7777
})
7878
expect(dependencies.ignoreController.validateAccess).not.toHaveBeenCalled()
79+
expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled()
7980
expect(dependencies.fileSystem.stat).not.toHaveBeenCalled()
81+
expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled()
82+
expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled()
83+
expect(dependencies.parser.parseFile).not.toHaveBeenCalled()
84+
expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled()
8085
},
8186
)
8287

@@ -96,6 +101,10 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo
96101
expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath)
97102
}
98103
expect(dependencies.fileSystem.stat).not.toHaveBeenCalled()
104+
expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled()
105+
expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled()
106+
expect(dependencies.parser.parseFile).not.toHaveBeenCalled()
107+
expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled()
99108
})
100109

101110
it("skips oversized files without reading them", async () => {
@@ -112,6 +121,9 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo
112121
reason: "File is too large",
113122
})
114123
expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled()
124+
expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled()
125+
expect(dependencies.parser.parseFile).not.toHaveBeenCalled()
126+
expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled()
115127
})
116128

117129
it("skips unchanged content before parsing or embedding", async () => {
@@ -199,9 +211,12 @@ describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platfo
199211
const dependencies = setup()
200212
expect((await new FilePreparation(dependencies).prepareFile(filePath)).status).toBe("processed_for_batching")
201213
expect(dependencies.ignoreController.validateAccess.mock.contexts).toEqual([dependencies.ignoreController])
214+
expect(dependencies.ignoreInstance.ignores.mock.contexts).toEqual([dependencies.ignoreInstance])
202215
expect(dependencies.fileSystem.stat.mock.contexts).toEqual([dependencies.fileSystem])
203216
expect(dependencies.fileSystem.readFile.mock.contexts).toEqual([dependencies.fileSystem])
204217
expect(dependencies.cacheManager.getHash.mock.contexts).toEqual([dependencies.cacheManager])
218+
expect(dependencies.parser.parseFile.mock.contexts).toEqual([dependencies.parser])
219+
expect(dependencies.embedder.createEmbeddings.mock.contexts).toEqual([dependencies.embedder])
205220
})
206221

207222
it("does not treat a hidden workspace ancestor as an excluded directory", async () => {
Lines changed: 61 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createHash } from "crypto"
22
import { Uri } from "vscode"
33
import { v5 as uuidv5 } from "uuid"
4-
import type { FileProcessingResult, PointStruct } from "../interfaces"
4+
import type { CodeBlock, FileProcessingResult, PointStruct } from "../interfaces"
55
import type { FilePreparationDependencies } from "./file-preparation-dependencies"
66
import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants"
77
import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path"
@@ -14,99 +14,87 @@ export class FilePreparation {
1414
public async prepareFile(filePath: string): Promise<FileProcessingResult> {
1515
const dependencies = this.dependencies
1616
try {
17-
// Get relative path for ignore checks
18-
const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath)
19-
20-
// Check if file is in an ignored directory
21-
// Use relative path to avoid matching parent directories outside the workspace
22-
if (isPathInIgnoredDirectory(relativeFilePath)) {
23-
return {
24-
path: filePath,
25-
status: "skipped" as const,
26-
reason: "File is in an ignored directory",
27-
}
28-
}
29-
30-
// Check if file should be ignored
31-
if (
32-
!dependencies.ignoreController.validateAccess(filePath) ||
33-
(dependencies.ignoreInstance && dependencies.ignoreInstance.ignores(relativeFilePath))
34-
) {
35-
return {
36-
path: filePath,
37-
status: "skipped" as const,
38-
reason: "File is ignored by .rooignore or .gitignore",
39-
}
40-
}
41-
42-
// Check file size
43-
const fileStat = await dependencies.fileSystem.stat(Uri.file(filePath))
44-
if (fileStat.size > MAX_FILE_SIZE_BYTES) {
45-
return {
46-
path: filePath,
47-
status: "skipped" as const,
48-
reason: "File is too large",
49-
}
17+
const skipReason = await this.getSkipReason(filePath)
18+
if (skipReason !== undefined) {
19+
return this.skippedResult(filePath, skipReason)
5020
}
5121

52-
// Read file content
5322
const fileContent = await dependencies.fileSystem.readFile(Uri.file(filePath))
5423
const content = fileContent.toString()
55-
56-
// Calculate hash
5724
const newHash = createHash("sha256").update(content).digest("hex")
5825

59-
// Check if file has changed
6026
if (dependencies.cacheManager.getHash(filePath) === newHash) {
61-
return {
62-
path: filePath,
63-
status: "skipped" as const,
64-
reason: "File has not changed",
65-
}
27+
return this.skippedResult(filePath, "File has not changed")
6628
}
6729

68-
// Parse file
6930
const blocks = await dependencies.parser.parseFile(filePath, { content, fileHash: newHash })
70-
71-
// Prepare points for batch processing
72-
let pointsToUpsert: PointStruct[] = []
73-
if (dependencies.embedder && blocks.length > 0) {
74-
const texts = blocks.map((block) => block.content)
75-
const { embeddings } = await dependencies.embedder.createEmbeddings(texts)
76-
77-
pointsToUpsert = blocks.map((block, index) => {
78-
const normalizedAbsolutePath = generateNormalizedAbsolutePath(
79-
block.file_path,
80-
dependencies.workspacePath,
81-
)
82-
const stableName = `${normalizedAbsolutePath}:${block.start_line}`
83-
const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE)
84-
85-
return {
86-
id: pointId,
87-
vector: embeddings[index],
88-
payload: {
89-
filePath: generateRelativeFilePath(normalizedAbsolutePath, dependencies.workspacePath),
90-
codeChunk: block.content,
91-
startLine: block.start_line,
92-
endLine: block.end_line,
93-
},
94-
}
95-
})
96-
}
31+
const pointsToUpsert = await this.preparePoints(blocks)
9732

9833
return {
9934
path: filePath,
100-
status: "processed_for_batching" as const,
35+
status: "processed_for_batching",
10136
newHash,
10237
pointsToUpsert,
10338
}
10439
} catch (error) {
10540
return {
10641
path: filePath,
107-
status: "local_error" as const,
42+
status: "local_error",
10843
error: error as Error,
10944
}
11045
}
11146
}
47+
48+
private async getSkipReason(filePath: string): Promise<string | undefined> {
49+
const dependencies = this.dependencies
50+
// Use relative paths so ignored directories outside the workspace do not exclude the file.
51+
const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath)
52+
if (isPathInIgnoredDirectory(relativeFilePath)) {
53+
return "File is in an ignored directory"
54+
}
55+
56+
if (
57+
!dependencies.ignoreController.validateAccess(filePath) ||
58+
dependencies.ignoreInstance?.ignores(relativeFilePath)
59+
) {
60+
return "File is ignored by .rooignore or .gitignore"
61+
}
62+
63+
const fileStat = await dependencies.fileSystem.stat(Uri.file(filePath))
64+
if (fileStat.size > MAX_FILE_SIZE_BYTES) {
65+
return "File is too large"
66+
}
67+
return undefined
68+
}
69+
70+
private skippedResult(filePath: string, reason: string): FileProcessingResult {
71+
return { path: filePath, status: "skipped", reason }
72+
}
73+
74+
private async preparePoints(blocks: CodeBlock[]): Promise<PointStruct[]> {
75+
const dependencies = this.dependencies
76+
if (!dependencies.embedder || blocks.length === 0) {
77+
return []
78+
}
79+
80+
const texts = blocks.map((block) => block.content)
81+
const { embeddings } = await dependencies.embedder.createEmbeddings(texts)
82+
83+
return blocks.map((block, index) => {
84+
const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, dependencies.workspacePath)
85+
const stableName = `${normalizedAbsolutePath}:${block.start_line}`
86+
const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE)
87+
88+
return {
89+
id: pointId,
90+
vector: embeddings[index],
91+
payload: {
92+
filePath: generateRelativeFilePath(normalizedAbsolutePath, dependencies.workspacePath),
93+
codeChunk: block.content,
94+
startLine: block.start_line,
95+
endLine: block.end_line,
96+
},
97+
}
98+
})
99+
}
112100
}

0 commit comments

Comments
 (0)