Skip to content

Commit c8b5491

Browse files
committed
fix(code-index): address plain text review feedback
1 parent f34dd54 commit c8b5491

7 files changed

Lines changed: 74 additions & 16 deletions

File tree

‎src/services/code-index/processors/__tests__/parser.spec.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,31 @@ describe("CodeParser", () => {
267267
})
268268

269269
describe("_chunkTextByLines", () => {
270+
it("should not emit a chunk whose segment hash has already been seen", async () => {
271+
const lines = ["Fallback content long enough to produce a chunk without being filtered out."]
272+
const seenSegmentHashes = new Set<string>()
273+
274+
const firstResult = await parser["_chunkTextByLines"](
275+
lines,
276+
"manual.txt",
277+
"hash",
278+
"fallback_chunk",
279+
seenSegmentHashes,
280+
)
281+
const duplicateResult = await parser["_chunkTextByLines"](
282+
lines,
283+
"manual.txt",
284+
"hash",
285+
"fallback_chunk",
286+
seenSegmentHashes,
287+
)
288+
289+
expect(firstResult).toHaveLength(1)
290+
expect(firstResult[0].segmentHash).toMatch(/^[a-f0-9]{64}$/)
291+
expect(seenSegmentHashes).toEqual(new Set([firstResult[0].segmentHash]))
292+
expect(duplicateResult).toEqual([])
293+
})
294+
270295
it("should handle oversized lines by splitting them", async () => {
271296
const longLine = "a".repeat(2000)
272297
const lines = ["normal", longLine, "normal"]

‎src/services/code-index/processors/__tests__/parser.txt.spec.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,26 @@ describe("CodeParser - plain text support", () => {
3333
end_line: 3,
3434
content,
3535
fileHash: "txt-file-hash",
36+
segmentHash: expect.any(String),
3637
})
3738
})
3839

39-
it("matches uppercase .TXT extensions", () => {
40+
it("parses uppercase .TXT extensions", async () => {
4041
expect(shouldUseFallbackChunking(".TXT")).toBe(true)
42+
43+
const content = "Uppercase plain-text extension content long enough to produce a fallback chunk."
44+
const blocks = await new CodeParser().parseFile("manual.TXT", {
45+
content,
46+
fileHash: "uppercase-txt-file-hash",
47+
})
48+
49+
expect(blocks).toHaveLength(1)
50+
expect(blocks[0]).toMatchObject({
51+
file_path: "manual.TXT",
52+
type: "fallback_chunk",
53+
content,
54+
fileHash: "uppercase-txt-file-hash",
55+
segmentHash: expect.any(String),
56+
})
4157
})
4258
})

‎src/services/code-index/processors/parser.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ export class CodeParser implements ICodeParser {
4545
let content: string
4646
let fileHash: string
4747

48-
if (options?.content) {
48+
if (options?.content !== undefined) {
4949
content = options.content
5050
fileHash = options.fileHash || this.createFileHash(content)
5151
} else {

‎src/services/code-index/shared/supported-extensions.ts‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
import { extensions as allExtensions } from "../../tree-sitter"
2+
import { fallbackExtensions, isFallbackExtension } from "../../shared/fallback-extensions"
3+
4+
export { fallbackExtensions }
25

36
// Include all extensions including markdown for the scanner
47
export const scannerExtensions = allExtensions
@@ -18,18 +21,11 @@ export const scannerExtensions = allExtensions
1821
*
1922
* Note: Do NOT remove parser cases from languageParser.ts as they may be used elsewhere
2023
*/
21-
export const fallbackExtensions = [
22-
".txt", // Plain text - no tree-sitter parser needed
23-
".vb", // Visual Basic .NET - no dedicated WASM parser
24-
".scala", // Scala - uses fallback chunking instead of Lua query workaround
25-
".swift", // Swift - uses fallback chunking due to parser instability
26-
]
27-
2824
/**
2925
* Check if a file extension should use fallback chunking
3026
* @param extension File extension (including the dot)
3127
* @returns true if the extension should use fallback chunking
3228
*/
3329
export function shouldUseFallbackChunking(extension: string): boolean {
34-
return fallbackExtensions.includes(extension.toLowerCase())
30+
return isFallbackExtension(extension)
3531
}
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/**
2+
* Extensions that should not be parsed for structural definitions and should
3+
* instead use line-based fallback chunking where indexing is supported.
4+
*/
5+
export const fallbackExtensions = [".txt", ".vb", ".scala", ".swift"] as const
6+
7+
/**
8+
* Check whether a file extension should bypass structural parsing.
9+
*
10+
* @param extension File extension, including the leading dot
11+
*/
12+
export function isFallbackExtension(extension: string): boolean {
13+
return (fallbackExtensions as readonly string[]).includes(extension.toLowerCase())
14+
}

‎src/services/tree-sitter/__tests__/plainTextIntegration.spec.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,21 @@ import * as fs from "fs/promises"
1616
import { loadRequiredLanguageParsers } from "../languageParser"
1717
import { parseSourceCodeDefinitionsForFile } from "../index"
1818

19-
describe("Plain Text Integration Tests", () => {
19+
describe("Fallback Extension Integration Tests", () => {
2020
beforeEach(() => {
2121
vi.clearAllMocks()
2222
})
2323

24-
it("returns undefined without loading a tree-sitter parser", async () => {
25-
const result = await parseSourceCodeDefinitionsForFile("manual.txt")
24+
it.each(["manual.txt", "legacy.vb", "service.scala", "client.swift"])(
25+
"returns undefined for %s without loading a tree-sitter parser",
26+
async (filePath) => {
27+
const result = await parseSourceCodeDefinitionsForFile(filePath)
2628

27-
expect(result).toBeUndefined()
29+
expect(result).toBeUndefined()
30+
},
31+
)
32+
33+
afterEach(() => {
2834
expect(loadRequiredLanguageParsers).not.toHaveBeenCalled()
2935
expect(fs.readFile).not.toHaveBeenCalled()
3036
})

‎src/services/tree-sitter/index.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { fileExistsAtPath } from "../../utils/fs"
55
import { parseMarkdown } from "./markdownParser"
66
import { RooIgnoreController } from "../../core/ignore/RooIgnoreController"
77
import { QueryCapture } from "web-tree-sitter"
8+
import { isFallbackExtension } from "../shared/fallback-extensions"
89

910
// Private constant
1011
const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4
@@ -113,8 +114,8 @@ export async function parseSourceCodeDefinitionsForFile(
113114
return undefined
114115
}
115116

116-
// Plain text files have no structural definitions to extract
117-
if (ext === ".txt") {
117+
// Fallback files have no structural definitions to extract
118+
if (isFallbackExtension(ext)) {
118119
return undefined
119120
}
120121

0 commit comments

Comments
 (0)