This repository was archived by the owner on May 15, 2026. It is now read-only.
Repository navigation
Expand file tree
/
Copy pathindex.ts
More file actions
331 lines (283 loc) · 9.89 KB
/
Copy pathindex.ts
File metadata and controls
331 lines (283 loc) · 9.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
import * as fs from "fs/promises"
import * as path from "path"
import { LanguageParser, loadRequiredLanguageParsers } from "./languageParser"
import { fileExistsAtPath } from "../../utils/fs"
import { parseMarkdown } from "./markdownParser"
import { RooIgnoreController } from "../../core/ignore/RooIgnoreController"
import { QueryCapture } from "web-tree-sitter"
// Private constant
const DEFAULT_MIN_COMPONENT_LINES_VALUE = 4
// Getter function for MIN_COMPONENT_LINES (for easier testing)
let currentMinComponentLines = DEFAULT_MIN_COMPONENT_LINES_VALUE
/**
* Get the current minimum number of lines for a component to be included
*/
export function getMinComponentLines(): number {
return currentMinComponentLines
}
/**
* Set the minimum number of lines for a component (for testing)
*/
export function setMinComponentLines(value: number): void {
currentMinComponentLines = value
}
const extensions = [
"tla",
"js",
"jsx",
"ts",
"vue",
"tsx",
"py",
// Rust
"rs",
"go",
// C
"c",
"h",
// C++
"cpp",
"hpp",
// C#
"cs",
// Ruby
"rb",
"java",
"php",
"swift",
// Solidity
"sol",
// Kotlin
"kt",
"kts",
// Elixir
"ex",
"exs",
// Elisp
"el",
// HTML
"html",
"htm",
// Markdown
"md",
"markdown",
// JSON
"json",
// CSS
"css",
// SystemRDL
"rdl",
// OCaml
"ml",
"mli",
// Lua
"lua",
// Scala
"scala",
// TOML
"toml",
// Zig
"zig",
// Elm
"elm",
// Embedded Template
"ejs",
"erb",
// Visual Basic .NET
"vb",
].map((e) => `.${e}`)
export { extensions }
export async function parseSourceCodeDefinitionsForFile(
filePath: string,
rooIgnoreController?: RooIgnoreController,
): Promise<string | undefined> {
// check if the file exists
const fileExists = await fileExistsAtPath(path.resolve(filePath))
if (!fileExists) {
return "This file does not exist or you do not have permission to access it."
}
// Get file extension to determine parser
const ext = path.extname(filePath).toLowerCase()
// Check if the file extension is supported
if (!extensions.includes(ext)) {
return undefined
}
// Special case for markdown files
if (ext === ".md" || ext === ".markdown") {
// Check if we have permission to access this file
if (rooIgnoreController && !rooIgnoreController.validateAccess(filePath)) {
return undefined
}
// Read file content
const fileContent = await fs.readFile(filePath, "utf8")
// Split the file content into individual lines
const lines = fileContent.split("\n")
// Parse markdown content to get captures
const markdownCaptures = parseMarkdown(fileContent)
// Process the captures
const markdownDefinitions = processCaptures(markdownCaptures, lines, "markdown")
if (markdownDefinitions) {
return `# ${path.basename(filePath)}\n${markdownDefinitions}`
}
return undefined
}
// For other file types, load parser and use tree-sitter
const languageParsers = await loadRequiredLanguageParsers([filePath])
// Parse the file if we have a parser for it
const definitions = await parseFile(filePath, languageParsers, rooIgnoreController)
if (definitions) {
return `# ${path.basename(filePath)}\n${definitions}`
}
return undefined
}
/*
Parsing files using tree-sitter
1. Parse the file content into an AST (Abstract Syntax Tree) using the appropriate language grammar (set of rules that define how the components of a language like keywords, expressions, and statements can be combined to create valid programs).
2. Create a query using a language-specific query string, and run it against the AST's root node to capture specific syntax elements.
- We use tag queries to identify named entities in a program, and then use a syntax capture to label the entity and its name. A notable example of this is GitHub's search-based code navigation.
- Our custom tag queries are based on tree-sitter's default tag queries, but modified to only capture definitions.
3. Sort the captures by their position in the file, output the name of the definition, and format by i.e. adding "|----\n" for gaps between captured sections.
This approach allows us to focus on the most relevant parts of the code (defined by our language-specific queries) and provides a concise yet informative view of the file's structure and key elements.
- https://github.com/tree-sitter/node-tree-sitter/blob/master/test/query_test.js
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/query-test.js
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/helper.js
- https://tree-sitter.github.io/tree-sitter/code-navigation-systems
*/
/**
* Parse a file and extract code definitions using tree-sitter
*
* @param filePath - Path to the file to parse
* @param languageParsers - Map of language parsers
* @param rooIgnoreController - Optional controller to check file access permissions
* @returns A formatted string with code definitions or null if no definitions found
*/
/**
* Process captures from tree-sitter or markdown parser
*
* @param captures - The captures to process
* @param lines - The lines of the file
* @param minComponentLines - Minimum number of lines for a component to be included
* @returns A formatted string with definitions
*/
function processCaptures(captures: QueryCapture[], lines: string[], language: string): string | null {
// Determine if HTML filtering is needed for this language
const needsHtmlFiltering = ["jsx", "tsx"].includes(language)
// Filter function to exclude HTML elements if needed
const isNotHtmlElement = (line: string): boolean => {
if (!needsHtmlFiltering) return true
// Common HTML elements pattern
const HTML_ELEMENTS = /^[^A-Z]*<\/?(?:div|span|button|input|h[1-6]|p|a|img|ul|li|form)\b/
const trimmedLine = line.trim()
return !HTML_ELEMENTS.test(trimmedLine)
}
// No definitions found
if (captures.length === 0) {
return null
}
let formattedOutput = ""
// Sort captures by their start position
captures.sort((a, b) => a.node.startPosition.row - b.node.startPosition.row)
// Track already processed lines to avoid duplicates
const processedLines = new Set<string>()
// First pass - categorize captures by type
captures.forEach((capture) => {
const { node, name } = capture
// Skip captures that don't represent definitions
if (!name.includes("definition") && !name.includes("name")) {
return
}
// Get the parent node that contains the full definition
const definitionNode = name.includes("name") ? node.parent : node
if (!definitionNode) return
// Get the start and end lines of the full definition
const startLine = definitionNode.startPosition.row
const endLine = definitionNode.endPosition.row
const lineCount = endLine - startLine + 1
// Skip components that don't span enough lines
if (lineCount < getMinComponentLines()) {
return
}
// Create unique key for this definition based on line range
// This ensures we don't output the same line range multiple times
const lineKey = `${startLine}-${endLine}`
// Skip already processed lines
if (processedLines.has(lineKey)) {
return
}
// Check if this is a valid component definition (not an HTML element)
const startLineContent = lines[startLine].trim()
// Special handling for component name definitions
if (name.includes("name.definition")) {
// Extract component name
const componentName = node.text
// Add component name to output regardless of HTML filtering
if (!processedLines.has(lineKey) && componentName) {
formattedOutput += `${startLine + 1}--${endLine + 1} | ${lines[startLine]}\n`
processedLines.add(lineKey)
}
}
// For other component definitions
else if (isNotHtmlElement(startLineContent)) {
formattedOutput += `${startLine + 1}--${endLine + 1} | ${lines[startLine]}\n`
processedLines.add(lineKey)
// If this is part of a larger definition, include its non-HTML context
if (node.parent && node.parent.lastChild) {
const contextEnd = node.parent.lastChild.endPosition.row
const contextSpan = contextEnd - node.parent.startPosition.row + 1
// Only include context if it spans multiple lines
if (contextSpan >= getMinComponentLines()) {
// Add the full range first
const rangeKey = `${node.parent.startPosition.row}-${contextEnd}`
if (!processedLines.has(rangeKey)) {
formattedOutput += `${node.parent.startPosition.row + 1}--${contextEnd + 1} | ${lines[node.parent.startPosition.row]}\n`
processedLines.add(rangeKey)
}
}
}
}
})
if (formattedOutput.length > 0) {
return formattedOutput
}
return null
}
/**
* Parse a file and extract code definitions using tree-sitter
*
* @param filePath - Path to the file to parse
* @param languageParsers - Map of language parsers
* @param rooIgnoreController - Optional controller to check file access permissions
* @returns A formatted string with code definitions or null if no definitions found
*/
async function parseFile(
filePath: string,
languageParsers: LanguageParser,
rooIgnoreController?: RooIgnoreController,
): Promise<string | null> {
// Check if we have permission to access this file
if (rooIgnoreController && !rooIgnoreController.validateAccess(filePath)) {
return null
}
// Read file content
const fileContent = await fs.readFile(filePath, "utf8")
const extLang = path.extname(filePath).toLowerCase().slice(1)
// Check if we have a parser for this file type
const { parser, query } = languageParsers[extLang] || {}
if (!parser || !query) {
return `Unsupported file type: ${filePath}`
}
try {
// Parse the file content into an Abstract Syntax Tree (AST)
const tree = parser.parse(fileContent)
// Apply the query to the AST and get the captures
const captures = tree ? query.captures(tree.rootNode) : []
// Split the file content into individual lines
const lines = fileContent.split("\n")
// Process the captures
return processCaptures(captures, lines, extLang)
} catch (error) {
console.log(`Error parsing file: ${error}\n`)
// Return null on parsing error to avoid showing error messages in the output
return null
}
}