Skip to content

Commit d5fb009

Browse files
committed
fix: share one bounded reader between read_file and grep_file
grep_file's scanner set a 1MB line buffer; ReadFileContent kept bufio's 64KB default. A file with one longer line -- a minified bundle, a single-line JSON blob -- therefore failed read_file outright, losing every other line in the file, while grep_file handled the same file fine and read_file's own tool description promises such lines are truncated. Python truncated correctly throughout, so this was a Go-only regression introduced alongside grep_file. Extract scanFileLines as the single reader behind both. It owns the non-regular-file rejection (previously duplicated), the buffer cap, and error wrapping, so the two paths can no longer drift on any of the three. Lines past maxLineBytes still error rather than truncate: uncapping would mean buffering an arbitrarily long line in a sandbox reading untrusted files. Also in this change: - Wrap ListDirContent's os.ReadDir error, the last bare return beside a wrapped one. - Correct file_search_tools_enabled()'s docstring, which still claimed list_files/grep_file are "disabled by default, same as bash". Bash's gate was removed upstream in #2498; this was the last of six such sites. - Drop the _validate_path call in grep_content's entry loop. The search-root bound added beside it is strictly narrower, and file_or_dir_path is already validated against allowed_root, so the wider check is dead. - Give Python the _MAX_LINE_CHARS/_truncate_line pair Go already had, replacing four repetitions of the literal 2000. - Express the three path resolvers as pathPolicy values. allowSkillsRoot had been picking the denial message as a side effect, which would misdescribe any future resolver that denied the skills root for a reason other than writability. - Fold TestResolveReadPath_AllowsSymlinkedSkillsDirectory and TestResolveWritePath_BlocksSkillsSymlink into TestResolvePathContainment, which already covered both cells of that matrix. - Correct the grep_file call site's no-timeout rationale, which cited RE2's linearity -- an answer about the match, not the walk that GrepContent's own doc comment identifies as the unbounded part. Signed-off-by: brandonkeung <[email protected]>
1 parent 07e4d62 commit d5fb009

6 files changed

Lines changed: 424 additions & 170 deletions

File tree

‎go/adk/pkg/tools/shell.go‎

Lines changed: 75 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,53 @@ type CommandExecutor struct{}
2020
// truncating. Counted in runes, not bytes -- see truncateRunes.
2121
const maxLineRunes = 2000
2222

23+
// maxLineBytes caps how much of a single line scanFileLines will buffer.
24+
// Lines are emitted truncated to maxLineRunes, but grep has to see the whole
25+
// line to match against it, so the read limit is far larger than the emit
26+
// limit. A line longer than this makes its file unreadable, which callers
27+
// surface as an error (read_file) or as a skip count (grep_file).
28+
const maxLineBytes = 1024 * 1024
29+
30+
// scanFileLines opens path and hands each line to visit, stopping early if
31+
// visit returns false. Line numbers are 1-based.
32+
//
33+
// This is the single reader behind read_file and grep_file. Both need the
34+
// same three guarantees -- reject non-regular files, cap how much of a line
35+
// is buffered, and surface failures wrapped -- and keeping them in one place
36+
// is what stops the two from drifting apart: an earlier revision set the
37+
// scanner buffer only in the grep path, which left read_file failing outright
38+
// on any file with a line over bufio's 64KB default.
39+
func scanFileLines(path string, visit func(lineNum int, line string) bool) error {
40+
// Stat before opening: os.Open on a FIFO with no writer connected blocks
41+
// indefinitely, and nothing on these paths imposes a timeout. Doing it
42+
// here rather than at each call site means no future caller can omit it.
43+
info, err := os.Stat(path)
44+
if err != nil {
45+
return fmt.Errorf("failed to stat %q: %w", path, err)
46+
}
47+
if !info.Mode().IsRegular() {
48+
return fmt.Errorf("%q is not a regular file", path)
49+
}
50+
51+
file, err := os.Open(path)
52+
if err != nil {
53+
return fmt.Errorf("failed to open %q: %w", path, err)
54+
}
55+
defer file.Close()
56+
57+
scanner := bufio.NewScanner(file)
58+
scanner.Buffer(make([]byte, 64*1024), maxLineBytes)
59+
for lineNum := 1; scanner.Scan(); lineNum++ {
60+
if !visit(lineNum, scanner.Text()) {
61+
return nil
62+
}
63+
}
64+
if err := scanner.Err(); err != nil {
65+
return fmt.Errorf("failed to read %q: %w", path, err)
66+
}
67+
return nil
68+
}
69+
2370
// truncateRunes shortens s to at most maxRunes code points, appending "..."
2471
// when it truncates.
2572
//
@@ -82,42 +129,19 @@ func GetSessionPath(sessionID, skillsDirectory string) (string, error) {
82129

83130
// ReadFileContent reads a file with line numbers.
84131
func ReadFileContent(path string, offset, limit int) (string, error) {
85-
// Stat before opening: os.Open on a FIFO with no writer connected blocks
86-
// indefinitely, and nothing on this path imposes a timeout. GrepContent
87-
// rejects non-regular files for the same reason.
88-
info, err := os.Stat(path)
89-
if err != nil {
90-
return "", fmt.Errorf("failed to stat %q: %w", path, err)
91-
}
92-
if !info.Mode().IsRegular() {
93-
return "", fmt.Errorf("%q is not a regular file", path)
94-
}
95-
96-
file, err := os.Open(path)
97-
if err != nil {
98-
return "", fmt.Errorf("failed to open %q: %w", path, err)
99-
}
100-
defer file.Close()
101-
102132
var result strings.Builder
103-
scanner := bufio.NewScanner(file)
104-
lineNum := 1
105133
start := max(offset, 1)
106134
count := 0
107135

108-
for scanner.Scan() {
109-
if lineNum >= start {
110-
line := truncateRunes(scanner.Text(), maxLineRunes)
111-
fmt.Fprintf(&result, "%6d|%s\n", lineNum, line)
112-
count++
113-
if limit > 0 && count >= limit {
114-
break
115-
}
136+
err := scanFileLines(path, func(lineNum int, line string) bool {
137+
if lineNum < start {
138+
return true
116139
}
117-
lineNum++
118-
}
119-
120-
if err := scanner.Err(); err != nil {
140+
fmt.Fprintf(&result, "%6d|%s\n", lineNum, truncateRunes(line, maxLineRunes))
141+
count++
142+
return limit <= 0 || count < limit
143+
})
144+
if err != nil {
121145
return "", err
122146
}
123147

@@ -183,7 +207,7 @@ func EditFileContent(path string, oldString, newString string, replaceAll bool)
183207
func ListDirContent(path string) (string, error) {
184208
entries, err := os.ReadDir(path)
185209
if err != nil {
186-
return "", err
210+
return "", fmt.Errorf("failed to read directory %q: %w", path, err)
187211
}
188212

189213
if len(entries) == 0 {
@@ -302,7 +326,13 @@ func classifyWalkEntry(root, p string, d fs.DirEntry) (walkEntryAction, string)
302326

303327
// GrepContent searches path for lines matching a regular expression pattern.
304328
// If path is a directory, recursive must be true to search its files.
305-
func GrepContent(path, pattern string, recursive, ignoreCase bool) (string, error) {
329+
//
330+
// A recursive search walks the whole tree under path, which is unbounded from
331+
// this function's point of view, so ctx is honored between entries and lets a
332+
// caller abort one. Note Go needs no equivalent of the Python runtime's regex
333+
// timeout: this uses RE2, which is linear-time and cannot backtrack
334+
// catastrophically, so it is the walk rather than the match that can run long.
335+
func GrepContent(ctx context.Context, path, pattern string, recursive, ignoreCase bool) (string, error) {
306336
expr := pattern
307337
if ignoreCase {
308338
expr = "(?i)" + expr
@@ -320,22 +350,12 @@ func GrepContent(path, pattern string, recursive, ignoreCase bool) (string, erro
320350
var result strings.Builder
321351
var skipped int
322352
grepFile := func(filePath string) error {
323-
file, err := os.Open(filePath)
324-
if err != nil {
325-
return err
326-
}
327-
defer file.Close()
328-
329-
scanner := bufio.NewScanner(file)
330-
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
331-
lineNum := 1
332-
for scanner.Scan() {
333-
if line := scanner.Text(); re.MatchString(line) {
353+
return scanFileLines(filePath, func(lineNum int, line string) bool {
354+
if re.MatchString(line) {
334355
fmt.Fprintf(&result, "%s:%d:%s\n", filePath, lineNum, truncateRunes(line, maxLineRunes))
335356
}
336-
lineNum++
337-
}
338-
return scanner.Err()
357+
return true
358+
})
339359
}
340360

341361
if info.IsDir() {
@@ -355,6 +375,11 @@ func GrepContent(path, pattern string, recursive, ignoreCase bool) (string, erro
355375
// symlink, WalkDir would see a non-directory at the root and never
356376
// descend into it at all.
357377
err = filepath.WalkDir(root, func(p string, d fs.DirEntry, walkErr error) error {
378+
// Checked per entry rather than per line: a single file's scan is
379+
// bounded by its size, but the number of entries is not.
380+
if ctxErr := ctx.Err(); ctxErr != nil {
381+
return ctxErr
382+
}
358383
if walkErr != nil {
359384
if p == root {
360385
// The search root itself couldn't be read (e.g.
@@ -385,6 +410,9 @@ func GrepContent(path, pattern string, recursive, ignoreCase bool) (string, erro
385410
return nil
386411
})
387412
} else {
413+
// scanFileLines rejects non-regular files too, but reusing the stat
414+
// already taken above lets an explicitly-targeted FIFO report the
415+
// plain reason rather than one wrapped in "failed to search".
388416
if !info.Mode().IsRegular() {
389417
return "", fmt.Errorf("%q is not a regular file", path)
390418
}

0 commit comments

Comments
 (0)