Skip to content

Commit 04c5686

Browse files
authored
Merge pull request #266 from alei-jia/fix/fix-grep-file-size-limit
fix(filesystem): enforce grep file size limits
2 parents d461923 + e982cc1 commit 04c5686

4 files changed

Lines changed: 303 additions & 2 deletions

File tree

‎.github/workflows/build-and-test.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ jobs:
5252
steps:
5353
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
5454
- uses: ./tools/github-actions/setup-deps
55+
- name: Install ripgrep
56+
run: |
57+
sudo apt-get update
58+
sudo apt-get install --yes ripgrep
5559
- run: make tools
5660
- run: ./mvnw -B install
5761
build:

‎argi-orchestration/argi-agent-framework/src/main/java/io/github/agentic/ai/agent/orchestration/extension/file/LocalFilesystemBackend.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -521,18 +521,30 @@ public Object grepRaw(String pattern, String path, String glob) {
521521
// Helper methods
522522

523523
private Map<String, List<LineMatch>> ripgrepSearch(String pattern, Path baseFull, String includeGlob) {
524+
boolean explicitFile = Files.isRegularFile(baseFull, LinkOption.NOFOLLOW_LINKS);
524525
List<String> cmd = new ArrayList<>();
525526
cmd.add("rg");
526527
cmd.add("--json");
527-
if (includeGlob != null) {
528+
cmd.add("--max-filesize");
529+
cmd.add(Long.toString(maxFileSizeBytes));
530+
if (explicitFile) {
531+
// Explicit file arguments bypass rg's size filter. Select the file through directory traversal instead.
532+
cmd.add("--max-depth");
533+
cmd.add("1");
534+
cmd.add("--glob");
535+
cmd.add(literalRipgrepGlob(baseFull.getFileName().toString()));
536+
} else if (includeGlob != null) {
528537
cmd.add("--glob");
529538
cmd.add(includeGlob);
530539
}
531540
cmd.add("--");
532541
cmd.add(pattern);
533-
cmd.add(baseFull.toString());
542+
cmd.add((explicitFile ? baseFull.getParent() : baseFull).toString());
534543

535544
try {
545+
if (explicitFile && Files.size(baseFull) > maxFileSizeBytes) {
546+
return Collections.emptyMap();
547+
}
536548
ProcessBuilder pb = new ProcessBuilder(cmd);
537549
Process process = pb.start();
538550

@@ -590,6 +602,17 @@ private Map<String, List<LineMatch>> ripgrepSearch(String pattern, Path baseFull
590602
}
591603
}
592604

605+
private String literalRipgrepGlob(String filename) {
606+
StringBuilder glob = new StringBuilder();
607+
for (char character : filename.toCharArray()) {
608+
if ("\\*?[]{}!".indexOf(character) >= 0) {
609+
glob.append('\\');
610+
}
611+
glob.append(character);
612+
}
613+
return glob.toString();
614+
}
615+
593616
private Map<String, List<LineMatch>> javaSearch(String pattern, Path baseFull, String includeGlob) {
594617
Pattern regex;
595618
try {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Copyright 2026 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.github.agentic.ai.agent.orchestration.extension.file;
17+
18+
import java.io.File;
19+
import java.nio.charset.StandardCharsets;
20+
import java.nio.file.Files;
21+
import java.nio.file.Path;
22+
import java.nio.file.StandardOpenOption;
23+
import java.nio.file.attribute.PosixFilePermissions;
24+
import java.util.List;
25+
import java.util.concurrent.TimeUnit;
26+
27+
import org.junit.jupiter.api.condition.EnabledOnOs;
28+
import org.junit.jupiter.api.condition.OS;
29+
import org.junit.jupiter.api.io.TempDir;
30+
import org.junit.jupiter.params.ParameterizedTest;
31+
import org.junit.jupiter.params.provider.ValueSource;
32+
33+
import static org.junit.jupiter.api.Assertions.assertEquals;
34+
import static org.junit.jupiter.api.Assertions.assertTrue;
35+
36+
/** 验证后端检查文件后、真实 rg 搜索前,文件增长越过大小上限的情况。 */
37+
@EnabledOnOs({ OS.MAC, OS.LINUX })
38+
public class LocalFilesystemBackendFileGrowthTest {
39+
40+
private static final String MARKER = "SIZE_LIMIT_MARKER";
41+
42+
@TempDir
43+
Path root;
44+
45+
/** 验证显式搜索只返回目标文件,兼容隐藏文件、忽略规则和文件名中的特殊字符。 */
46+
@ParameterizedTest
47+
@ValueSource(strings = { "plain.txt", ".hidden[?]*{x}.txt", "!special.txt", "space name.txt", "back\\slash.txt" })
48+
void explicitSearchKeepsItsOriginalFileScope(String name) throws Exception {
49+
findRipgrep();
50+
Files.writeString(this.root.resolve(name), MARKER + "\n");
51+
Files.writeString(this.root.resolve("other.txt"), MARKER + "\n");
52+
Files.writeString(Files.createDirectory(this.root.resolve("nested")).resolve(name), MARKER + "\n");
53+
Files.writeString(this.root.resolve(".ignore"), "*\n");
54+
final LocalFilesystemBackend backend = new LocalFilesystemBackend(this.root.toString(), true, 1);
55+
final Object result = backend.grepRaw(MARKER, "/" + name, null);
56+
assertTrue(result instanceof List<?>, "Expected a match list");
57+
final List<?> matches = (List<?>) result;
58+
assertEquals(1, matches.size(), "Explicit search must not include sibling or nested files");
59+
assertTrue(matches.get(0) instanceof GrepMatch);
60+
assertEquals("/" + name.replace("\\", "/"), ((GrepMatch) matches.get(0)).getPath());
61+
}
62+
63+
/** 用信号文件同步扩容与搜索,分别验证目录输入和显式文件输入。 */
64+
@ParameterizedTest(name = "explicitFile={0}")
65+
@ValueSource(booleans = { false, true })
66+
void skipsFileThatGrowsAfterBackendCheck(boolean explicitFile) throws Exception {
67+
final Path realRipgrep = findRipgrep();
68+
final Path workspace = Files.createDirectory(this.root.resolve("workspace"));
69+
final Path growingFile = Files.writeString(workspace.resolve("growing.txt"), "initial content\n");
70+
assertTrue(Files.size(growingFile) < 1024L * 1024L);
71+
final Path ready = this.root.resolve("rg-ready");
72+
final Path release = this.root.resolve("rg-release");
73+
final Path result = this.root.resolve("result.txt");
74+
final Path log = this.root.resolve("probe.log");
75+
final Path bin = createRipgrepGate();
76+
final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
77+
final String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"));
78+
final ProcessBuilder builder = new ProcessBuilder(java, "-cp", classpath,
79+
LocalFilesystemBackendFileGrowthTest.class.getName(), workspace.toString(),
80+
Boolean.toString(explicitFile), result.toString());
81+
builder.environment().put("PATH", bin.toString());
82+
builder.environment().put("ARGI_GROWTH_READY", ready.toString());
83+
builder.environment().put("ARGI_GROWTH_RELEASE", release.toString());
84+
builder.environment().put("ARGI_GROWTH_REAL_RG", realRipgrep.toString());
85+
builder.environment().remove("RIPGREP_CONFIG_PATH");
86+
builder.redirectErrorStream(true).redirectOutput(log.toFile());
87+
final Process process = builder.start();
88+
try {
89+
awaitRipgrepGate(ready, process, log);
90+
Files.writeString(growingFile, "padding\n".repeat(150_000) + MARKER + "\n", StandardCharsets.UTF_8,
91+
StandardOpenOption.APPEND);
92+
assertTrue(Files.size(growingFile) > 1024L * 1024L);
93+
Files.writeString(release, "continue");
94+
assertTrue(process.waitFor(15, TimeUnit.SECONDS), "Probe JVM did not finish after releasing rg");
95+
assertEquals(0, process.exitValue(), Files.readString(log));
96+
assertTrue(Files.isRegularFile(result), "Probe did not produce a search result");
97+
assertEquals("matches=0", Files.readString(result),
98+
"The file grew beyond 1 MiB after the backend check; explicitFile=" + explicitFile);
99+
}
100+
finally {
101+
process.descendants().forEach(ProcessHandle::destroyForcibly);
102+
process.destroyForcibly();
103+
assertTrue(process.waitFor(5, TimeUnit.SECONDS), "Probe JVM did not stop");
104+
}
105+
}
106+
107+
/** 定位 PATH 中真实可执行的 rg,缺失时明确指出测试环境不满足要求。 */
108+
private static Path findRipgrep() throws Exception {
109+
final String path = System.getenv().getOrDefault("PATH", "");
110+
for (String directory : path.split(File.pathSeparator)) {
111+
final Path candidate = Path.of(directory, "rg");
112+
if (Files.isRegularFile(candidate) && Files.isExecutable(candidate)) {
113+
return candidate.toRealPath();
114+
}
115+
}
116+
throw new IllegalStateException("Install ripgrep and add its directory to the test JVM PATH");
117+
}
118+
119+
/** 创建测试专用 rg 包装脚本,收到放行信号后以原参数执行真实 rg。 */
120+
private Path createRipgrepGate() throws Exception {
121+
final Path bin = Files.createDirectory(this.root.resolve("bin"));
122+
final Path wrapper = Files.writeString(bin.resolve("rg"), """
123+
#!/bin/sh
124+
set -eu
125+
: > "$ARGI_GROWTH_READY"
126+
attempt=0
127+
while [ ! -f "$ARGI_GROWTH_RELEASE" ]; do
128+
if [ "$attempt" -ge 2000 ]; then
129+
exit 70
130+
fi
131+
/bin/sleep 0.01
132+
attempt=$((attempt + 1))
133+
done
134+
exec "$ARGI_GROWTH_REAL_RG" "$@"
135+
""", StandardCharsets.UTF_8);
136+
Files.setPosixFilePermissions(wrapper, PosixFilePermissions.fromString("rwx------"));
137+
return bin;
138+
}
139+
140+
/** 等待 rg 包装脚本发出就绪信号,不通过固定延时猜测文件检查时序。 */
141+
private static void awaitRipgrepGate(Path ready, Process process, Path log) throws Exception {
142+
final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15);
143+
while (!Files.exists(ready)) {
144+
assertTrue(process.isAlive(), "Probe exited before reaching rg: " + Files.readString(log));
145+
assertTrue(System.nanoTime() < deadline, "Timed out waiting for rg gate");
146+
Thread.sleep(10);
147+
}
148+
}
149+
150+
/** 子 JVM 入口:以 1 MiB 上限调用真实后端,并将搜索匹配数量写入结果文件。 */
151+
public static void main(String[] args) throws Exception {
152+
final LocalFilesystemBackend backend = new LocalFilesystemBackend(args[0], true, 1);
153+
final boolean explicitFile = Boolean.parseBoolean(args[1]);
154+
final Object result = backend.grepRaw(MARKER, explicitFile ? "/growing.txt" : "/", "growing.txt");
155+
if (!(result instanceof List<?> matches)) {
156+
throw new IllegalStateException("Expected grepRaw to return a match list");
157+
}
158+
Files.writeString(Path.of(args[2]), "matches=" + matches.size(), StandardCharsets.UTF_8);
159+
}
160+
161+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Copyright 2026 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package io.github.agentic.ai.agent.orchestration.extension.file;
17+
18+
import java.nio.charset.StandardCharsets;
19+
import java.nio.file.Files;
20+
import java.nio.file.Path;
21+
import java.util.List;
22+
import java.util.concurrent.TimeUnit;
23+
24+
import org.junit.jupiter.params.ParameterizedTest;
25+
import org.junit.jupiter.params.provider.ValueSource;
26+
import org.junit.jupiter.api.io.TempDir;
27+
28+
import static org.junit.jupiter.api.Assertions.assertEquals;
29+
import static org.junit.jupiter.api.Assertions.assertTrue;
30+
31+
/** 使用真实文件和两个独立 JVM,复现 #231:rg 搜索绕过单文件大小限制。 */
32+
public class LocalFilesystemBackendSizeLimitReproductionTest {
33+
34+
private static final String MARKER = "SIZE_LIMIT_MARKER";
35+
36+
@TempDir
37+
Path root;
38+
39+
/** 验证目录与显式文件输入在大小低于、等于、超过 1 MiB 时的行为。 */
40+
@ParameterizedTest(name = "explicitFile={0}")
41+
@ValueSource(booleans = { false, true })
42+
void oversizedFileMustBeSkippedWithAndWithoutRipgrep(boolean explicitFile) throws Exception {
43+
final String marker = "\n" + MARKER + "\n";
44+
final String padding = "x\n".repeat(524_289);
45+
final String[] names = { "below.txt", "at.txt", "above.txt" };
46+
for (int i = 0; i < names.length; i++) {
47+
final int size = 1024 * 1024 + i - 1;
48+
final Path file = this.root.resolve(names[i]);
49+
Files.writeString(file, padding.substring(0, size - marker.length()) + marker, StandardCharsets.UTF_8);
50+
assertEquals(size, Files.size(file));
51+
}
52+
final String fallback = probe(false, explicitFile);
53+
final String ripgrep = probe(true, explicitFile);
54+
assertTrue(ripgrep.contains("rg=true;"), "rg is required on the test JVM PATH: " + ripgrep);
55+
assertTrue(fallback.contains("rg=false;below.txt=1;at.txt=1;above.txt=0"), fallback);
56+
assertTrue(ripgrep.contains("rg=true;below.txt=1;at.txt=1;above.txt=0"),
57+
"Issue #231: both backends must enforce the same size limit.\nJava: " + fallback + "\nrg: " + ripgrep);
58+
}
59+
60+
/** 在独立 JVM 中运行搜索,通过 PATH 控制是否使用 rg,并返回搜索结果。 */
61+
private String probe(boolean useRipgrep, boolean explicitFile) throws Exception {
62+
final String java = Path.of(System.getProperty("java.home"), "bin", "java").toString();
63+
final String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path"));
64+
final Path log = this.root.resolve(useRipgrep ? "rg.log" : "java.log");
65+
final ProcessBuilder builder = new ProcessBuilder(java, "-cp", classpath,
66+
LocalFilesystemBackendSizeLimitReproductionTest.class.getName(), this.root.toString(), Boolean.toString(explicitFile));
67+
if (!useRipgrep) {
68+
builder.environment().put("PATH", Files.createDirectory(this.root.resolve("empty-path")).toString());
69+
}
70+
builder.redirectErrorStream(true).redirectOutput(log.toFile());
71+
final Process process = builder.start();
72+
try {
73+
assertTrue(process.waitFor(30, TimeUnit.SECONDS), "Probe JVM timed out");
74+
final String output = Files.readString(log);
75+
assertEquals(0, process.exitValue(), output);
76+
return output;
77+
}
78+
finally {
79+
process.destroyForcibly();
80+
}
81+
}
82+
83+
/** 子 JVM 入口:创建文件后端,搜索大小文件并输出匹配数量。 */
84+
public static void main(String[] args) throws Exception {
85+
boolean hasRipgrep = false;
86+
try {
87+
final Process version = new ProcessBuilder("rg", "--version")
88+
.redirectOutput(ProcessBuilder.Redirect.DISCARD)
89+
.redirectError(ProcessBuilder.Redirect.DISCARD)
90+
.start();
91+
try {
92+
hasRipgrep = version.waitFor(5, TimeUnit.SECONDS) && version.exitValue() == 0;
93+
}
94+
finally {
95+
version.destroyForcibly();
96+
}
97+
}
98+
catch (java.io.IOException ignored) {
99+
}
100+
final LocalFilesystemBackend backend = new LocalFilesystemBackend(args[0], true, 1);
101+
final boolean explicitFile = Boolean.parseBoolean(args[1]);
102+
final StringBuilder result = new StringBuilder("rg=" + hasRipgrep);
103+
for (String name : List.of("below.txt", "at.txt", "above.txt")) {
104+
final Object matches = backend.grepRaw(MARKER, explicitFile ? "/" + name : "/", name);
105+
if (!(matches instanceof List<?> matchList)) {
106+
throw new IllegalStateException("Expected grepRaw to return a match list");
107+
}
108+
result.append(';').append(name).append('=').append(matchList.size());
109+
}
110+
System.out.println(result);
111+
}
112+
113+
}

0 commit comments

Comments
 (0)