Skip to content

Commit d73996b

Browse files
Preserve searchable siblings of invalid filenames
1 parent cb9f955 commit d73996b

3 files changed

Lines changed: 169 additions & 36 deletions

File tree

‎packages/codex/patches/0060-bounded-nofollow-filesystem.patch‎

Lines changed: 137 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -334,10 +334,10 @@ index d30766370c5962861ea2ae538a82d310cef63890..0a865812a441af89ead939bc120f8779
334334
.walk(&params.path, params.options, /*sandbox*/ None)
335335
.await
336336
diff --git a/codex-rs/exec-server/src/local_file_system.rs b/codex-rs/exec-server/src/local_file_system.rs
337-
index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce1d3d08fb 100644
337+
index ec8b81c52a3a4a96d47e413af388a1873f49c676..e019be54400dacdb62f19b3a38c79087d893111a 100644
338338
--- a/codex-rs/exec-server/src/local_file_system.rs
339339
+++ b/codex-rs/exec-server/src/local_file_system.rs
340-
@@ -703,6 +703,106 @@ impl DirectFileSystem {
340+
@@ -703,6 +703,120 @@ impl DirectFileSystem {
341341
Ok(entries)
342342
}
343343

@@ -385,6 +385,20 @@ index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce
385385
+ for name in names.into_iter().take(remaining) {
386386
+ check_walk_cancelled(cancelled)?;
387387
+ examined += 1;
388+
+ let name = match name {
389+
+ Ok(name) => name,
390+
+ Err(error) => {
391+
+ if !push_walk_error(
392+
+ &mut outcome,
393+
+ &mut response_bytes,
394+
+ directory.clone(),
395+
+ error.to_string(),
396+
+ ) {
397+
+ return Ok(outcome);
398+
+ }
399+
+ continue;
400+
+ }
401+
+ };
388402
+ let path = directory.join(&name).map_err(io::Error::other)?;
389403
+ let native = path.to_abs_path()?;
390404
+ let metadata = match no_follow::walk_metadata_sync(native.as_path()) {
@@ -444,7 +458,7 @@ index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce
444458
fn sync_walk(
445459
root: &PathUri,
446460
options: WalkOptions,
447-
@@ -727,6 +827,9 @@ impl DirectFileSystem {
461+
@@ -727,6 +841,9 @@ impl DirectFileSystem {
448462
}
449463

450464
check_walk_cancelled(cancelled)?;
@@ -454,28 +468,28 @@ index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce
454468
let (root_metadata, root_is_symlink) = walk_metadata(root)?;
455469
if !root_metadata.is_dir() || (root_is_symlink && !options.follow_directory_symlinks) {
456470
return Ok(WalkOutcome::default());
457-
@@ -1341,6 +1444,7 @@ mod walk_tests {
471+
@@ -1341,6 +1458,7 @@ mod walk_tests {
458472
root.clone(),
459473
);
460474
let options = WalkOptions {
461475
+ no_follow: false,
462476
max_depth: 1,
463477
max_directories: 1,
464478
max_entries: 1,
465-
@@ -1365,6 +1469,7 @@ mod walk_tests {
479+
@@ -1365,6 +1483,7 @@ mod walk_tests {
466480
let temp = tempfile::tempdir()?;
467481
let missing = PathUri::from_host_native_path(temp.path().join("missing"))?;
468482
let options = WalkOptions {
469483
+ no_follow: false,
470484
max_depth: 1,
471485
max_directories: 1,
472486
max_entries: 1,
473-
@@ -1421,3 +1526,92 @@ mod walk_tests {
487+
@@ -1421,3 +1540,172 @@ mod walk_tests {
474488
Ok(())
475489
}
476490
}
477491
+
478-
+#[cfg(all(test, unix))]
492+
+#[cfg(test)]
479493
+mod project_search_walk_tests {
480494
+ use super::*;
481495
+ fn options() -> WalkOptions {
@@ -488,6 +502,84 @@ index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce
488502
+ prune_hidden_directories: false,
489503
+ }
490504
+ }
505+
+ fn invalid_filename(index: usize) -> std::ffi::OsString {
506+
+ let name = format!("invalid-{index}-");
507+
+ #[cfg(unix)]
508+
+ {
509+
+ use std::os::unix::ffi::OsStringExt;
510+
+ let mut bytes = name.into_bytes();
511+
+ bytes.push(0xff);
512+
+ std::ffi::OsString::from_vec(bytes)
513+
+ }
514+
+ #[cfg(windows)]
515+
+ {
516+
+ use std::os::windows::ffi::OsStringExt;
517+
+ let units: Vec<u16> = name.encode_utf16().chain([0xd800]).collect();
518+
+ std::ffi::OsString::from_wide(&units)
519+
+ }
520+
+ }
521+
+
522+
+ #[test]
523+
+ fn invalid_filename_preserves_valid_siblings_and_descendants() -> io::Result<()> {
524+
+ let temp = tempfile::tempdir()?;
525+
+ let base = temp.path().canonicalize()?;
526+
+ std::fs::write(base.join("good.txt"), "needle")?;
527+
+ std::fs::write(base.join(invalid_filename(0)), "unrepresentable")?;
528+
+ std::fs::create_dir(base.join("nested"))?;
529+
+ std::fs::write(base.join("nested/match.txt"), "needle")?;
530+
+ let root = PathUri::from_host_native_path(&base)?;
531+
+ let result = DirectFileSystem::sync_walk(&root, options(), &CancellationToken::new())?;
532+
+ assert_eq!(result.entries.len(), 3);
533+
+ for name in ["good.txt", "nested", "nested/match.txt"] {
534+
+ let expected = root.join(name).unwrap();
535+
+ assert!(result.entries.iter().any(|entry| entry.path == expected));
536+
+ }
537+
+ assert_eq!(result.errors.len(), 1);
538+
+ assert_eq!(result.errors[0].path, root);
539+
+ assert!(
540+
+ !result.truncated,
541+
+ "the error is not an entry-limit truncation"
542+
+ );
543+
+ Ok(())
544+
+ }
545+
+
546+
+ #[test]
547+
+ fn invalid_filenames_consume_the_entry_budget() -> io::Result<()> {
548+
+ let temp = tempfile::tempdir()?;
549+
+ let base = temp.path().canonicalize()?;
550+
+ for index in 0..8 {
551+
+ std::fs::write(base.join(invalid_filename(index)), "unrepresentable")?;
552+
+ }
553+
+ let root = PathUri::from_host_native_path(&base)?;
554+
+ let mut limits = options();
555+
+ limits.max_entries = 3;
556+
+ let result = DirectFileSystem::sync_walk(&root, limits, &CancellationToken::new())?;
557+
+ assert!(result.entries.is_empty());
558+
+ assert_eq!(result.errors.len(), 3);
559+
+ assert!(result.errors.iter().all(|error| error.path == root));
560+
+ assert!(result.truncated);
561+
+ let entries = no_follow::read_directory_bounded(&base, 4, &CancellationToken::new())?;
562+
+ assert_eq!(entries.len(), 4, "including only the bounded extra probe");
563+
+ Ok(())
564+
+ }
565+
+
566+
+ #[test]
567+
+ fn invalid_filename_cancellation_and_setup_failures_stay_outer() -> io::Result<()> {
568+
+ let temp = tempfile::tempdir()?;
569+
+ let base = temp.path().canonicalize()?;
570+
+ std::fs::write(base.join(invalid_filename(0)), "unrepresentable")?;
571+
+ let cancelled = CancellationToken::new();
572+
+ cancelled.cancel();
573+
+ let error = no_follow::read_directory_bounded(&base, 4, &cancelled).unwrap_err();
574+
+ assert_eq!(error.kind(), io::ErrorKind::Interrupted);
575+
+ assert!(
576+
+ no_follow::read_directory_bounded(&base.join("missing"), 4, &CancellationToken::new(),)
577+
+ .is_err()
578+
+ );
579+
+ Ok(())
580+
+ }
581+
+
582+
+ #[cfg(unix)]
491583
+ #[test]
492584
+ fn nofollow_walk_bounds_wide_directories_and_reports_depth() -> io::Result<()> {
493585
+ let temp = tempfile::tempdir()?;
@@ -514,6 +606,7 @@ index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce
514606
+ );
515607
+ Ok(())
516608
+ }
609+
+ #[cfg(unix)]
517610
+ #[test]
518611
+ fn nofollow_walk_rejects_root_symlink_ancestors() -> io::Result<()> {
519612
+ let temp = tempfile::tempdir()?;
@@ -524,6 +617,7 @@ index ec8b81c52a3a4a96d47e413af388a1873f49c676..87c6e36cdf4e48cc2a7ce74db2259fce
524617
+ assert!(DirectFileSystem::sync_walk(&root, options(), &CancellationToken::new()).is_err());
525618
+ Ok(())
526619
+ }
620+
+ #[cfg(unix)]
527621
+ #[test]
528622
+ fn nofollow_walk_never_reads_swapped_ancestor() -> io::Result<()> {
529623
+ let temp = tempfile::tempdir()?;
@@ -669,29 +763,31 @@ index 354a9dab58c7eabae8cfaee910a0427db6428e0c..3cb0e13a670b4f448e1e0c725d8a669b
669763
+ Ok(())
670764
+}
671765
diff --git a/codex-rs/exec-server/src/no_follow/mod.rs b/codex-rs/exec-server/src/no_follow/mod.rs
672-
index d7abd7ff9c7b38a8dd44fd266b103a8093452432..b66985045b8d6d366e24cc53e427c51ec7435eea 100644
766+
index d7abd7ff9c7b38a8dd44fd266b103a8093452432..c095c1483ff3d7c4f252d2c832ac06cf3f18462c 100644
673767
--- a/codex-rs/exec-server/src/no_follow/mod.rs
674768
+++ b/codex-rs/exec-server/src/no_follow/mod.rs
675-
@@ -58,3 +58,14 @@ pub(crate) async fn create_directory(path: &Path, recursive: bool) -> io::Result
769+
@@ -58,3 +58,16 @@ pub(crate) async fn create_directory(path: &Path, recursive: bool) -> io::Result
676770
pub(crate) async fn remove(path: &Path, recursive: bool, force: bool) -> io::Result<()> {
677771
imp::remove(path.to_path_buf(), recursive, force).await
678772
}
679773
+
774+
+/// Each item, including an unrepresentable name or enumeration error, consumes
775+
+/// one entry slot. Setup and cancellation failures remain outer errors.
680776
+pub(crate) fn read_directory_bounded(
681777
+ path: &Path,
682778
+ limit: usize,
683779
+ cancelled: &tokio_util::sync::CancellationToken,
684-
+) -> io::Result<Vec<String>> {
780+
+) -> io::Result<Vec<io::Result<String>>> {
685781
+ imp::read_directory_bounded(path, limit, cancelled)
686782
+}
687783
+pub(crate) fn walk_metadata_sync(path: &Path) -> io::Result<FileMetadata> {
688784
+ imp::walk_metadata_sync(path)
689785
+}
690786
diff --git a/codex-rs/exec-server/src/no_follow/unix.rs b/codex-rs/exec-server/src/no_follow/unix.rs
691-
index 330a316ab03fa9fe5bb122beccbd4f7527056e8b..528dd5e8592683b6724c7ba78bdef3b66fb9a93f 100644
787+
index 330a316ab03fa9fe5bb122beccbd4f7527056e8b..2b237e256c2654007c445816359b4c55a15f6960 100644
692788
--- a/codex-rs/exec-server/src/no_follow/unix.rs
693789
+++ b/codex-rs/exec-server/src/no_follow/unix.rs
694-
@@ -321,3 +321,45 @@ pub(super) async fn remove(path: PathBuf, recursive: bool, force: bool) -> io::R
790+
@@ -321,3 +321,53 @@ pub(super) async fn remove(path: PathBuf, recursive: bool, force: bool) -> io::R
695791
.await
696792
.map_err(|error| io::Error::other(format!("filesystem task failed: {error}")))?
697793
}
@@ -701,7 +797,7 @@ index 330a316ab03fa9fe5bb122beccbd4f7527056e8b..528dd5e8592683b6724c7ba78bdef3b6
701797
+ path: &Path,
702798
+ limit: usize,
703799
+ cancelled: &tokio_util::sync::CancellationToken,
704-
+) -> io::Result<Vec<String>> {
800+
+) -> io::Result<Vec<io::Result<String>>> {
705801
+ let mut parent = root()?;
706802
+ for component in components(path)? {
707803
+ parent = open_directory(&parent, &component)?;
@@ -715,21 +811,29 @@ index 330a316ab03fa9fe5bb122beccbd4f7527056e8b..528dd5e8592683b6724c7ba78bdef3b6
715811
+ .map_err(io::Error::from)?;
716812
+ let mut dir = rustix::fs::Dir::new(fd).map_err(io::Error::from)?;
717813
+ let mut names = Vec::new();
718-
+ for entry in &mut dir {
814+
+ while names.len() < limit {
719815
+ if cancelled.is_cancelled() {
720816
+ return Err(io::Error::new(io::ErrorKind::Interrupted, "walk cancelled"));
721817
+ }
722-
+ let entry = entry.map_err(io::Error::from)?;
818+
+ let Some(entry) = dir.next() else {
819+
+ break;
820+
+ };
821+
+ let entry = match entry {
822+
+ Ok(entry) => entry,
823+
+ Err(error) => {
824+
+ names.push(Err(io::Error::from(error)));
825+
+ break;
826+
+ }
827+
+ };
723828
+ let name = entry.file_name().to_bytes();
724829
+ if name == b"." || name == b".." {
725830
+ continue;
726831
+ }
727-
+ let name = std::str::from_utf8(name)
728-
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "non UTF-8 filename"))?;
729-
+ names.push(name.to_owned());
730-
+ if names.len() >= limit {
731-
+ break;
732-
+ }
832+
+ names.push(
833+
+ std::str::from_utf8(name)
834+
+ .map(str::to_owned)
835+
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "non UTF-8 filename")),
836+
+ );
733837
+ }
734838
+ Ok(names)
735839
+}
@@ -738,10 +842,10 @@ index 330a316ab03fa9fe5bb122beccbd4f7527056e8b..528dd5e8592683b6724c7ba78bdef3b6
738842
+ metadata_sync(path)
739843
+}
740844
diff --git a/codex-rs/exec-server/src/no_follow/windows.rs b/codex-rs/exec-server/src/no_follow/windows.rs
741-
index 0a668826785ee34b7ccfd7b0b4e2014040a64777..37740fae4eafc0fc89e4bcc274d2bbef61fe40c4 100644
845+
index 0a668826785ee34b7ccfd7b0b4e2014040a64777..9cb3955cf22df366ab55edb72bb0356857cbc2d6 100644
742846
--- a/codex-rs/exec-server/src/no_follow/windows.rs
743847
+++ b/codex-rs/exec-server/src/no_follow/windows.rs
744-
@@ -321,3 +321,100 @@ pub(super) async fn remove(path: PathBuf, recursive: bool, force: bool) -> io::R
848+
@@ -321,3 +321,99 @@ pub(super) async fn remove(path: PathBuf, recursive: bool, force: bool) -> io::R
745849
#[cfg(test)]
746850
#[path = "windows_tests.rs"]
747851
mod tests;
@@ -751,7 +855,7 @@ index 0a668826785ee34b7ccfd7b0b4e2014040a64777..37740fae4eafc0fc89e4bcc274d2bbef
751855
+ path: &Path,
752856
+ limit: usize,
753857
+ cancelled: &tokio_util::sync::CancellationToken,
754-
+) -> io::Result<Vec<String>> {
858+
+) -> io::Result<Vec<io::Result<String>>> {
755859
+ #[link(name = "ntdll")]
756860
+ unsafe extern "system" {
757861
+ fn NtQueryDirectoryFile(
@@ -772,7 +876,7 @@ index 0a668826785ee34b7ccfd7b0b4e2014040a64777..37740fae4eafc0fc89e4bcc274d2bbef
772876
+ let mut names = Vec::new();
773877
+ // FILE_DIRECTORY_INFORMATION: filename byte length at offset 60, UTF-16 at 64.
774878
+ let mut buffer = vec![0u64; 8192];
775-
+ loop {
879+
+ while names.len() < limit {
776880
+ if cancelled.is_cancelled() {
777881
+ return Err(io::Error::new(io::ErrorKind::Interrupted, "walk cancelled"));
778882
+ }
@@ -799,32 +903,31 @@ index 0a668826785ee34b7ccfd7b0b4e2014040a64777..37740fae4eafc0fc89e4bcc274d2bbef
799903
+ break;
800904
+ } // STATUS_NO_MORE_FILES
801905
+ if result < 0 {
802-
+ return Err(io::Error::from_raw_os_error(
906+
+ names.push(Err(io::Error::from_raw_os_error(
803907
+ unsafe { RtlNtStatusToDosError(result) } as i32,
804-
+ ));
908+
+ )));
909+
+ break;
805910
+ }
806911
+ let bytes =
807912
+ unsafe { std::slice::from_raw_parts(buffer.as_ptr().cast::<u8>(), buffer.len() * 8) };
808913
+ let length = u32::from_ne_bytes(bytes[60..64].try_into().unwrap()) as usize;
809914
+ if length % 2 != 0 || length + 64 > status.Information || length + 64 > bytes.len() {
810-
+ return Err(io::Error::new(
915+
+ names.push(Err(io::Error::new(
811916
+ io::ErrorKind::InvalidData,
812917
+ "invalid directory entry",
813-
+ ));
918+
+ )));
919+
+ break;
814920
+ }
815921
+ let units: Vec<u16> = bytes[64..64 + length]
816922
+ .chunks_exact(2)
817923
+ .map(|v| u16::from_ne_bytes([v[0], v[1]]))
818924
+ .collect();
819925
+ let name = String::from_utf16(&units)
820-
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid filename"))?;
821-
+ if name == "." || name == ".." {
926+
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid filename"));
927+
+ if matches!(name.as_deref(), Ok("." | "..")) {
822928
+ continue;
823929
+ }
824930
+ names.push(name);
825-
+ if names.len() >= limit {
826-
+ break;
827-
+ }
828931
+ }
829932
+ Ok(names)
830933
+}

‎packages/codex/patches/series.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,12 +266,12 @@
266266
},
267267
{
268268
"path": "0060-bounded-nofollow-filesystem.patch",
269-
"sha256": "d2048388044ec4004ba13d1111b2da602a401fb4c572280ae24474cd6f480bc1"
269+
"sha256": "ae328ba82ae5c1bb7e70a40710ad423105b24fda28a6e64db5e0bd0f5273d073"
270270
},
271271
{
272272
"path": "0061-early-project-search-result-policy.patch",
273273
"sha256": "ac413ec794493e2047724a5ee0473744c4a0464313d4330ba6975780bda6e5d3"
274274
}
275275
],
276-
"result_tree": "523577a78bb75d7334cd0f2e6dba82e5250bc552"
276+
"result_tree": "b6035054c0d8fdbf94ff686c21c994816dea6f91"
277277
}

‎packages/codex/tests/test_native_project_search.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,36 @@ async def test_complete_no_match_is_not_truncation(
185185
assert result["errors"] == []
186186

187187

188+
@local_search
189+
@pytest.mark.skipif(os.name != "posix", reason="Requires raw Unix filename bytes")
190+
@pytest.mark.parametrize("name", ["project_glob", "project_grep"])
191+
async def test_invalid_filename_preserves_valid_siblings_and_descendants(
192+
responses: ResponsesFixture, tmp_path: Path, name: ExecutorToolName
193+
) -> None:
194+
(tmp_path / "valid.txt").write_text("needle root\n", encoding="utf-8")
195+
nested = tmp_path / "nested"
196+
nested.mkdir()
197+
(nested / "valid.txt").write_text("needle nested\n", encoding="utf-8")
198+
with open(os.fsencode(tmp_path) + b"/invalid-\xff", "wb") as invalid:
199+
invalid.write(b"unrelated\n")
200+
201+
result = await run_search(
202+
responses,
203+
Environment.local(cwd=tmp_path, tools=[name]),
204+
name,
205+
{"pattern": "**/*.txt" if name == "project_glob" else "needle"},
206+
)
207+
paths = (
208+
result["matches"]
209+
if name == "project_glob"
210+
else [match["path"] for match in result["matches"]]
211+
)
212+
assert set(paths) == {"valid.txt", "nested/valid.txt"}
213+
assert result["errors"] == [{"path": "", "code": "path_unavailable"}]
214+
assert result["truncated"] is True
215+
assert result["truncation_reasons"] == ["read_errors"]
216+
217+
188218
@local_search
189219
async def test_result_limit_is_explicit(responses: ResponsesFixture, tmp_path: Path) -> None:
190220
(tmp_path / "many.txt").write_text("needle\n" * 2000)

0 commit comments

Comments
 (0)