Skip to content

Commit db2c685

Browse files
[HfFileSystem] Fix bucket prefix collisions (#4630)
* [HfFileSystem] Fix bucket prefix collisions * Update src/huggingface_hub/hf_file_system.py --------- Co-authored-by: célina <[email protected]>
1 parent 0234ea1 commit db2c685

2 files changed

Lines changed: 13 additions & 0 deletions

File tree

‎src/huggingface_hub/hf_file_system.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -663,6 +663,11 @@ def _list_bucket_tree_with_folders(
663663
out: list[BucketFile | BucketFolder] = []
664664

665665
for bucket_entry in bucket_files:
666+
# The server matches `prefix` lexically, so listing "logs" also returns "logs_existing/...".
667+
# Filesystem semantics require path components => drop entries that aren't `prefix` itself
668+
# or below it.
669+
if prefix and bucket_entry.path != prefix and not bucket_entry.path.startswith(f"{prefix}/"):
670+
continue
666671
out.append(bucket_entry)
667672

668673
# If recursive=False, both files and folders are returned by the server => nothing to do

‎tests/test_buckets_hf_file_system.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,14 @@ def _shared_bucket(self, request):
4545
def _new_hffs(self):
4646
self.hffs = HfFileSystem(endpoint=ENDPOINT_STAGING, token=TOKEN, skip_instance_cache=True)
4747

48+
def test_prefix_collision(self):
49+
colliding_path = f"{self.hf_path}/dat"
50+
51+
assert not self.hffs.exists(f"{colliding_path}/new.txt")
52+
assert self.hffs.glob(f"{colliding_path}/*") == []
53+
with pytest.raises(FileNotFoundError):
54+
self.hffs.ls(colliding_path)
55+
4856

4957
class TestHfFileSystemBucketRW(_HfFileSystemBucketChecks, _HfFileSystemBaseRWTests):
5058
__test__ = True

0 commit comments

Comments
 (0)