Skip to content

Commit 4adecd3

Browse files
authored
tar: make members with duplicate slashes reachable via ls/find/glob (#1947) (#2063)
1 parent 59f98d7 commit 4adecd3

2 files changed

Lines changed: 61 additions & 10 deletions

File tree

‎fsspec/implementations/tar.py‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import logging
2+
import re
23
import tarfile
34

45
import fsspec
@@ -94,8 +95,11 @@ def _index(self):
9495
for ti in self.tar:
9596
info = ti.get_info()
9697
info["type"] = typemap.get(info["type"], "file")
97-
name = ti.get_info()["name"].rstrip("/")
98-
out[name] = (info, ti.offset_data)
98+
orig_name = info["name"].rstrip("/")
99+
# Collapse duplicate slashes in the filesystem-facing name.
100+
name = re.sub("/+", "/", orig_name)
101+
info["name"] = name
102+
out[name] = (info, ti.offset_data, orig_name)
99103

100104
self.index = out
101105
# TODO: save index to self.index_store here, if set
@@ -107,21 +111,21 @@ def _get_dirs(self):
107111
# This enables ls to get directories as children as well as files
108112
self.dir_cache = {
109113
dirname: {"name": dirname, "size": 0, "type": "directory"}
110-
for dirname in self._all_dirnames(self.tar.getnames())
114+
for dirname in self._all_dirnames(self.index)
111115
}
112-
for member in self.tar.getmembers():
113-
info = member.get_info()
114-
info["name"] = info["name"].rstrip("/")
115-
info["type"] = typemap.get(info["type"], "file")
116-
self.dir_cache[info["name"]] = info
116+
self.dir_cache.update(
117+
{info["name"]: info for info, _, _ in self.index.values()}
118+
)
117119

118120
def _open(self, path, mode="rb", **kwargs):
119121
if mode != "rb":
120122
raise ValueError("Read-only filesystem implementation")
121-
details, offset = self.index[path]
123+
# Accept paths containing the archive's duplicate slashes too.
124+
path = re.sub("/+", "/", path)
125+
details, _, orig_name = self.index[path]
122126
if details["type"] != "file":
123127
raise ValueError("Can only handle regular files")
124-
return self.tar.extractfile(path)
128+
return self.tar.extractfile(orig_name)
125129

126130
def close(self):
127131
"""Commits any write changes to the file. Done on ``del`` too."""

‎fsspec/implementations/tests/test_tar.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,3 +244,50 @@ def test_ls_with_folders(compression: str, tmp_path: Path):
244244
"d/e/f.pdf",
245245
"d/g.pdf",
246246
]
247+
248+
249+
@pytest.mark.parametrize(
250+
"compression", ["", "gz", "bz2", "xz"], ids=["tar", "tar-gz", "tar-bz2", "tar-xz"]
251+
)
252+
def test_ls_with_duplicate_slashes(compression: str, tmp_path: Path):
253+
"""
254+
Members whose names contain redundant duplicate slashes (e.g.
255+
``"a/b//c.txt"``) must still be reachable through the directory listing
256+
and openable, rather than becoming silently invisible to
257+
``find``/``glob``/``ls``/``walk``. Regression test for
258+
https://github.com/fsspec/filesystem_spec/issues/1947.
259+
"""
260+
tar_data: dict[str, bytes] = {
261+
"path/with/extra/slash//test.txt": b"Hello slash!",
262+
"regular/file.txt": b"Hello regular!",
263+
}
264+
if compression:
265+
temp_archive_file = tmp_path / f"test_tar_file.tar.{compression}"
266+
else:
267+
temp_archive_file = tmp_path / "test_tar_file.tar"
268+
with open(temp_archive_file, "wb") as fd:
269+
with tarfile.open(fileobj=fd, mode=f"w:{compression}") as tf:
270+
for tar_file_path, data in tar_data.items():
271+
info = tarfile.TarInfo(name=tar_file_path)
272+
info.size = len(data)
273+
tf.addfile(info, BytesIO(data))
274+
275+
with open(temp_archive_file, "rb") as fd:
276+
fs = TarFileSystem(fd)
277+
278+
# The duplicate-slash member is discoverable with its slashes collapsed.
279+
assert fs.find("/") == [
280+
"path/with/extra/slash/test.txt",
281+
"regular/file.txt",
282+
]
283+
assert fs.glob("path/**/*.txt") == ["path/with/extra/slash/test.txt"]
284+
assert fs.ls("path/with/extra/slash", detail=False) == [
285+
"path/with/extra/slash/test.txt"
286+
]
287+
288+
# The intermediate directory is inferred without a trailing slash.
289+
assert fs.isdir("path/with/extra/slash")
290+
291+
# It can be opened both by its normalised name and its original name.
292+
assert fs.cat("path/with/extra/slash/test.txt") == b"Hello slash!"
293+
assert fs.cat("path/with/extra/slash//test.txt") == b"Hello slash!"

0 commit comments

Comments
 (0)