Skip to content

Commit 30925b3

Browse files
authored
Validate maxdepth kwarg (#1151)
* Validate maxdepth in AbstractFileSystem.walk * maxdepth in expand_path * maxdepth in async _expand_path and _walk * Fix windows tests
1 parent c64d7f0 commit 30925b3

7 files changed

Lines changed: 121 additions & 7 deletions

File tree

‎docs/source/changelog.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ Fixes:
464464
- random appending of a directory within the filesystems ``find()`` method (#507, 537)
465465
- fix git tests (#501)
466466
- fix recursive memfs operations (#502)
467-
- fix recorsive/maxdepth for cp (#508)
467+
- fix recursive/maxdepth for cp (#508)
468468
- fix listings cache timeout (#513)
469469
- big endian bytes tests (#519)
470470
- docs syntax (#535, 524, 520, 542)

‎fsspec/asyn.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,9 @@ async def _ls(self, path, detail=True, **kwargs):
577577
raise NotImplementedError
578578

579579
async def _walk(self, path, maxdepth=None, **kwargs):
580+
if maxdepth is not None and maxdepth < 1:
581+
raise ValueError("maxdepth must be at least 1")
582+
580583
path = self._strip_protocol(path)
581584
full_dirs = {}
582585
dirs = {}
@@ -727,11 +730,12 @@ async def _find(self, path, maxdepth=None, withdirs=False, **kwargs):
727730
return {name: out[name] for name in names}
728731

729732
async def _expand_path(self, path, recursive=False, maxdepth=None):
733+
if maxdepth is not None and maxdepth < 1:
734+
raise ValueError("maxdepth must be at least 1")
735+
730736
if isinstance(path, str):
731737
out = await self._expand_path([path], recursive, maxdepth)
732738
else:
733-
# reduce depth on each recursion level unless None or 0
734-
maxdepth = maxdepth if not maxdepth else maxdepth - 1
735739
out = set()
736740
path = [self._strip_protocol(p) for p in path]
737741
for p in path: # can gather here
@@ -751,6 +755,8 @@ async def _expand_path(self, path, recursive=False, maxdepth=None):
751755
if p not in out and (recursive is False or (await self._exists(p))):
752756
# should only check once, for the root
753757
out.add(p)
758+
# reduce depth on each recursion level unless None or 0
759+
maxdepth = maxdepth if not maxdepth else maxdepth - 1
754760
if not out:
755761
raise FileNotFoundError(path)
756762
return list(sorted(out))

‎fsspec/implementations/tests/test_http.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,3 +486,40 @@ def test_with_cache(server):
486486
with fs1.open(fn, "rb") as f:
487487
out = f.read()
488488
assert out == fs1.cat(fn)
489+
490+
491+
@pytest.mark.asyncio
492+
async def test_async_expand_path(server):
493+
fs = fsspec.filesystem("http", asynchronous=True, skip_instance_cache=True)
494+
495+
# maxdepth=1
496+
assert await fs._expand_path(server + "/index", recursive=True, maxdepth=1) == [
497+
server + "/index",
498+
server + "/index/realfile",
499+
]
500+
501+
# maxdepth=0
502+
with pytest.raises(ValueError):
503+
await fs._expand_path(server + "/index", maxdepth=0)
504+
with pytest.raises(ValueError):
505+
await fs._expand_path(server + "/index", recursive=True, maxdepth=0)
506+
507+
await fs._session.close()
508+
509+
510+
@pytest.mark.asyncio
511+
async def test_async_walk(server):
512+
fs = fsspec.filesystem("http", asynchronous=True, skip_instance_cache=True)
513+
514+
# No maxdepth
515+
res = []
516+
async for a in fs._walk(server + "/index"):
517+
res.append(a)
518+
assert res == [(server + "/index", [], ["realfile"])]
519+
520+
# maxdepth=0
521+
with pytest.raises(ValueError):
522+
async for a in fs._walk(server + "/index", maxdepth=0):
523+
pass
524+
525+
await fs._session.close()

‎fsspec/implementations/tests/test_local.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -858,7 +858,8 @@ def test_du(tmpdir):
858858

859859
assert fs.du(tmpdir, maxdepth=2) == 11
860860
assert fs.du(tmpdir, maxdepth=1) == 4
861-
assert fs.du(tmpdir, maxdepth=0) == 4
861+
with pytest.raises(ValueError):
862+
fs.du(tmpdir, maxdepth=0)
862863

863864
# Size of file only.
864865
assert fs.du(file) == 4

‎fsspec/spec.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,9 @@ def walk(self, path, maxdepth=None, topdown=True, **kwargs):
390390
the bottom upwards.
391391
kwargs: passed to ``ls``
392392
"""
393+
if maxdepth is not None and maxdepth < 1:
394+
raise ValueError("maxdepth must be at least 1")
395+
393396
path = self._strip_protocol(path)
394397
full_dirs = {}
395398
dirs = {}
@@ -429,6 +432,8 @@ def walk(self, path, maxdepth=None, topdown=True, **kwargs):
429432
if maxdepth is not None:
430433
maxdepth -= 1
431434
if maxdepth < 1:
435+
if not topdown:
436+
yield path, dirs, files
432437
return
433438

434439
for d in full_dirs:
@@ -977,11 +982,12 @@ def copy(self, path1, path2, recursive=False, on_error=None, **kwargs):
977982
def expand_path(self, path, recursive=False, maxdepth=None):
978983
"""Turn one or more globs or directories into a list of all matching paths
979984
to files or directories."""
985+
if maxdepth is not None and maxdepth < 1:
986+
raise ValueError("maxdepth must be at least 1")
987+
980988
if isinstance(path, str):
981989
out = self.expand_path([path], recursive, maxdepth)
982990
else:
983-
# reduce depth on each recursion level unless None or 0
984-
maxdepth = maxdepth if not maxdepth else maxdepth - 1
985991
out = set()
986992
path = [self._strip_protocol(p) for p in path]
987993
for p in path:
@@ -1003,6 +1009,8 @@ def expand_path(self, path, recursive=False, maxdepth=None):
10031009
if p not in out and (recursive is False or self.exists(p)):
10041010
# should only check once, for the root
10051011
out.add(p)
1012+
# reduce depth on each recursion level unless None or 0
1013+
maxdepth = maxdepth if not maxdepth else maxdepth - 1
10061014
if not out:
10071015
raise FileNotFoundError(path)
10081016
return list(sorted(out))

‎fsspec/tests/test_api.py‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ def test_du(m):
116116
"/dir/dirb/afile": 2,
117117
"/dir/dirb/bfile": 3,
118118
}
119-
assert fs.du("/dir", maxdepth=0) == 1
119+
with pytest.raises(ValueError):
120+
assert fs.du("/dir", maxdepth=0) == 1
120121
assert fs.du("/dir", total=False, withdirs=True, maxdepth=1) == {
121122
"/dir": 0,
122123
"/dir/afile": 1,
@@ -363,3 +364,53 @@ def test_url_to_fs():
363364

364365
assert isinstance(fs, MemoryFileSystem)
365366
assert url2 == "/a.txt"
367+
368+
369+
def test_walk(m):
370+
dir0 = "/dir0"
371+
dir1 = dir0 + "/dir1"
372+
dir2 = dir1 + "/dir2"
373+
file1 = dir0 + "/file1"
374+
file2 = dir1 + "/file2"
375+
file3 = dir2 + "/file3"
376+
377+
m.mkdir(dir2) # Creates parents too
378+
m.touch(file1)
379+
m.touch(file2)
380+
m.touch(file3)
381+
382+
# No maxdepth
383+
assert list(m.walk(dir0, topdown=True)) == [
384+
(dir0, ["dir1"], ["file1"]),
385+
(dir1, ["dir2"], ["file2"]),
386+
(dir2, [], ["file3"]),
387+
]
388+
assert list(m.walk(dir0, topdown=False)) == [
389+
(dir2, [], ["file3"]),
390+
(dir1, ["dir2"], ["file2"]),
391+
(dir0, ["dir1"], ["file1"]),
392+
]
393+
394+
# maxdepth=2
395+
assert list(m.walk(dir0, maxdepth=2, topdown=True)) == [
396+
(dir0, ["dir1"], ["file1"]),
397+
(dir1, ["dir2"], ["file2"]),
398+
]
399+
assert list(m.walk(dir0, maxdepth=2, topdown=False)) == [
400+
(dir1, ["dir2"], ["file2"]),
401+
(dir0, ["dir1"], ["file1"]),
402+
]
403+
404+
# maxdepth=1
405+
assert list(m.walk(dir0, maxdepth=1, topdown=True)) == [
406+
(dir0, ["dir1"], ["file1"]),
407+
]
408+
assert list(m.walk(dir0, maxdepth=1, topdown=False)) == [
409+
(dir0, ["dir1"], ["file1"]),
410+
]
411+
412+
# maxdepth=0
413+
with pytest.raises(ValueError):
414+
list(m.walk(dir0, maxdepth=0, topdown=True))
415+
with pytest.raises(ValueError):
416+
list(m.walk(dir0, maxdepth=0, topdown=False))

‎fsspec/tests/test_spec.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,17 @@ def test_expand_path_recursive(test_paths, expected):
202202
paths = test_fs.expand_path(list(test_paths), recursive=True)
203203
assert sorted(paths) == sorted(expected)
204204

205+
# test with maxdepth
206+
assert test_fs.expand_path("top_level", recursive=True, maxdepth=1) == [
207+
"top_level",
208+
"top_level/second_level",
209+
]
210+
211+
with pytest.raises(ValueError):
212+
test_fs.expand_path("top_level", recursive=True, maxdepth=0)
213+
with pytest.raises(ValueError):
214+
test_fs.expand_path("top_level", maxdepth=0)
215+
205216

206217
@pytest.mark.xfail
207218
def test_find():

0 commit comments

Comments
 (0)