Skip to content

Commit 71b8a6b

Browse files
authored
Reflect async _walk correctly (#2040)
1 parent 5eec9f9 commit 71b8a6b

3 files changed

Lines changed: 41 additions & 9 deletions

File tree

‎fsspec/asyn.py‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,22 @@ def wrapper(*args, **kwargs):
120120
return wrapper
121121

122122

123+
def async_gen_wrapper(func, obj=None):
124+
"""Given a async generator, make so can be called in blocking contexts"""
125+
126+
@functools.wraps(func)
127+
def wrapper(*args, **kwargs):
128+
self = obj or args[0]
129+
gen = func(*args, **kwargs)
130+
while True:
131+
try:
132+
yield sync(self.loop, gen.__anext__)
133+
except StopAsyncIteration:
134+
break
135+
136+
return wrapper
137+
138+
123139
def get_loop():
124140
"""Create or return the default fsspec IO loop
125141
@@ -959,7 +975,7 @@ def mirror_sync_methods(obj):
959975
"""
960976
from fsspec import AbstractFileSystem
961977

962-
for method in async_methods + dir(AsyncFileSystem):
978+
for method in set(async_methods + dir(AsyncFileSystem)):
963979
if not method.startswith("_"):
964980
continue
965981
smethod = method[1:]
@@ -969,11 +985,15 @@ def mirror_sync_methods(obj):
969985
is_default = unsync is getattr(AbstractFileSystem, smethod, "")
970986
if isco and is_default:
971987
mth = sync_wrapper(getattr(obj, method), obj=obj)
972-
setattr(obj, smethod, mth)
973-
if not mth.__doc__:
974-
mth.__doc__ = getattr(
975-
getattr(AbstractFileSystem, smethod, None), "__doc__", ""
976-
)
988+
elif inspect.isasyncgenfunction(getattr(obj, method, None)) and is_default:
989+
mth = async_gen_wrapper(getattr(obj, method), obj=obj)
990+
else:
991+
continue
992+
setattr(obj, smethod, mth)
993+
if not mth.__doc__:
994+
mth.__doc__ = getattr(
995+
getattr(AbstractFileSystem, smethod, None), "__doc__", ""
996+
)
977997

978998

979999
class FSSpecCoroutineCancel(Exception):

‎fsspec/implementations/tests/test_dirfs.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -394,9 +394,13 @@ async def _walk(path, *args, **kwargs):
394394

395395

396396
def test_walk(dirfs):
397-
dirfs.fs.walk.return_value = iter(
398-
[(f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])]
399-
)
397+
if dirfs.fs.async_impl:
398+
pytest.skip()
399+
else:
400+
dirfs.fs.walk.return_value = iter(
401+
[(f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])]
402+
)
403+
400404
assert list(dirfs.walk("root", *ARGS, **KWARGS)) == [
401405
("root", ["foo", "bar"], ["baz", "qux"])
402406
]

‎fsspec/tests/test_async.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,14 @@ def test_sync_methods():
1818
assert not inspect.iscoroutinefunction(inst.info)
1919

2020

21+
def test_async_gen():
22+
inst = fsspec.asyn.AsyncFileSystem()
23+
assert inst.walk is not fsspec.AbstractFileSystem.walk
24+
gen = inst.walk("path") # does not raise yet
25+
with pytest.raises(NotImplementedError):
26+
list(gen)
27+
28+
2129
def test_when_sync_methods_are_disabled():
2230
class TestFS(fsspec.asyn.AsyncFileSystem):
2331
mirror_sync_methods = False

0 commit comments

Comments
 (0)