Skip to content

Commit 652325d

Browse files
authored
Fix async cat ranges on error (#2045)
1 parent 073aac8 commit 652325d

2 files changed

Lines changed: 37 additions & 2 deletions

File tree

‎fsspec/asyn.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -528,8 +528,11 @@ async def _cat_ranges(
528528
starts, ends: int or list
529529
Bytes limits of the read. If using a single int, the same value will be
530530
used to read all the specified files.
531+
on_error: "return" or "raise"
532+
If "return" (default), any per-range exception is placed in the output
533+
list at the corresponding position. Otherwise the first such exception
534+
is raised. Matches ``AbstractFileSystem.cat_ranges``.
531535
"""
532-
# TODO: on_error
533536
if max_gap is not None:
534537
# use utils.merge_offset_ranges
535538
raise NotImplementedError
@@ -546,9 +549,14 @@ async def _cat_ranges(
546549
for p, s, e in zip(paths, starts, ends)
547550
]
548551
batch_size = batch_size or self.batch_size
549-
return await _run_coros_in_chunks(
552+
out = await _run_coros_in_chunks(
550553
coros, batch_size=batch_size, nofiles=True, return_exceptions=True
551554
)
555+
if on_error != "return":
556+
ex = next(filter(is_exception, out), None)
557+
if ex is not None:
558+
raise ex
559+
return out
552560

553561
async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs):
554562
raise NotImplementedError

‎fsspec/tests/test_async.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,33 @@ def test_rm_file_without_implementation():
254254
fs.rm_file("test/file.txt")
255255

256256

257+
class _CatRangesFS(fsspec.asyn.AsyncFileSystem):
258+
# Mirrors the gcsfs/s3fs pattern: overrides _cat_file, inherits _cat_ranges.
259+
cachable = False
260+
store = {"ok": b"0123456789"}
261+
262+
async def _cat_file(self, path, start=None, end=None, **kwargs):
263+
if path not in self.store:
264+
raise FileNotFoundError(path)
265+
return self.store[path][start:end]
266+
267+
268+
def test_cat_ranges_on_error_return_keeps_exceptions_inline():
269+
# Default on_error="return": a failed range comes back as an exception value,
270+
# positionally aligned with the inputs (matches AbstractFileSystem.cat_ranges).
271+
fs = _CatRangesFS()
272+
out = fs.cat_ranges(["ok", "missing"], [0, 0], [4, 4])
273+
assert out[0] == b"0123"
274+
assert isinstance(out[1], FileNotFoundError)
275+
276+
277+
def test_cat_ranges_on_error_raise_propagates():
278+
# on_error="raise" must raise the first error instead of returning it inline.
279+
fs = _CatRangesFS()
280+
with pytest.raises(FileNotFoundError):
281+
fs.cat_ranges(["ok", "missing"], [0, 0], [4, 4], on_error="raise")
282+
283+
257284
# ---------------------------------------------------------------------------
258285
# Tests for the prefix= hint that _glob passes to _find
259286
# ---------------------------------------------------------------------------

0 commit comments

Comments
 (0)