Skip to content

Commit c5fea6d

Browse files
authored
Fix cat_ranges on whole-file caches after #2154 (#2163)
1 parent 9efb18b commit c5fea6d

2 files changed

Lines changed: 66 additions & 23 deletions

File tree

‎fsspec/implementations/cached.py‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -780,13 +780,37 @@ async def _cat_file(self, path, start=None, end=None, **kwargs):
780780
size = -1 if end is None else end - f.tell()
781781
return f.read(size)
782782

783+
def _local_paths_for_ranges(self, paths):
784+
"""Cache file for each of paths, and which remote files to download
785+
786+
A path given more than once is downloaded once, and every occurrence
787+
reads from the same cache file.
788+
"""
789+
lpaths = []
790+
local = {}
791+
download = []
792+
rpaths = []
793+
for p in paths:
794+
if p not in local:
795+
detail = self._check_file(p)
796+
if not detail:
797+
fn = os.path.join(self.storage[-1], self._mapper(p))
798+
download.append(fn)
799+
rpaths.append(p)
800+
else:
801+
# filecache returns (detail, fn), simplecache just fn
802+
fn = detail[1] if isinstance(detail, tuple) else detail
803+
local[p] = fn
804+
lpaths.append(local[p])
805+
return lpaths, rpaths, download
806+
783807
async def _cat_ranges(
784808
self, paths, starts, ends, max_gap=None, on_error="return", **kwargs
785809
):
786810
logger.debug("async cat ranges %s", paths)
787811
lpaths, rpaths, download = self._local_paths_for_ranges(paths)
788812
if download:
789-
await self.fs._get(rpaths, download, on_error=on_error)
813+
await self.fs._get(rpaths, download)
790814

791815
return LocalFileSystem().cat_ranges(
792816
lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
@@ -902,32 +926,11 @@ def cat_ranges(
902926
logger.debug("cat ranges %s", paths)
903927
lpaths, rpaths, download = self._local_paths_for_ranges(paths)
904928
if download:
905-
self.fs.get(rpaths, download, on_error=on_error)
929+
self.fs.get(rpaths, download)
906930
return LocalFileSystem().cat_ranges(
907931
lpaths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs
908932
)
909933

910-
def _local_paths_for_ranges(self, paths):
911-
"""Cache file for each of paths, and which remote files to download
912-
913-
A path given more than once is downloaded once, and every occurrence
914-
reads from the same cache file.
915-
"""
916-
lpaths = []
917-
local = {}
918-
download = []
919-
rpaths = []
920-
for p in paths:
921-
if p not in local:
922-
fn = self._check_file(p)
923-
if fn is None:
924-
fn = os.path.join(self.storage[-1], self._mapper(p))
925-
download.append(fn)
926-
rpaths.append(p)
927-
local[p] = fn
928-
lpaths.append(local[p])
929-
return lpaths, rpaths, download
930-
931934
def _get_cached_file_before_open(self, path, **kwargs):
932935
sha = self._mapper(path)
933936
fn = os.path.join(self.storage[-1], sha)

‎fsspec/implementations/tests/test_cached.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1469,3 +1469,43 @@ def test_simplecache_cat_ranges_downloads_uncached(tmp_path):
14691469
out = fs.cat_ranges(paths, [0, 5, 1], [2, 8, 3], on_error="raise")
14701470
assert out == [b"01", b"567", b"bc"]
14711471
assert len(os.listdir(tmp_path)) == 2
1472+
1473+
1474+
@pytest.mark.asyncio
1475+
@pytest.mark.parametrize("protocol", ["filecache", "simplecache"])
1476+
async def test_async_cat_ranges_downloads_uncached(tmp_path, protocol):
1477+
from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
1478+
1479+
m = fsspec.filesystem("memory")
1480+
m.pipe({"/cat_ranges/one": b"0123456789", "/cat_ranges/two": b"abcdef"})
1481+
target = AsyncFileSystemWrapper(m, asynchronous=True)
1482+
fs = fsspec.filesystem(
1483+
protocol,
1484+
fs=target,
1485+
cache_storage=str(tmp_path),
1486+
asynchronous=True,
1487+
skip_instance_cache=True,
1488+
)
1489+
paths = ["/cat_ranges/one", "/cat_ranges/one", "/cat_ranges/two"]
1490+
out = await fs._cat_ranges(paths, [0, 5, 1], [2, 8, 3], on_error="raise")
1491+
assert out == [b"01", b"567", b"bc"]
1492+
# a second call reads the files downloaded by the first one
1493+
assert await fs._cat_ranges(paths[:1], [8], [10]) == [b"89"]
1494+
1495+
1496+
def test_simplecache_cat_ranges_target_get_file_without_on_error(tmp_path):
1497+
class StrictGetFile(fsspec.implementations.memory.MemoryFileSystem):
1498+
# like HTTPFileSystem in http_sync: no **kwargs on get_file
1499+
def get_file(self, rpath, lpath, callback=None, outfile=None):
1500+
with open(lpath, "wb") as f:
1501+
f.write(self.cat_file(rpath))
1502+
1503+
target = StrictGetFile(skip_instance_cache=True)
1504+
target.pipe("/cat_ranges/one", b"0123456789")
1505+
fs = fsspec.filesystem(
1506+
"simplecache",
1507+
fs=target,
1508+
cache_storage=str(tmp_path),
1509+
skip_instance_cache=True,
1510+
)
1511+
assert fs.cat_ranges(["/cat_ranges/one"], [0], [3]) == [b"012"]

0 commit comments

Comments
 (0)