Skip to content

Commit 33e6da3

Browse files
Shut down background cache workers on file close (#2069)
Co-authored-by: Martin Durant <[email protected]>
1 parent 4adecd3 commit 33e6da3

3 files changed

Lines changed: 59 additions & 0 deletions

File tree

‎fsspec/caching.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,7 @@ def __init__(
801801
self._fetch_future_block_number: int | None = None
802802
self._fetch_future: Future[bytes] | None = None
803803
self._fetch_future_lock = threading.Lock()
804+
self._closed = False
804805

805806
def cache_info(self) -> UpdatableLRU.CacheInfo:
806807
"""
@@ -813,6 +814,23 @@ def cache_info(self) -> UpdatableLRU.CacheInfo:
813814
"""
814815
return self._fetch_block_cached.cache_info()
815816

817+
def close(self) -> None:
818+
"""Cancel pending work and shut down the background worker."""
819+
with self._fetch_future_lock:
820+
if self._closed:
821+
return
822+
self._closed = True
823+
future = self._fetch_future
824+
self._fetch_future = None
825+
self._fetch_future_block_number = None
826+
827+
if future is not None:
828+
future.cancel()
829+
self._thread_executor.shutdown(wait=True, cancel_futures=True)
830+
831+
# UpdatableLRU stores a bound method and otherwise forms a reference cycle.
832+
del self._fetch_block_cached
833+
816834
def __getstate__(self) -> dict[str, Any]:
817835
state = self.__dict__
818836
del state["_fetch_block_cached"]
@@ -829,6 +847,7 @@ def __setstate__(self, state) -> None:
829847
self._fetch_future_block_number = None
830848
self._fetch_future = None
831849
self._fetch_future_lock = threading.Lock()
850+
self._closed = False
832851

833852
def _fetch(self, start: int | None, end: int | None) -> bytes:
834853
if start is None:

‎fsspec/spec.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2267,6 +2267,11 @@ def close(self):
22672267
return
22682268
try:
22692269
if self.mode == "rb":
2270+
cache = getattr(self, "cache", None)
2271+
if cache is not None:
2272+
close = getattr(cache, "close", None)
2273+
if callable(close):
2274+
close()
22702275
self.cache = None
22712276
else:
22722277
if not getattr(self, "forced", True):

‎fsspec/tests/test_caches.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import pytest
55

66
from fsspec.caching import (
7+
BackgroundBlockCache,
78
BlockCache,
89
FirstChunkCache,
910
MMapCache,
@@ -303,6 +304,40 @@ def wrapped(*a, **kw):
303304
assert len(thread_ids) == 2
304305

305306

307+
def test_background_shutdown_on_close():
308+
import weakref
309+
310+
from fsspec.spec import AbstractBufferedFile
311+
312+
data = b"abcdefgh"
313+
314+
class TestFile(AbstractBufferedFile):
315+
DEFAULT_BLOCK_SIZE = 4
316+
317+
def _fetch_range(self, start, end):
318+
return data[start:end]
319+
320+
f = TestFile(
321+
None,
322+
"test",
323+
mode="rb",
324+
cache_type="background",
325+
size=len(data),
326+
)
327+
f.read(1)
328+
cache = f.cache
329+
assert isinstance(cache, BackgroundBlockCache)
330+
cache_ref = weakref.ref(cache)
331+
executor = cache._thread_executor
332+
del cache
333+
334+
f.close()
335+
336+
with pytest.raises(RuntimeError, match="cannot schedule new futures"):
337+
executor.submit(lambda: None)
338+
assert cache_ref() is None
339+
340+
306341
def test_register_cache():
307342
# just test that we have them populated and fail to re-add again unless overload
308343
with pytest.raises(ValueError):

0 commit comments

Comments
 (0)