Skip to content

Commit 695f831

Browse files
authored
Add age-based cleanup for simplecache (#2118)
1 parent 42aa468 commit 695f831

4 files changed

Lines changed: 82 additions & 0 deletions

File tree

‎docs/source/changelog.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ Fixes
2121

2222
- Allow filesystem implementations to assign ``protocol`` per instance
2323

24+
- Allow ``simplecache`` files to be removed by age using their modification times
25+
2426
- Avoid mutating live ``BlockCache`` and ``BackgroundBlockCache`` instances
2527
when pickling (#2102)
2628

‎docs/source/features.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,9 @@ should not mix block- and file-caches in the same directory. "simplecache" is th
310310
except without the options for cache expiry and to check the original source - it can be used where the
311311
target can be considered static, and particularly where a large number of target files are expected
312312
(because no metadata is written to disc). Only "simplecache" is guaranteed thread/process-safe.
313+
Cached files can still be removed by age using their local modification times, for example
314+
``fs.clear_expired_cache(expiry_time=7 * 24 * 60 * 60)``. This only changes the writable cache
315+
location when multiple cache directories are configured.
313316

314317
Remote Write Caching
315318
--------------------

‎fsspec/implementations/cached.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,38 @@ def save_cache(self):
860860
def load_cache(self):
861861
pass
862862

863+
def clear_expired_cache(self, expiry_time=None):
864+
"""Remove cached files older than ``expiry_time`` seconds.
865+
866+
SimpleCache does not store metadata, so cached file modification times
867+
are used instead. Only the last (writable) cache location is changed.
868+
869+
Parameters
870+
----------
871+
expiry_time: int or float
872+
Maximum age of a cached file in seconds. This must be supplied
873+
explicitly because SimpleCache has no configured expiry time.
874+
"""
875+
if expiry_time is None:
876+
raise ValueError("expiry_time must be provided for simplecache")
877+
if expiry_time < 0:
878+
raise ValueError("expiry_time must be non-negative")
879+
880+
self._mkcache()
881+
cutoff = time.time() - expiry_time
882+
with os.scandir(self.storage[-1]) as entries:
883+
for entry in entries:
884+
try:
885+
if (
886+
entry.is_file(follow_symlinks=False)
887+
and entry.stat(follow_symlinks=False).st_mtime < cutoff
888+
):
889+
os.remove(entry.path)
890+
except FileNotFoundError:
891+
# Another process may be using the same simple cache.
892+
pass
893+
self._cache_size = None
894+
863895
def pipe_file(self, path, value=None, **kwargs):
864896
if self._intrans:
865897
with self.open(path, "wb") as f:

‎fsspec/implementations/tests/test_cached.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,51 @@ def __ager(cache_fn, fn, del_fn=False):
455455
fs.clear_expired_cache()
456456

457457

458+
def test_simplecache_clear_expired_uses_file_times(tmp_path):
459+
source = tmp_path / "source"
460+
source.mkdir()
461+
old_source = source / "old"
462+
recent_source = source / "recent"
463+
cache = tmp_path / "cache"
464+
fs = fsspec.filesystem(
465+
"simplecache",
466+
target_protocol="file",
467+
cache_storage=str(cache),
468+
skip_instance_cache=True,
469+
)
470+
471+
with fs.open(str(old_source), "wb") as f:
472+
f.write(b"old")
473+
with fs.open(str(recent_source), "wb") as f:
474+
f.write(b"recent")
475+
old_cached = fs._check_file(fs._strip_protocol(str(old_source)))
476+
recent_cached = fs._check_file(fs._strip_protocol(str(recent_source)))
477+
assert old_cached is not None
478+
assert recent_cached is not None
479+
os.utime(old_cached, (1, 1))
480+
481+
fs.clear_expired_cache(expiry_time=60)
482+
483+
assert not os.path.exists(old_cached)
484+
assert os.path.exists(recent_cached)
485+
assert cache.is_dir()
486+
487+
488+
def test_simplecache_clear_expired_requires_age(tmp_path):
489+
fs = fsspec.filesystem(
490+
"simplecache",
491+
target_protocol="file",
492+
cache_storage=str(tmp_path / "cache"),
493+
skip_instance_cache=True,
494+
)
495+
496+
with pytest.raises(ValueError, match="expiry_time must be provided"):
497+
fs.clear_expired_cache()
498+
499+
with pytest.raises(ValueError, match="expiry_time must be non-negative"):
500+
fs.clear_expired_cache(expiry_time=-1)
501+
502+
458503
def test_pop():
459504
import tempfile
460505

0 commit comments

Comments
 (0)