Skip to content

Commit 04f0367

Browse files
authored
Merge pull request #3 from shikharsg/chunk_cache_mapping_refactor
factoring out mapping code from LRUStoreCache and LRUChunkCache
2 parents a2a05fb + b8b9056 commit 04f0367

1 file changed

Lines changed: 71 additions & 99 deletions

File tree

‎zarr/storage.py‎

Lines changed: 71 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1695,7 +1695,75 @@ def __len__(self):
16951695
return self.db.stat()['entries']
16961696

16971697

1698-
class LRUStoreCache(MutableMapping):
1698+
class LRUMappingCache(MutableMapping):
1699+
"""Abstract base class for Mapping Cache
1700+
"""
1701+
1702+
def __init__(self, max_size):
1703+
self._max_size = max_size
1704+
self._current_size = 0
1705+
self._values_cache = OrderedDict()
1706+
self._mutex = Lock()
1707+
self.hits = self.misses = 0
1708+
1709+
def __len__(self):
1710+
return len(self._keys())
1711+
1712+
def __iter__(self):
1713+
return self.keys()
1714+
1715+
def keys(self):
1716+
with self._mutex:
1717+
return iter(self._keys())
1718+
1719+
def _keys(self):
1720+
raise NotImplementedError
1721+
1722+
def _pop_value(self):
1723+
# remove the first value from the cache, as this will be the least recently
1724+
# used value
1725+
_, v = self._values_cache.popitem(last=False)
1726+
return v
1727+
1728+
def _accommodate_value(self, value_size):
1729+
if self._max_size is None:
1730+
return
1731+
# ensure there is enough space in the cache for a new value
1732+
while self._current_size + value_size > self._max_size:
1733+
v = self._pop_value()
1734+
self._current_size -= buffer_size(v)
1735+
1736+
def _cache_value(self, key, value):
1737+
# cache a value
1738+
value_size = buffer_size(value)
1739+
# check size of the value against max size, as if the value itself exceeds max
1740+
# size then we are never going to cache it
1741+
if self._max_size is None or value_size <= self._max_size:
1742+
self._accommodate_value(value_size)
1743+
self._values_cache[key] = value
1744+
self._current_size += value_size
1745+
1746+
def invalidate_values(self):
1747+
"""Clear the values cache."""
1748+
with self._mutex:
1749+
self._values_cache.clear()
1750+
1751+
def _invalidate_value(self, key):
1752+
if key in self._values_cache:
1753+
value = self._values_cache.pop(key)
1754+
self._current_size -= buffer_size(value)
1755+
1756+
def __getitem__(self, key):
1757+
raise NotImplementedError
1758+
1759+
def __setitem__(self, key, value):
1760+
raise NotImplementedError
1761+
1762+
def __delitem__(self, key):
1763+
raise NotImplementedError
1764+
1765+
1766+
class LRUStoreCache(LRUMappingCache):
16991767
"""Storage class that implements a least-recently-used (LRU) cache layer over
17001768
some other store. Intended primarily for use with stores that can be slow to
17011769
access, e.g., remote stores that require network communication to store and
@@ -1733,15 +1801,11 @@ class LRUStoreCache(MutableMapping):
17331801
"""
17341802

17351803
def __init__(self, store, max_size):
1804+
super().__init__(max_size)
17361805
self._store = store
1737-
self._max_size = max_size
1738-
self._current_size = 0
17391806
self._keys_cache = None
17401807
self._contains_cache = None
17411808
self._listdir_cache = dict()
1742-
self._values_cache = OrderedDict()
1743-
self._mutex = Lock()
1744-
self.hits = self.misses = 0
17451809

17461810
def __getstate__(self):
17471811
return (self._store, self._max_size, self._current_size, self._keys_cache,
@@ -1754,12 +1818,6 @@ def __setstate__(self, state):
17541818
self.misses) = state
17551819
self._mutex = Lock()
17561820

1757-
def __len__(self):
1758-
return len(self._keys())
1759-
1760-
def __iter__(self):
1761-
return self.keys()
1762-
17631821
def __contains__(self, key):
17641822
with self._mutex:
17651823
if self._contains_cache is None:
@@ -1770,10 +1828,6 @@ def clear(self):
17701828
self._store.clear()
17711829
self.invalidate()
17721830

1773-
def keys(self):
1774-
with self._mutex:
1775-
return iter(self._keys())
1776-
17771831
def _keys(self):
17781832
if self._keys_cache is None:
17791833
self._keys_cache = list(self._store.keys())
@@ -1791,41 +1845,12 @@ def listdir(self, path=None):
17911845
def getsize(self, path=None):
17921846
return getsize(self._store, path=path)
17931847

1794-
def _pop_value(self):
1795-
# remove the first value from the cache, as this will be the least recently
1796-
# used value
1797-
_, v = self._values_cache.popitem(last=False)
1798-
return v
1799-
1800-
def _accommodate_value(self, value_size):
1801-
if self._max_size is None:
1802-
return
1803-
# ensure there is enough space in the cache for a new value
1804-
while self._current_size + value_size > self._max_size:
1805-
v = self._pop_value()
1806-
self._current_size -= buffer_size(v)
1807-
1808-
def _cache_value(self, key, value):
1809-
# cache a value
1810-
value_size = buffer_size(value)
1811-
# check size of the value against max size, as if the value itself exceeds max
1812-
# size then we are never going to cache it
1813-
if self._max_size is None or value_size <= self._max_size:
1814-
self._accommodate_value(value_size)
1815-
self._values_cache[key] = value
1816-
self._current_size += value_size
1817-
18181848
def invalidate(self):
18191849
"""Completely clear the cache."""
18201850
with self._mutex:
18211851
self._values_cache.clear()
18221852
self._invalidate_keys()
18231853

1824-
def invalidate_values(self):
1825-
"""Clear the values cache."""
1826-
with self._mutex:
1827-
self._values_cache.clear()
1828-
18291854
def invalidate_keys(self):
18301855
"""Clear the keys cache."""
18311856
with self._mutex:
@@ -1836,11 +1861,6 @@ def _invalidate_keys(self):
18361861
self._contains_cache = None
18371862
self._listdir_cache.clear()
18381863

1839-
def _invalidate_value(self, key):
1840-
if key in self._values_cache:
1841-
value = self._values_cache.pop(key)
1842-
self._current_size -= buffer_size(value)
1843-
18441864
def __getitem__(self, key):
18451865
try:
18461866
# first try to obtain the value from the cache
@@ -1994,11 +2014,7 @@ class LRUChunkCache(MutableMapping):
19942014
"""
19952015

19962016
def __init__(self, max_size):
1997-
self._max_size = max_size
1998-
self._current_size = 0
1999-
self._values_cache = OrderedDict()
2000-
self._mutex = Lock()
2001-
self.hits = self.misses = 0
2017+
super().__init__(max_size)
20022018

20032019
def __getstate__(self):
20042020
return (self._max_size, self._current_size,
@@ -2011,23 +2027,13 @@ def __setstate__(self, state):
20112027
self.misses) = state
20122028
self._mutex = Lock()
20132029

2014-
def __len__(self):
2015-
return len(self._keys())
2016-
2017-
def __iter__(self):
2018-
return self.keys()
2019-
20202030
def __contains__(self, key):
20212031
with self._mutex:
20222032
return key in self._keys()
20232033

20242034
def clear(self):
20252035
self.invalidate()
20262036

2027-
def keys(self):
2028-
with self._mutex:
2029-
return iter(self._keys())
2030-
20312037
def _keys(self):
20322038
return self._values_cache.keys()
20332039

@@ -2037,45 +2043,11 @@ def values(self):
20372043
def items(self):
20382044
return self._values_cache.items()
20392045

2040-
def _pop_value(self):
2041-
# remove the first value from the cache, as this will be the least recently
2042-
# used value
2043-
_, v = self._values_cache.popitem(last=False)
2044-
return v
2045-
2046-
def _accommodate_value(self, value_size):
2047-
if self._max_size is None:
2048-
return
2049-
# ensure there is enough space in the cache for a new value
2050-
while self._current_size + value_size > self._max_size:
2051-
v = self._pop_value()
2052-
self._current_size -= buffer_size(v)
2053-
2054-
def _cache_value(self, key, value):
2055-
# cache a value
2056-
value_size = buffer_size(value)
2057-
# check size of the value against max size, as if the value itself exceeds max
2058-
# size then we are never going to cache it
2059-
if self._max_size is None or value_size <= self._max_size:
2060-
self._accommodate_value(value_size)
2061-
self._values_cache[key] = value
2062-
self._current_size += value_size
2063-
20642046
def invalidate(self):
20652047
"""Completely clear the cache."""
20662048
with self._mutex:
20672049
self._values_cache.clear()
20682050

2069-
def invalidate_values(self):
2070-
"""Clear the values cache."""
2071-
with self._mutex:
2072-
self._values_cache.clear()
2073-
2074-
def _invalidate_value(self, key):
2075-
if key in self._values_cache:
2076-
value = self._values_cache.pop(key)
2077-
self._current_size -= buffer_size(value)
2078-
20792051
def __getitem__(self, key):
20802052
try:
20812053
# try to obtain the value from the cache

0 commit comments

Comments
 (0)