@@ -1695,7 +1695,75 @@ def __len__(self):
16951695 return self .db .stat ()['entries' ]
16961696
16971697
1698- class LRUStoreCache (MutableMapping ):
1698+ class LRUMappingCache (MutableMapping ):
1699+ """Abstract base class for Mapping Cache
1700+ """
1701+
1702+ def __init__ (self , max_size ):
1703+ self ._max_size = max_size
1704+ self ._current_size = 0
1705+ self ._values_cache = OrderedDict ()
1706+ self ._mutex = Lock ()
1707+ self .hits = self .misses = 0
1708+
1709+ def __len__ (self ):
1710+ return len (self ._keys ())
1711+
1712+ def __iter__ (self ):
1713+ return self .keys ()
1714+
1715+ def keys (self ):
1716+ with self ._mutex :
1717+ return iter (self ._keys ())
1718+
1719+ def _keys (self ):
1720+ raise NotImplementedError
1721+
1722+ def _pop_value (self ):
1723+ # remove the first value from the cache, as this will be the least recently
1724+ # used value
1725+ _ , v = self ._values_cache .popitem (last = False )
1726+ return v
1727+
1728+ def _accommodate_value (self , value_size ):
1729+ if self ._max_size is None :
1730+ return
1731+ # ensure there is enough space in the cache for a new value
1732+ while self ._current_size + value_size > self ._max_size :
1733+ v = self ._pop_value ()
1734+ self ._current_size -= buffer_size (v )
1735+
1736+ def _cache_value (self , key , value ):
1737+ # cache a value
1738+ value_size = buffer_size (value )
1739+ # check size of the value against max size, as if the value itself exceeds max
1740+ # size then we are never going to cache it
1741+ if self ._max_size is None or value_size <= self ._max_size :
1742+ self ._accommodate_value (value_size )
1743+ self ._values_cache [key ] = value
1744+ self ._current_size += value_size
1745+
1746+ def invalidate_values (self ):
1747+ """Clear the values cache."""
1748+ with self ._mutex :
1749+ self ._values_cache .clear ()
1750+
1751+ def _invalidate_value (self , key ):
1752+ if key in self ._values_cache :
1753+ value = self ._values_cache .pop (key )
1754+ self ._current_size -= buffer_size (value )
1755+
1756+ def __getitem__ (self , key ):
1757+ raise NotImplementedError
1758+
1759+ def __setitem__ (self , key , value ):
1760+ raise NotImplementedError
1761+
1762+ def __delitem__ (self , key ):
1763+ raise NotImplementedError
1764+
1765+
1766+ class LRUStoreCache (LRUMappingCache ):
16991767 """Storage class that implements a least-recently-used (LRU) cache layer over
17001768 some other store. Intended primarily for use with stores that can be slow to
17011769 access, e.g., remote stores that require network communication to store and
@@ -1733,15 +1801,11 @@ class LRUStoreCache(MutableMapping):
17331801 """
17341802
17351803 def __init__ (self , store , max_size ):
1804+ super ().__init__ (max_size )
17361805 self ._store = store
1737- self ._max_size = max_size
1738- self ._current_size = 0
17391806 self ._keys_cache = None
17401807 self ._contains_cache = None
17411808 self ._listdir_cache = dict ()
1742- self ._values_cache = OrderedDict ()
1743- self ._mutex = Lock ()
1744- self .hits = self .misses = 0
17451809
17461810 def __getstate__ (self ):
17471811 return (self ._store , self ._max_size , self ._current_size , self ._keys_cache ,
@@ -1754,12 +1818,6 @@ def __setstate__(self, state):
17541818 self .misses ) = state
17551819 self ._mutex = Lock ()
17561820
1757- def __len__ (self ):
1758- return len (self ._keys ())
1759-
1760- def __iter__ (self ):
1761- return self .keys ()
1762-
17631821 def __contains__ (self , key ):
17641822 with self ._mutex :
17651823 if self ._contains_cache is None :
@@ -1770,10 +1828,6 @@ def clear(self):
17701828 self ._store .clear ()
17711829 self .invalidate ()
17721830
1773- def keys (self ):
1774- with self ._mutex :
1775- return iter (self ._keys ())
1776-
17771831 def _keys (self ):
17781832 if self ._keys_cache is None :
17791833 self ._keys_cache = list (self ._store .keys ())
@@ -1791,41 +1845,12 @@ def listdir(self, path=None):
17911845 def getsize (self , path = None ):
17921846 return getsize (self ._store , path = path )
17931847
1794- def _pop_value (self ):
1795- # remove the first value from the cache, as this will be the least recently
1796- # used value
1797- _ , v = self ._values_cache .popitem (last = False )
1798- return v
1799-
1800- def _accommodate_value (self , value_size ):
1801- if self ._max_size is None :
1802- return
1803- # ensure there is enough space in the cache for a new value
1804- while self ._current_size + value_size > self ._max_size :
1805- v = self ._pop_value ()
1806- self ._current_size -= buffer_size (v )
1807-
1808- def _cache_value (self , key , value ):
1809- # cache a value
1810- value_size = buffer_size (value )
1811- # check size of the value against max size, as if the value itself exceeds max
1812- # size then we are never going to cache it
1813- if self ._max_size is None or value_size <= self ._max_size :
1814- self ._accommodate_value (value_size )
1815- self ._values_cache [key ] = value
1816- self ._current_size += value_size
1817-
18181848 def invalidate (self ):
18191849 """Completely clear the cache."""
18201850 with self ._mutex :
18211851 self ._values_cache .clear ()
18221852 self ._invalidate_keys ()
18231853
1824- def invalidate_values (self ):
1825- """Clear the values cache."""
1826- with self ._mutex :
1827- self ._values_cache .clear ()
1828-
18291854 def invalidate_keys (self ):
18301855 """Clear the keys cache."""
18311856 with self ._mutex :
@@ -1836,11 +1861,6 @@ def _invalidate_keys(self):
18361861 self ._contains_cache = None
18371862 self ._listdir_cache .clear ()
18381863
1839- def _invalidate_value (self , key ):
1840- if key in self ._values_cache :
1841- value = self ._values_cache .pop (key )
1842- self ._current_size -= buffer_size (value )
1843-
18441864 def __getitem__ (self , key ):
18451865 try :
18461866 # first try to obtain the value from the cache
@@ -1994,11 +2014,7 @@ class LRUChunkCache(MutableMapping):
19942014 """
19952015
19962016 def __init__ (self , max_size ):
1997- self ._max_size = max_size
1998- self ._current_size = 0
1999- self ._values_cache = OrderedDict ()
2000- self ._mutex = Lock ()
2001- self .hits = self .misses = 0
2017+ super ().__init__ (max_size )
20022018
20032019 def __getstate__ (self ):
20042020 return (self ._max_size , self ._current_size ,
@@ -2011,23 +2027,13 @@ def __setstate__(self, state):
20112027 self .misses ) = state
20122028 self ._mutex = Lock ()
20132029
2014- def __len__ (self ):
2015- return len (self ._keys ())
2016-
2017- def __iter__ (self ):
2018- return self .keys ()
2019-
20202030 def __contains__ (self , key ):
20212031 with self ._mutex :
20222032 return key in self ._keys ()
20232033
20242034 def clear (self ):
20252035 self .invalidate ()
20262036
2027- def keys (self ):
2028- with self ._mutex :
2029- return iter (self ._keys ())
2030-
20312037 def _keys (self ):
20322038 return self ._values_cache .keys ()
20332039
@@ -2037,45 +2043,11 @@ def values(self):
20372043 def items (self ):
20382044 return self ._values_cache .items ()
20392045
2040- def _pop_value (self ):
2041- # remove the first value from the cache, as this will be the least recently
2042- # used value
2043- _ , v = self ._values_cache .popitem (last = False )
2044- return v
2045-
2046- def _accommodate_value (self , value_size ):
2047- if self ._max_size is None :
2048- return
2049- # ensure there is enough space in the cache for a new value
2050- while self ._current_size + value_size > self ._max_size :
2051- v = self ._pop_value ()
2052- self ._current_size -= buffer_size (v )
2053-
2054- def _cache_value (self , key , value ):
2055- # cache a value
2056- value_size = buffer_size (value )
2057- # check size of the value against max size, as if the value itself exceeds max
2058- # size then we are never going to cache it
2059- if self ._max_size is None or value_size <= self ._max_size :
2060- self ._accommodate_value (value_size )
2061- self ._values_cache [key ] = value
2062- self ._current_size += value_size
2063-
20642046 def invalidate (self ):
20652047 """Completely clear the cache."""
20662048 with self ._mutex :
20672049 self ._values_cache .clear ()
20682050
2069- def invalidate_values (self ):
2070- """Clear the values cache."""
2071- with self ._mutex :
2072- self ._values_cache .clear ()
2073-
2074- def _invalidate_value (self , key ):
2075- if key in self ._values_cache :
2076- value = self ._values_cache .pop (key )
2077- self ._current_size -= buffer_size (value )
2078-
20792051 def __getitem__ (self , key ):
20802052 try :
20812053 # try to obtain the value from the cache
0 commit comments