Skip to content

Commit 3ecee99

Browse files
committed
Refactor logic into zarr.util.retry_call
1 parent 7e37e4e commit 3ecee99

3 files changed

Lines changed: 54 additions & 17 deletions

File tree

‎zarr/storage.py‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
from zarr.meta import encode_array_metadata, encode_group_metadata
5454
from zarr.util import (buffer_size, json_loads, nolock, normalize_chunks,
5555
normalize_dtype, normalize_fill_value, normalize_order,
56-
normalize_shape, normalize_storage_path)
56+
normalize_shape, normalize_storage_path, retry_call)
5757

5858
__doctest_requires__ = {
5959
('RedisStore', 'RedisStore.*'): ['redis'],
@@ -859,22 +859,10 @@ def __setitem__(self, key, value):
859859
temp_path = os.path.join(dir_path, temp_name)
860860
try:
861861
self._tofile(value, temp_path)
862-
862+
# move temporary file into place;
863863
# make several attempts at writing the temporary file to get past
864864
# potential antivirus file locking issues
865-
attempts = 0
866-
while attempts < 10:
867-
try:
868-
# move temporary file into place
869-
os.replace(temp_path, file_path)
870-
break
871-
except PermissionError:
872-
# wait for file lock to release before attempting again
873-
time.sleep(0.1)
874-
attempts += 1
875-
if attempts == 10:
876-
os.replace(temp_path, file_path)
877-
865+
retry_call(os.replace, temp_path, file_path)
878866
finally:
879867
# clean up if temp file still exists for whatever reason
880868
if os.path.exists(temp_path): # pragma: no cover

‎zarr/tests/test_util.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from zarr.util import (guess_chunks, human_readable_size, info_html_report,
88
info_text_report, is_total_slice, normalize_chunks,
99
normalize_fill_value, normalize_order,
10-
normalize_resize_args, normalize_shape,
10+
normalize_resize_args, normalize_shape, retry_call,
1111
tree_array_icon, tree_group_icon, tree_get_icon,
1212
tree_widget)
1313

@@ -175,3 +175,20 @@ def test_tree_widget_missing_ipytree():
175175
)
176176
with pytest.raises(ImportError, match=re.escape(pattern)):
177177
tree_widget(None, None, None)
178+
179+
def test_retry_call():
180+
181+
class F:
182+
183+
def __init__(self):
184+
self.c = 0
185+
186+
def __call__(self):
187+
self.c += 1
188+
if self.c == 1:
189+
raise PermissionError();
190+
191+
retry_call(F(), exceptions=(PermissionError,), wait=0)
192+
193+
pytest.raises(PermissionError,
194+
retry_call, F(), exceptions=(NotImplementedError,), wait=0)

‎zarr/util.py‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import numbers
55
from textwrap import TextWrapper
66
import mmap
7+
import time
78

89
import numpy as np
910
from asciitree import BoxStyle, LeftAligned
@@ -12,7 +13,7 @@
1213
from numcodecs.registry import codec_registry
1314
from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo
1415

15-
from typing import Any, Dict, Tuple, Union
16+
from typing import Any, Callable, Dict, Tuple, Union
1617

1718
# codecs to use for object dtype convenience API
1819
object_codecs = {
@@ -609,3 +610,34 @@ def read_part(self, start, nitems):
609610

610611
def read_full(self):
611612
return self.chunk_store[self.store_key]
613+
614+
615+
def retry_call(
616+
callable: Callable,
617+
args: Tuple[Any]=None,
618+
kwargs: Dict[Any, Any]=None,
619+
exceptions=(),
620+
retries=10,
621+
wait=0.1,
622+
) -> Any:
623+
"""
624+
Make several attempts to invoke the callable. If one of the given exceptions
625+
is raised, wait the given period of time and retry up to the given number of
626+
retries.
627+
"""
628+
629+
if args is None:
630+
args = ()
631+
if kwargs is None:
632+
kwargs = {}
633+
634+
attempts = 0
635+
while attempts < retries:
636+
try:
637+
return callable(*args, **kwargs)
638+
break
639+
except exceptions:
640+
time.sleep(wait)
641+
attempts += 1
642+
if attempts == retries:
643+
return callable(*args, **kwargs)

0 commit comments

Comments
 (0)