Skip to content

Commit 26b7d2d

Browse files
committed
Implement getitems() always
1 parent 7c26e6c commit 26b7d2d

2 files changed

Lines changed: 59 additions & 26 deletions

File tree

‎zarr/_storage/store.py‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
import os
33
from collections.abc import MutableMapping
44
from string import ascii_letters, digits
5-
from typing import Any, List, Mapping, Optional, Union
5+
from typing import Any, Iterable, List, Mapping, Optional, Union
6+
7+
from numcodecs.ndarray_like import NDArrayLike
68

79
from zarr.meta import Metadata2, Metadata3
810
from zarr.util import normalize_storage_path
@@ -129,6 +131,43 @@ def _ensure_store(store: Any):
129131
f"wrap it in Zarr.storage.KVStore. Got {store}"
130132
)
131133

134+
def getitems(
135+
self, keys: Iterable[str], meta_array: NDArrayLike, *, on_error: str = "omit"
136+
) -> Mapping[str, Any]:
137+
"""Retrieve data from multiple keys.
138+
139+
Parameters
140+
----------
141+
keys : Iterable[str]
142+
The keys to retrieve
143+
meta_array : array-like
144+
An array instance to use for determining the output type. For now, this is
145+
only a hint and can be ignore by the implementation, in which case the type
146+
of the output is the same as calling __getitem__() for each key in keys.
147+
on_error : str, optional
148+
The policy on how to handle exceptions when retrieving keys. For now, the
149+
only supported policy is "omit", which means that failing keys are omitted
150+
from the returned result.
151+
152+
Returns
153+
-------
154+
Mapping
155+
A collection mapping the input keys to their results.
156+
157+
Developer Notes
158+
---------------
159+
This default implementation use __getitem__() to read each key sequential and
160+
ignores the meta_array argument. Overwrite this method to implement concurrent
161+
reads of multiple keys and/or to utilize the meta_array argument.
162+
"""
163+
164+
# Please overwrite `getitems` to support non-default values of `on_error`
165+
if on_error != "omit":
166+
raise ValueError(f"{self.__class__} doesn't support on_error='{on_error}'")
167+
168+
# Please overwrite `getitems` to support non-default values of `meta_array`
169+
return {k: self[k] for k in keys if k in self}
170+
132171

133172
class Store(BaseStore):
134173
"""Abstract store class used by implementations following the Zarr v2 spec.

‎zarr/core.py‎

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1949,36 +1949,30 @@ def _chunk_getitems(self, lchunk_coords, lchunk_selection, out, lout_selection,
19491949
except TypeError: # pragma: no cover
19501950
out_is_ndarray = False
19511951

1952+
if math.prod(self.shape) <= 0:
1953+
return
1954+
19521955
# Keys to retrieve
19531956
ckeys = [self._chunk_key(ch) for ch in lchunk_coords]
19541957

1955-
# Check if we should retrieve multiple keys at a time
1956-
use_getitems = (
1957-
hasattr(self.chunk_store, "getitems")
1958-
and math.prod(self.shape) > 0
1959-
)
1960-
19611958
partial_read_decode = False
1962-
if use_getitems:
1963-
# Check if we can do a partial read
1964-
if (
1965-
self._partial_decompress
1966-
and self._compressor
1967-
and self._compressor.codec_id == "blosc"
1968-
and hasattr(self._compressor, "decode_partial")
1969-
and not fields
1970-
and self.dtype != object
1971-
):
1972-
partial_read_decode = True
1973-
cdatas = {
1974-
ckey: PartialReadBuffer(ckey, self.chunk_store)
1975-
for ckey in ckeys
1976-
if ckey in self.chunk_store
1977-
}
1978-
else:
1979-
cdatas = self.chunk_store.getitems(ckeys, on_error="omit")
1959+
# Check if we can do a partial read
1960+
if (
1961+
self._partial_decompress
1962+
and self._compressor
1963+
and self._compressor.codec_id == "blosc"
1964+
and hasattr(self._compressor, "decode_partial")
1965+
and not fields
1966+
and self.dtype != object
1967+
):
1968+
partial_read_decode = True
1969+
cdatas = {
1970+
ckey: PartialReadBuffer(ckey, self.chunk_store)
1971+
for ckey in ckeys
1972+
if ckey in self.chunk_store
1973+
}
19801974
else:
1981-
cdatas = {k: self.chunk_store[k] for k in ckeys if k in self.chunk_store}
1975+
cdatas = self.chunk_store.getitems(ckeys, meta_array=self._meta_array)
19821976

19831977
for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection):
19841978
if ckey in cdatas:

0 commit comments

Comments
 (0)