Skip to content

Commit a1ded92

Browse files
stanbot8dcherianheadtr1ckpre-commit-ci[bot]
authored
fix: preserve stringdtype in zarr v3 round trips (#11474)
Co-authored-by: Deepak Cherian <[email protected]> Co-authored-by: Michael Niklas <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 5bf7aa2 commit a1ded92

4 files changed

Lines changed: 81 additions & 12 deletions

File tree

‎doc/whats-new.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ Bug Fixes
9191
which caused ``AsyncMock`` objects to leak through instead of real array data
9292
(:pull:`11232`).
9393
By `Joe Hamman <https://github.com/jhamman>`_.
94+
- Preserve NumPy ``StringDType`` variables and coordinates in Zarr format 3
95+
round trips (:issue:`11466`, :pull:`11474`).
96+
By `stanbot8 <https://github.com/stanbot8>`_.
9497
- Fixed dask-backed bottleneck rolling reductions declaring a dtype that could
9598
differ from the dtype returned by the matching numpy-backed bottleneck path,
9699
notably ``object`` instead of ``float64`` for boolean inputs.

‎xarray/backends/zarr.py‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -235,20 +235,25 @@ def __init__(self, zarr_array):
235235
self._array = zarr_array
236236
self.shape = self._array.shape
237237

238-
# preserve vlen string object dtype (GH 7328)
238+
# preserve vlen string object dtype (GH 7328), but keep a native StringDType
239+
dtype = self._array.dtype
239240
if (
240-
self._array.serializer
241+
dtype.kind != "T"
242+
and self._array.serializer
241243
and self._array.serializer.to_dict()["name"] == "vlen-utf8"
242244
):
243245
dtype = coding.strings.create_vlen_dtype(str)
244-
else:
245-
dtype = self._array.dtype
246246

247247
self.dtype = dtype
248248

249249
def get_array(self):
250250
return self._array
251251

252+
def _preserve_string_dtype(self, value):
253+
if self.dtype.kind == "T" and isinstance(value, str):
254+
return np.asarray(value, dtype=self.dtype)
255+
return value
256+
252257
def _oindex(self, key):
253258
return self._array.oindex[key]
254259

@@ -288,13 +293,15 @@ def __getitem__(self, key):
288293
method = self._vindex
289294
elif isinstance(key, indexing.OuterIndexer):
290295
method = self._oindex
291-
return indexing.explicit_indexing_adapter(
296+
value = indexing.explicit_indexing_adapter(
292297
key, array.shape, indexing.IndexingSupport.VECTORIZED, method
293298
)
294299

295300
# if self.ndim == 0:
296301
# could possibly have a work-around for 0d data here
297302

303+
return self._preserve_string_dtype(value)
304+
298305
async def async_getitem(self, key):
299306
array = self._array
300307
if isinstance(key, indexing.BasicIndexer):
@@ -303,9 +310,10 @@ async def async_getitem(self, key):
303310
method = self._async_vindex
304311
elif isinstance(key, indexing.OuterIndexer):
305312
method = self._async_oindex
306-
return await indexing.async_explicit_indexing_adapter(
313+
value = await indexing.async_explicit_indexing_adapter(
307314
key, array.shape, indexing.IndexingSupport.VECTORIZED, method
308315
)
316+
return self._preserve_string_dtype(value)
309317

310318

311319
def _determine_zarr_chunks(enc_chunks, var_chunks, ndim, name):
@@ -512,7 +520,9 @@ def extract_zarr_variable_encoding(
512520

513521
# Function below is copied from conventions.encode_cf_variable.
514522
# The only change is to raise an error for object dtypes.
515-
def encode_zarr_variable(var, needs_copy=True, name=None):
523+
def encode_zarr_variable(
524+
var, needs_copy=True, name=None, *, zarr_format: ZarrFormat | None = None
525+
):
516526
"""
517527
Converts a Variable into another Variable which follows some
518528
of the CF conventions:
@@ -535,6 +545,13 @@ def encode_zarr_variable(var, needs_copy=True, name=None):
535545
var = conventions.encode_cf_variable(var, name=name, coders=ZARR_CODERS)
536546
var = ensure_dtype_not_object(var, name=name)
537547

548+
if (
549+
zarr_format == 3
550+
and var.dtype.kind == "T"
551+
and var.dtype == np.dtypes.StringDType()
552+
):
553+
return var
554+
538555
# zarr allows unicode, but not variable-length strings, so it's both
539556
# simpler and more compact to always encode as UTF-8 explicitly.
540557
# TODO: allow toggling this explicitly via dtype in encoding.
@@ -854,17 +871,20 @@ def ds(self):
854871

855872
def open_store_variable(self, name):
856873
zarr_array = self.members[name]
857-
data = indexing.LazilyIndexedArray(ZarrArrayWrapper(zarr_array))
858874
try_nczarr = self._mode == "r"
859875
dimensions, attributes = _get_zarr_dims_and_attrs(
860876
zarr_array, DIMENSION_KEY, try_nczarr
861877
)
862878
attributes = dict(attributes)
879+
array_wrapper = ZarrArrayWrapper(zarr_array)
880+
data = indexing.LazilyIndexedArray(array_wrapper)
863881

864882
encoding = {
865883
"chunks": zarr_array.chunks,
866884
"preferred_chunks": dict(zip(dimensions, zarr_array.chunks, strict=True)),
867885
}
886+
if array_wrapper.dtype.kind == "T":
887+
encoding["dtype"] = array_wrapper.dtype
868888

869889
encoding.update(
870890
{
@@ -930,7 +950,8 @@ def set_attributes(self, attributes):
930950
_put_attrs(self.zarr_group, attributes)
931951

932952
def encode_variable(self, variable, name=None):
933-
variable = encode_zarr_variable(variable, name=name)
953+
zarr_format = self.zarr_group.metadata.zarr_format
954+
variable = encode_zarr_variable(variable, name=name, zarr_format=zarr_format)
934955
return variable
935956

936957
def encode_attribute(self, a):

‎xarray/coding/variables.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -680,10 +680,12 @@ def encode(self):
680680
raise NotImplementedError
681681

682682
def decode(self, variable: Variable, name: T_Name = None) -> Variable:
683-
if variable.dtype.kind == "T":
683+
if (
684+
variable.dtype.kind == "T"
685+
and variable.encoding.get("dtype") != variable.dtype
686+
):
684687
return variable.astype(object)
685-
else:
686-
return variable
688+
return variable
687689

688690

689691
class NativeEnumCoder(VariableCoder):

‎xarray/tests/test_backends.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
requires_zarr,
105105
requires_zarr_v3,
106106
requires_zarr_v3_async_oindex,
107+
requires_zarr_v3_dtypes,
107108
)
108109
from xarray.tests.test_coding_times import (
109110
_ALL_CALENDARS,
@@ -4127,6 +4128,48 @@ def test_region_write(self) -> None:
41274128
self.check_requests(expected, patches)
41284129

41294130

4131+
@requires_zarr_v3_dtypes
4132+
@pytest.mark.skipif(not HAS_STRING_DTYPE, reason="requires StringDType")
4133+
def test_roundtrip_stringdtype_zarr_v3() -> None:
4134+
dtype = np.dtypes.StringDType()
4135+
data = np.array(["a", "bb", "ccc"], dtype=dtype)
4136+
expected = Dataset(
4137+
{
4138+
"data": ("dim", data.copy()),
4139+
"scalar": np.array("a", dtype=dtype),
4140+
},
4141+
coords={
4142+
"dim": ("dim", data.copy()),
4143+
"nondim": ("dim", data.copy()),
4144+
},
4145+
)
4146+
store = zarr.storage.MemoryStore({}, read_only=False)
4147+
4148+
with assert_no_warnings():
4149+
expected.to_zarr(store, zarr_format=3, consolidated=False)
4150+
actual = xr.open_zarr(store, consolidated=False).load()
4151+
4152+
for name in expected.variables:
4153+
assert zarr.open_array(store=store, path=str(name), mode="r").dtype == dtype
4154+
assert actual[name].dtype == dtype
4155+
assert_identical(expected, actual)
4156+
4157+
4158+
@requires_zarr_v3_dtypes
4159+
@pytest.mark.skipif(not HAS_STRING_DTYPE, reason="requires StringDType")
4160+
def test_stringdtype_with_na_object_uses_the_compatibility_path() -> None:
4161+
dtype = np.dtypes.StringDType(na_object=np.nan)
4162+
data = np.array(["a", "bb", "ccc"], dtype=dtype)
4163+
expected = Dataset({"data": ("dim", data.copy())})
4164+
store = zarr.storage.MemoryStore({}, read_only=False)
4165+
4166+
expected.to_zarr(store, zarr_format=3, consolidated=False)
4167+
actual = xr.open_zarr(store, consolidated=False).load()
4168+
4169+
assert zarr.open_array(store=store, path="data", mode="r").dtype.kind != "T"
4170+
assert (actual["data"].values == data.astype(object)).all()
4171+
4172+
41304173
@requires_zarr
41314174
class TestZarrDictStore(ZarrBase):
41324175
@contextlib.contextmanager

0 commit comments

Comments
 (0)