Skip to content

Commit aa2b8e2

Browse files
sehoffmannd-v-b
andauthored
fix(ArraySpec): proper and robust equality semantics for ArraySpec by checking fill_value for byte-identicality, fixes zarr-developers#3054. (zarr-developers#4183)
* fix(ArraySpec): proper and robust equality semantics for ArraySpec by checking fill_value for byte-identicality, fixes zarr-developers#3054. * fix: added extra case for unequal types * Update tests/test_array_spec.py * addressed reviewers comments * test: add end-to-end regression test for structured-dtype fills in sharded arrays, and changelog entry The new test exercises the sharding codec's chunk-spec caches with an unhashable np.void fill value (zarr-developers#3054), which the ArraySpec test suite only covers at the unit level. Assisted-by: ClaudeCode:claude-fable-5 --------- Co-authored-by: Davis Bennett <[email protected]> Co-authored-by: Davis Bennett <[email protected]>
1 parent bd24f4f commit aa2b8e2

5 files changed

Lines changed: 228 additions & 7 deletions

File tree

‎changes/4183.bugfix.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fixed `TypeError: unhashable type: 'writeable void-scalar'` when writing to sharded arrays whose fill value is a `np.void` scalar, e.g. arrays with a structured dtype.
2+
3+
`ArraySpec` equality and hashing now compare the fill value by its byte representation rather than numeric equality. As a result, two specs with a `NaN` (or `NaT`) fill value now compare equal, while fill values of `-0.0` and `0.0` now compare unequal. This also restores the sharding codec's per-chunk spec cache, which had been disabled because of this bug.

‎src/zarr/codecs/sharding.py‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -411,11 +411,7 @@ def __init__(
411411
object.__setattr__(self, "subchunk_write_order", subchunk_write_order)
412412

413413
# Use instance-local lru_cache to avoid memory leaks
414-
415-
# numpy void scalars are not hashable, which means an array spec with a fill value that is
416-
# a numpy void scalar will break the lru_cache. This is commented for now but should be
417-
# fixed. See https://github.com/zarr-developers/zarr-python/issues/3054
418-
# object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
414+
object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
419415
object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
420416
object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
421417
object.__setattr__(self, "_shard_index_size", lru_cache()(self._shard_index_size))
@@ -441,7 +437,7 @@ def __setstate__(self, state: dict[str, Any]) -> None:
441437
object.__setattr__(self, "subchunk_write_order", state["subchunk_write_order"])
442438

443439
# Use instance-local lru_cache to avoid memory leaks
444-
# object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
440+
object.__setattr__(self, "_get_chunk_spec", lru_cache()(self._get_chunk_spec))
445441
object.__setattr__(self, "_get_index_chunk_spec", lru_cache()(self._get_index_chunk_spec))
446442
object.__setattr__(self, "_get_chunks_per_shard", lru_cache()(self._get_chunks_per_shard))
447443
object.__setattr__(self, "_shard_index_size", lru_cache()(self._shard_index_size))

‎src/zarr/core/array_spec.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
from dataclasses import dataclass, fields
44
from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast
55

6+
import numpy as np
7+
68
from zarr.core.common import (
79
MemoryOrder,
810
parse_bool,
@@ -132,7 +134,7 @@ def parse_array_config(data: ArrayConfigLike | None) -> ArrayConfig:
132134
return ArrayConfig.from_dict(data)
133135

134136

135-
@dataclass(frozen=True)
137+
@dataclass(frozen=True, eq=False)
136138
class ArraySpec:
137139
shape: tuple[int, ...]
138140
dtype: ZDType[TBaseDType, TBaseScalar]
@@ -157,6 +159,24 @@ def __init__(
157159
object.__setattr__(self, "config", config)
158160
object.__setattr__(self, "prototype", prototype)
159161

162+
def _key(self) -> tuple[object, ...]:
163+
"""Returns the tuple used for equality/hash identity."""
164+
fill_value = self.fill_value
165+
if isinstance(fill_value, np.generic):
166+
# fill_values should be byte-identical, otherwise they correspond to different values in memory / on disk.
167+
# Importantly, this ensures np.nan == np.nan, NaT == NaT, and -0.0 != 0.0.
168+
# It also fixes np.void fill_values being unhashable (#3054).
169+
fill_value = fill_value.tobytes()
170+
return (self.shape, self.dtype, fill_value, self.config, self.prototype)
171+
172+
def __eq__(self, other: object) -> bool:
173+
if not isinstance(other, ArraySpec):
174+
return NotImplemented
175+
return self._key() == other._key()
176+
177+
def __hash__(self) -> int:
178+
return hash(self._key())
179+
160180
@property
161181
def ndim(self) -> int:
162182
return len(self.shape)

‎tests/test_array_spec.py‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
from __future__ import annotations
2+
3+
from typing import TYPE_CHECKING, Any
4+
5+
import numpy as np
6+
import pytest
7+
8+
from zarr.core.array_spec import ArrayConfig, ArraySpec
9+
from zarr.core.buffer import BufferPrototype, default_buffer_prototype
10+
from zarr.core.buffer.cpu import NDBuffer
11+
from zarr.core.dtype import get_data_type_from_native_dtype
12+
13+
if TYPE_CHECKING:
14+
from collections.abc import Callable
15+
16+
from zarr.core.common import MemoryOrder
17+
18+
19+
def _make_spec(
20+
*,
21+
shape: tuple[int, ...] = (4, 4),
22+
native_dtype: Any = "int16",
23+
fill_value: Any = 0,
24+
order: MemoryOrder = "C",
25+
write_empty_chunks: bool = False,
26+
prototype: BufferPrototype | None = None,
27+
) -> ArraySpec:
28+
"""Creates an ArraySpec with common defaults"""
29+
zdtype = get_data_type_from_native_dtype(np.dtype(native_dtype))
30+
fill_value = zdtype.cast_scalar(fill_value) # mirrors ArrayV3Metadata's fill_value
31+
return ArraySpec(
32+
shape=shape,
33+
dtype=zdtype,
34+
fill_value=fill_value,
35+
config=ArrayConfig(order=order, write_empty_chunks=write_empty_chunks),
36+
prototype=prototype if prototype is not None else default_buffer_prototype(),
37+
)
38+
39+
40+
class _AltNDBuffer(NDBuffer):
41+
"""A distinct NDBuffer subclass"""
42+
43+
44+
_ALT_PROTOTYPE = BufferPrototype(
45+
buffer=default_buffer_prototype().buffer,
46+
nd_buffer=_AltNDBuffer,
47+
) # a distinct BufferPrototype with a different nd_buffer subclass
48+
49+
50+
# Difficult / important cases:
51+
# issue #3054: np.void is unhashable when writeable
52+
# nan/NaT aren't self-equal yet must compare equal for a ArraySpec
53+
SPECS = [
54+
pytest.param({"native_dtype": "int16", "fill_value": 7}, id="int16"),
55+
pytest.param({"native_dtype": "float64", "fill_value": 1.5}, id="float64"),
56+
pytest.param({"native_dtype": "float64", "fill_value": float("nan")}, id="float64-nan"),
57+
pytest.param({"native_dtype": "float64", "fill_value": -0.0}, id="float64-negzero"),
58+
pytest.param({"native_dtype": "complex128", "fill_value": 1 + 2j}, id="complex128"),
59+
pytest.param(
60+
{"native_dtype": "complex128", "fill_value": complex(-0.0, -0.0)},
61+
id="complex128-negzero",
62+
),
63+
pytest.param({"native_dtype": "bool", "fill_value": True}, id="bool"),
64+
pytest.param(
65+
{"native_dtype": "datetime64[s]", "fill_value": np.datetime64("2020-01-01")},
66+
id="datetime64",
67+
),
68+
pytest.param(
69+
{"native_dtype": "datetime64[s]", "fill_value": np.datetime64("NaT", "s")},
70+
id="datetime64-NaT",
71+
),
72+
pytest.param(
73+
{"native_dtype": [("a", "f8"), ("b", "i8")], "fill_value": (1.0, 2)},
74+
id="structured-void",
75+
),
76+
pytest.param({"native_dtype": "U5", "fill_value": "hello"}, id="fixed-string"),
77+
pytest.param({"shape": ()}, id="scalar-shape"),
78+
pytest.param({"shape": (0,)}, id="zero-size"),
79+
pytest.param({"order": "F"}, id="order-F"),
80+
]
81+
82+
83+
# Mutations: each mutate kwargs to an unequal version
84+
def _grow_shape(kw: dict[str, Any]) -> dict[str, Any]:
85+
return {"shape": (*kw.get("shape", (4, 4)), 1)}
86+
87+
88+
def _flip_order(kw: dict[str, Any]) -> dict[str, Any]:
89+
return {"order": "F" if kw.get("order", "C") == "C" else "C"}
90+
91+
92+
def _swap_prototype(_kw: dict[str, Any]) -> dict[str, Any]:
93+
return {"prototype": _ALT_PROTOTYPE}
94+
95+
96+
MUTATIONS = [
97+
pytest.param(_grow_shape, id="shape"),
98+
pytest.param(_flip_order, id="order"),
99+
pytest.param(_swap_prototype, id="prototype"),
100+
]
101+
102+
103+
@pytest.mark.parametrize("kwargs", SPECS)
104+
def test_hashable(kwargs: dict[str, Any]) -> None:
105+
"""Every ArraySpec is hashable, including structured (np.void) fill values."""
106+
assert isinstance(hash(_make_spec(**kwargs)), int)
107+
108+
109+
@pytest.mark.parametrize("kwargs", SPECS)
110+
def test_equal_specs_hash_equal(kwargs: dict[str, Any]) -> None:
111+
"""Independently built specs with identical fields are equal and hash equal."""
112+
a = _make_spec(**kwargs)
113+
b = _make_spec(**kwargs)
114+
assert a == b
115+
assert hash(a) == hash(b)
116+
117+
118+
@pytest.mark.parametrize("kwargs", SPECS)
119+
@pytest.mark.parametrize("mutate", MUTATIONS)
120+
def test_distinct_specs_unequal(
121+
mutate: Callable[[dict[str, Any]], dict[str, Any]],
122+
kwargs: dict[str, Any],
123+
) -> None:
124+
"""Changing one dtype-independent field makes a spec unequal to its base."""
125+
base = _make_spec(**kwargs)
126+
variant = _make_spec(**{**kwargs, **mutate(kwargs)})
127+
assert base != variant
128+
129+
130+
@pytest.mark.parametrize(
131+
("base", "variant"),
132+
[
133+
pytest.param({"fill_value": 0}, {"fill_value": 1}, id="fill_value"),
134+
pytest.param({"native_dtype": "int16"}, {"native_dtype": "int32"}, id="dtype"),
135+
pytest.param(
136+
{"native_dtype": "float32", "fill_value": 1.0},
137+
{"native_dtype": "float64", "fill_value": 1.0},
138+
id="dtype-float-promote",
139+
),
140+
],
141+
)
142+
def test_dtype_and_fill_value_matter(base: dict[str, Any], variant: dict[str, Any]) -> None:
143+
"""dtype and fill_value participate in equality; they can't join the cross
144+
product because fill_value is coupled to dtype."""
145+
assert _make_spec(**base) != _make_spec(**variant)
146+
147+
148+
@pytest.mark.parametrize(
149+
("native_dtype", "neg_fill", "pos_fill"),
150+
[
151+
pytest.param("float16", -0.0, 0.0, id="float16"),
152+
pytest.param("float32", -0.0, 0.0, id="float32"),
153+
pytest.param("float64", -0.0, 0.0, id="float64"),
154+
pytest.param("complex128", complex(-0.0, -0.0), 0j, id="complex128-both"),
155+
pytest.param("complex128", complex(0.0, -0.0), 0j, id="complex128-imag"),
156+
pytest.param("complex128", complex(-0.0, 0.0), 0j, id="complex128-real"),
157+
pytest.param([("a", "f8")], (-0.0,), (0.0,), id="structured"),
158+
],
159+
)
160+
def test_signed_zero_fills_are_distinct(native_dtype: Any, neg_fill: Any, pos_fill: Any) -> None:
161+
"""A -0.0 fill writes different bytes than +0.0, so the specs are not equal."""
162+
neg = _make_spec(native_dtype=native_dtype, fill_value=neg_fill)
163+
pos = _make_spec(native_dtype=native_dtype, fill_value=pos_fill)
164+
assert neg != pos
165+
166+
167+
@pytest.mark.parametrize(
168+
("obj"),
169+
[
170+
pytest.param(None, id="None"),
171+
pytest.param(42, id="int"),
172+
pytest.param("hello", id="str"),
173+
pytest.param([1, 2, 3], id="list"),
174+
pytest.param({"a": 1}, id="dict"),
175+
],
176+
)
177+
def test_unequal_with_invalid_type(obj: Any) -> None:
178+
assert (_make_spec() == obj) is False
179+
assert _make_spec() != obj

‎tests/test_codecs/test_sharding.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -734,6 +734,29 @@ async def test_delete_empty_shards(store: Store) -> None:
734734
assert len(chunk_bytes) == 16 * 2 + 8 * 8 * 2 + 4
735735

736736

737+
def test_structured_dtype_fill_value() -> None:
738+
"""Sharded arrays with a structured dtype are writable and readable even though
739+
the fill value is an (unhashable) ``np.void`` scalar: the sharding codec's
740+
chunk-spec caches key on ``ArraySpec``, whose hash must handle void fills
741+
(see https://github.com/zarr-developers/zarr-python/issues/3054)."""
742+
dtype = np.dtype([("a", "i4"), ("b", "f4")])
743+
arr = zarr.create_array(
744+
MemoryStore(),
745+
shape=(8,),
746+
chunks=(2,),
747+
shards=(4,),
748+
dtype=dtype,
749+
fill_value=(1, 2.0),
750+
)
751+
data = np.array([(i, i / 2) for i in range(8)], dtype=dtype)
752+
arr[:4] = data[:4]
753+
754+
expected = np.zeros(8, dtype=dtype)
755+
expected[:4] = data[:4]
756+
expected[4:] = (1, 2.0) # untouched shard reads back as the fill value
757+
assert np.array_equal(arr[:], expected)
758+
759+
737760
def test_pickle() -> None:
738761
"""ShardingCodec round-trips through pickle, including the non-serialized
739762
``subchunk_write_order`` (which ``to_dict`` omits and which must not silently

0 commit comments

Comments
 (0)