Skip to content

Commit 08e82b5

Browse files
committed
fix: byte-order handling for structured dtypes in the bytes codec
The bytes codec neither byte-swapped structured-dtype fields to its configured endian on encode (numpy reports byteorder '|' for void dtypes, so the top-level byteorder comparison never detected a mismatch) nor honored its endian when decoding, silently corrupting any structured data whose field byte order differed from the stored one (e.g. virtual references to external big-endian data). Encode now detects byte-order mismatches by comparing full dtypes via newbyteorder, and decode reinterprets raw bytes in the stored byte order before converting to the data type's declared byte order, so the stored layout (codec state) and the in-memory layout (array data type) are independent. Closes zarr-developers#4141 Assisted-by: ClaudeCode:claude-fable-5
1 parent c9ca0b9 commit 08e82b5

3 files changed

Lines changed: 113 additions & 9 deletions

File tree

‎changes/4141.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix silent byte-order corruption for structured dtypes with the `bytes` codec: multi-byte fields are now byte-swapped to the codec's configured `endian` on write and decoded honoring it on read, so non-native-endian structured data (e.g. big-endian fields, as produced by virtual references to external data) round-trips correctly.

‎src/zarr/codecs/bytes.py‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -100,14 +100,25 @@ def _decode_sync(
100100
chunk_spec: ArraySpec,
101101
) -> NDBuffer:
102102
endian_str = self.endian
103+
dtype = chunk_spec.dtype.to_native_dtype()
104+
# The byte order of the stored data is set by this codec's `endian`
105+
# configuration; the byte order of the decoded array is set by the array's
106+
# data type. The two are independent: the raw bytes are reinterpreted in
107+
# the stored byte order, then converted to the declared dtype if needed.
103108
if isinstance(chunk_spec.dtype, HasEndianness):
104-
dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg]
109+
stored_dtype = replace(chunk_spec.dtype, endianness=endian_str).to_native_dtype() # type: ignore[call-arg]
110+
elif isinstance(chunk_spec.dtype, Struct) and endian_str is not None:
111+
# Per the struct data type spec, all multi-byte fields are stored in the
112+
# byte order configured on this codec.
113+
stored_dtype = dtype.newbyteorder(endian_str)
105114
else:
106-
dtype = chunk_spec.dtype.to_native_dtype()
115+
stored_dtype = dtype
107116
as_array_like = chunk_bytes.as_array_like()
108117
chunk_array = chunk_spec.prototype.nd_buffer.from_ndarray_like(
109-
as_array_like.view(dtype=dtype) # type: ignore[attr-defined]
118+
as_array_like.view(dtype=stored_dtype) # type: ignore[attr-defined]
110119
)
120+
if stored_dtype != dtype:
121+
chunk_array = chunk_array.astype(dtype)
111122

112123
# ensure correct chunk shape
113124
if chunk_array.shape != chunk_spec.shape:
@@ -129,13 +140,14 @@ def _encode_sync(
129140
chunk_spec: ArraySpec,
130141
) -> Buffer | None:
131142
assert isinstance(chunk_array, NDBuffer)
132-
if (
133-
chunk_array.dtype.itemsize > 1
134-
and self.endian is not None
135-
and self.endian != chunk_array.byteorder
136-
):
143+
if chunk_array.dtype.itemsize > 1 and self.endian is not None:
144+
# Compare full dtypes rather than the top-level byteorder: numpy reports
145+
# byteorder '|' for structured dtypes even when their fields are
146+
# byte-order-sensitive, so newbyteorder is the only reliable way to
147+
# detect (and normalize) a byte-order mismatch.
137148
new_dtype = chunk_array.dtype.newbyteorder(self.endian)
138-
chunk_array = chunk_array.astype(new_dtype)
149+
if new_dtype != chunk_array.dtype:
150+
chunk_array = chunk_array.astype(new_dtype)
139151

140152
nd_array = chunk_array.as_ndarray_like()
141153
# Flatten the nd-array (only copy if needed) and reinterpret as bytes

‎tests/test_codecs/test_bytes.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,56 @@ async def test_endian(
7474
assert np.array_equal(data, readback_data)
7575

7676

77+
@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
78+
@pytest.mark.parametrize("input_byteorder", [">", "<"])
79+
@pytest.mark.parametrize("store_endian", ["big", "little"])
80+
async def test_endian_structured(
81+
store: Store,
82+
input_byteorder: Literal[">", "<"],
83+
store_endian: Literal["big", "little"],
84+
) -> None:
85+
"""
86+
The `bytes` codec applies its configured byte order to every multi-byte
87+
field of a structured dtype, per the `struct` data type spec: the stored
88+
chunk is laid out with all fields in the codec's byte order regardless of
89+
the input array's field byte order, and the data reads back to the
90+
original values. This guards against the structured-dtype endianness bugs
91+
from https://github.com/zarr-developers/zarr-python/issues/4141, where the
92+
encode path never byte-swapped struct fields (numpy reports byteorder '|'
93+
for void dtypes) and the decode path ignored the codec's endian entirely.
94+
Compression is disabled so the stored chunk's byte layout can be asserted
95+
directly.
96+
"""
97+
input_dtype = np.dtype([("flux", f"{input_byteorder}f4"), ("mask", f"{input_byteorder}i4")])
98+
data = np.array([(1.5, 2), (3.5, 4), (5.5, 6)], dtype=input_dtype)
99+
path = "endian_structured"
100+
spath = StorePath(store, path)
101+
a = await zarr.api.asynchronous.create_array(
102+
spath,
103+
shape=data.shape,
104+
chunks=data.shape,
105+
dtype=input_dtype,
106+
compressors=None,
107+
serializer=BytesCodec(endian=store_endian),
108+
)
109+
110+
await _AsyncArrayProxy(a)[:].set(data)
111+
112+
# The stored chunk has every multi-byte field in the codec's byte order.
113+
stored = await store.get(f"{path}/c/0", prototype=default_buffer_prototype())
114+
assert stored is not None
115+
stored_byteorder = ">" if store_endian == "big" else "<"
116+
expected_dtype = np.dtype(
117+
[("flux", f"{stored_byteorder}f4"), ("mask", f"{stored_byteorder}i4")]
118+
)
119+
assert stored.to_bytes() == data.astype(expected_dtype).tobytes()
120+
121+
# ... and the data reads back to the original values.
122+
readback_data = np.asarray(await _AsyncArrayProxy(a)[:].get())
123+
for field in ("flux", "mask"):
124+
assert np.array_equal(data[field], readback_data[field])
125+
126+
77127
def test_bytes_codec_supports_sync() -> None:
78128
assert isinstance(BytesCodec(), SupportsSyncCodec)
79129

@@ -99,6 +149,47 @@ def test_bytes_codec_sync_roundtrip() -> None:
99149
np.testing.assert_array_equal(arr, decoded.as_numpy_array())
100150

101151

152+
@pytest.mark.parametrize("endian", ENDIAN)
153+
@pytest.mark.parametrize(
154+
"native_dtype",
155+
[np.dtype(">u2"), np.dtype("<u2"), np.dtype([("a", ">f4"), ("b", "<i4")])],
156+
ids=["big-scalar", "little-scalar", "mixed-endian-struct"],
157+
)
158+
def test_bytes_codec_byte_orders_are_independent(
159+
endian: EndianLiteral, native_dtype: np.dtype[Any]
160+
) -> None:
161+
"""
162+
The codec's `endian` configuration governs only the stored byte layout;
163+
the byte order of the decoded array is governed by the array's data type.
164+
Encoding lays out every multi-byte value (including struct fields) in the
165+
codec's byte order regardless of the input array's byte order, and decoding
166+
returns a buffer in the data type's declared byte order regardless of the
167+
codec's. The mixed-endian struct case pins that per-field byte order of the
168+
in-memory dtype survives a roundtrip through a single stored byte order.
169+
"""
170+
if native_dtype.fields is None:
171+
data = np.arange(4, dtype=native_dtype)
172+
else:
173+
data = np.array([(1.5, 2), (3.5, 4), (5.5, 6), (7.5, 8)], dtype=native_dtype)
174+
zdtype = get_data_type_from_native_dtype(native_dtype)
175+
spec = ArraySpec(
176+
shape=data.shape,
177+
dtype=zdtype,
178+
fill_value=zdtype.cast_scalar(0),
179+
config=ArrayConfig(order="C", write_empty_chunks=True),
180+
prototype=default_buffer_prototype(),
181+
)
182+
codec = BytesCodec(endian=endian)
183+
184+
encoded = codec._encode_sync(default_buffer_prototype().nd_buffer.from_numpy_array(data), spec)
185+
assert encoded is not None
186+
assert encoded.to_bytes() == data.astype(native_dtype.newbyteorder(endian)).tobytes()
187+
188+
decoded = codec._decode_sync(encoded, spec)
189+
assert decoded.dtype == zdtype.to_native_dtype()
190+
np.testing.assert_array_equal(decoded.as_numpy_array(), data)
191+
192+
102193
@pytest.mark.parametrize("endian", ENDIAN)
103194
def test_bytes_codec_accepts_all_endians(endian: EndianLiteral) -> None:
104195
"""

0 commit comments

Comments
 (0)