Skip to content

Commit 7105939

Browse files
authored
test(codecs): check numcodecs filters encode to their declared shape (#4591)
Replace the PackBits round-trip test, which passed with or without the gh-4553 fix, with one parametrized test asserting that every numcodecs filter's encoded chunk has the shape `resolve_metadata` declares. It covers the 2-element PackBits chunk, where the packed length equals the chunk size. Rewrite the 4560 changelog fragment without private names. Assisted-by: ClaudeCode:claude-opus-5-5
1 parent 9e44021 commit 7105939

2 files changed

Lines changed: 37 additions & 22 deletions

File tree

‎changes/4560.bugfix.md‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
Encoding a chunk through a numcodecs filter no longer passes the
2-
filter's flattened output to the next codec: `_encode_sync` reshapes the
3-
flattened array to the filter's output shape (from
4-
`resolve_metadata`), so downstream codecs (e.g. `TransposeCodec` or an `imagecodecs` serializer chained after a
5-
`Delta` or `FixedScaleOffset` filter) receive a chunk of the expected
6-
shape instead of a flat 1-D array that could make writing fail or
7-
silently corrupt data. Shape-changing codecs such as `PackBits` get
8-
their own resolved output shape.
1+
Numcodecs filters such as `Delta` and `FixedScaleOffset` now pass chunks of
2+
their declared shape to the next codec when writing. Before, the next codec
3+
received a flattened 1-D chunk, so chaining a filter with `TransposeCodec` or
4+
a 2-D image codec failed or lost data.

‎tests/test_codecs/test_numcodecs.py‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls
1313
from zarr.codecs import numcodecs as _numcodecs
1414
from zarr.codecs.transpose import TransposeCodec
15+
from zarr.core.array_spec import ArrayConfig, ArraySpec
16+
from zarr.core.buffer import default_buffer_prototype
17+
from zarr.core.dtype import get_data_type_from_native_dtype
1518
from zarr.errors import UnknownCodecError
1619
from zarr.registry import get_codec_class, get_numcodec
1720

@@ -214,22 +217,38 @@ def test_generic_filter_flattened_output_reshaped() -> None:
214217
np.testing.assert_array_equal(data, b[:, :])
215218

216219

217-
def test_generic_filter_packedbits_output_shape() -> None:
218-
# PackBits changes the size of the chunk, so `_encode_sync` must reshape its
219-
# output to the shape from `resolve_metadata`, not the input chunk shape
220-
data = np.tril(np.ones((16, 16), dtype=bool))
221-
222-
a = create_array(
223-
{},
224-
shape=data.shape,
225-
chunks=(16, 16),
226-
dtype=data.dtype,
227-
fill_value=0,
228-
filters=[_numcodecs.PackBits()],
220+
@pytest.mark.parametrize(
221+
("codec", "dtype", "shape"),
222+
[
223+
(_numcodecs.Delta(dtype="int64"), "int64", (4, 6)),
224+
(_numcodecs.FixedScaleOffset(offset=0, scale=1), "int64", (4, 6)),
225+
(_numcodecs.Quantize(digits=3), "float32", (4, 6)),
226+
(_numcodecs.BitRound(keepbits=10), "float32", (4, 6)),
227+
(_numcodecs.AsType(encode_dtype="float32", decode_dtype="float64"), "float64", (4, 6)),
228+
(_numcodecs.PackBits(), "bool", (16, 16)),
229+
# the packed length (1 + ceil(2 / 8) == 2) equals the chunk size
230+
(_numcodecs.PackBits(), "bool", (1, 2)),
231+
],
232+
ids=["delta", "fixedscaleoffset", "quantize", "bitround", "astype", "packbits", "packbits-2"],
233+
)
234+
def test_generic_filter_encoded_shape(
235+
codec: _numcodecs._NumcodecsArrayArrayCodec, dtype: str, shape: tuple[int, ...]
236+
) -> None:
237+
# gh-4553: the next codec receives the chunk shape that resolve_metadata declares
238+
zdtype = get_data_type_from_native_dtype(np.dtype(dtype))
239+
spec = ArraySpec(
240+
shape=shape,
241+
dtype=zdtype,
242+
fill_value=zdtype.cast_scalar(0),
243+
config=ArrayConfig(order="C", write_empty_chunks=True),
244+
prototype=default_buffer_prototype(),
229245
)
246+
codec = codec.evolve_from_array_spec(spec)
247+
chunk = default_buffer_prototype().nd_buffer.from_ndarray_like(np.ones(shape, dtype=dtype))
230248

231-
a[:, :] = data
232-
np.testing.assert_array_equal(data, a[:, :])
249+
encoded = codec._encode_sync(chunk, spec)
250+
251+
assert encoded.shape == codec.resolve_metadata(spec).shape
233252

234253

235254
def test_generic_filter_bitround() -> None:

0 commit comments

Comments
 (0)