Skip to content

Commit 3fae14d

Browse files
authored
Merge branch 'main' into feat/zarr-metadata-v3-package-reexports
2 parents 025a31f + 7e81965 commit 3fae14d

3 files changed

Lines changed: 54 additions & 1 deletion

File tree

‎changes/4560.bugfix.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Encoding a chunk through a numcodecs filter no longer passes the
2+
filter's flattened output to the next codec: `_encode_sync` reshapes the
3+
flattened array to the filter's output shape (from
4+
`resolve_metadata`), so downstream codecs (e.g. `TransposeCodec` or an `imagecodecs` serializer chained after a
5+
`Delta` or `FixedScaleOffset` filter) receive a chunk of the expected
6+
shape instead of a flat 1-D array that could make writing fail or
7+
silently corrupt data. Shape-changing codecs such as `PackBits` get
8+
their own resolved output shape.

‎src/zarr/codecs/numcodecs/_codecs.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,12 @@ def _encode_sync(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer:
164164
# would be encoded in transposed element order (gh-3558)
165165
chunk_ndarray = np.ascontiguousarray(chunk_data.as_ndarray_like())
166166
out = self._codec.encode(chunk_ndarray)
167-
return chunk_spec.prototype.nd_buffer.from_ndarray_like(out)
167+
# numcodecs filters return a flattened array; reshape it to the
168+
# output shape so downstream codecs receive the chunk shape that
169+
# resolve_metadata promised them (gh-4553)
170+
return chunk_spec.prototype.nd_buffer.from_ndarray_like(
171+
out.reshape(self.resolve_metadata(chunk_spec).shape)
172+
)
168173

169174
async def _encode_single(self, chunk_data: NDBuffer, chunk_spec: ArraySpec) -> NDBuffer:
170175
return await asyncio.to_thread(self._encode_sync, chunk_data, chunk_spec)

‎tests/test_codecs/test_numcodecs.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from zarr import config, create_array, open_array
1212
from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls
1313
from zarr.codecs import numcodecs as _numcodecs
14+
from zarr.codecs.transpose import TransposeCodec
1415
from zarr.errors import UnknownCodecError
1516
from zarr.registry import get_codec_class, get_numcodec
1617

@@ -192,6 +193,45 @@ def test_generic_filter_f_contiguous(
192193
np.testing.assert_array_equal(data, a[:, :])
193194

194195

196+
def test_generic_filter_flattened_output_reshaped() -> None:
197+
# gh-4553: numcodecs filters return a flattened array; `_encode_sync` must
198+
# reshape it back to the chunk shape, so downstream codecs receive chunks
199+
# with the shape the pipeline promised them
200+
data = np.arange(256, dtype="int64").reshape(16, 16)
201+
202+
a = create_array(
203+
{},
204+
shape=data.shape,
205+
chunks=(16, 16),
206+
dtype=data.dtype,
207+
fill_value=0,
208+
filters=[_numcodecs.Delta(dtype="int64"), TransposeCodec(order=(1, 0))],
209+
)
210+
211+
a[:, :] = data
212+
with codec_conf():
213+
b = open_array(a.store, mode="r")
214+
np.testing.assert_array_equal(data, b[:, :])
215+
216+
217+
def test_generic_filter_packedbits_output_shape() -> None:
218+
# PackBits changes the size of the chunk, so `_encode_sync` must reshape its
219+
# output to the shape from `resolve_metadata`, not the input chunk shape
220+
data = np.tril(np.ones((16, 16), dtype=bool))
221+
222+
a = create_array(
223+
{},
224+
shape=data.shape,
225+
chunks=(16, 16),
226+
dtype=data.dtype,
227+
fill_value=0,
228+
filters=[_numcodecs.PackBits()],
229+
)
230+
231+
a[:, :] = data
232+
np.testing.assert_array_equal(data, a[:, :])
233+
234+
195235
def test_generic_filter_bitround() -> None:
196236
data = np.linspace(0, 1, 256, dtype="float32").reshape((16, 16))
197237

0 commit comments

Comments
 (0)