Skip to content

Commit edba2ee

Browse files
authored
fix(sharding): reject variable-size codecs in index_codecs (zarr-developers#4397)
* fix(sharding): reject variable-size codecs in index_codecs The spec forbids variable-size codecs (e.g. compressors) in a sharding codec's `index_codecs`. ShardingCodec accepted them, and the first write then failed with a bare NotImplementedError from compute_encoded_size. Check `is_fixed_size` on every index codec at construction time, and mark the numcodecs checksum wrappers as fixed-size so they stay valid there. Closes zarr-developers#2030 Assisted-by: ClaudeCode:claude-opus-5-5 * docs: add changelog fragment for zarr-developers#4397 Assisted-by: ClaudeCode:claude-opus-5-5
1 parent aa2ba28 commit edba2ee

7 files changed

Lines changed: 64 additions & 4 deletions

File tree

‎changes/4397.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Sharding `index_codecs` containing a variable-size codec, such as a compressor, are now rejected when the `ShardingCodec` is built, with a clear error. Before, writing failed later with an unexplained `NotImplementedError`.

‎src/zarr/abc/codec.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,9 @@ class BaseCodec[CI: CodecInput, CO: CodecOutput](Metadata):
109109

110110
# Whether this codec's encoded output is a fixed size given a fixed input
111111
# size. Defaults to False (the conservative answer): a codec that does not
112-
# explicitly opt in is treated as variable-size, which only disables
113-
# size-dependent fast paths (e.g. the sharding bulk-decode), never
114-
# correctness. Codecs with genuinely fixed-size output (BytesCodec,
112+
# explicitly opt in is treated as variable-size, which disables
113+
# size-dependent fast paths (e.g. the sharding bulk-decode) and makes it
114+
# ineligible as a sharding index codec. Codecs with genuinely fixed-size output (BytesCodec,
115115
# TransposeCodec, ...) override this with True. The default also keeps
116116
# third-party / variable-length codecs (VLenUTF8, numcodecs wrappers) that
117117
# never set the attribute from raising AttributeError where it is read.

‎src/zarr/codecs/numcodecs/_codecs.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,8 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> AsType:
303303

304304
# bytes-to-bytes checksum codecs
305305
class _NumcodecsChecksumCodec(_NumcodecsBytesBytesCodec):
306+
is_fixed_size = True
307+
306308
def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> int:
307309
return input_byte_length + 4 # pragma: no cover
308310

‎src/zarr/codecs/sharding.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,21 @@ def _drops_only_unit_axes(shape: tuple[int, ...], full: tuple[int, ...]) -> bool
422422
return all(full_size == 1 for full_size in remaining)
423423

424424

425+
def _check_index_codecs_fixed_size(index_codecs: tuple[Codec, ...]) -> None:
426+
"""Reject index codecs whose encoded size is not fixed.
427+
428+
The spec forbids variable-size codecs (such as compressors) in `index_codecs`:
429+
the shard index is located by its byte size, which must be known before the
430+
index is read.
431+
"""
432+
variable_size = [codec for codec in index_codecs if not codec.is_fixed_size]
433+
if variable_size:
434+
raise ValueError(
435+
"Sharding `index_codecs` must produce a fixed-size encoding, but these codecs "
436+
f"do not: {variable_size}. Compression codecs cannot be used for the shard index."
437+
)
438+
439+
425440
@dataclass(frozen=True)
426441
class ShardingCodec(
427442
ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin
@@ -453,6 +468,7 @@ def __init__(
453468
chunk_shape_parsed = parse_shapelike(chunk_shape)
454469
codecs_parsed = parse_codecs(codecs)
455470
index_codecs_parsed = parse_codecs(index_codecs)
471+
_check_index_codecs_fixed_size(index_codecs_parsed)
456472
index_location_coerced = _coerce_enum_input(
457473
index_location, "index_location", "ShardingCodec"
458474
)

‎tests/test_codecs/test_sharding.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@
1818
BloscCodec,
1919
BytesCodec,
2020
Crc32cCodec,
21+
GzipCodec,
2122
ShardingCodec,
2223
TransposeCodec,
2324
)
25+
from zarr.codecs.numcodecs import CRC32
2426
from zarr.codecs.sharding import (
2527
INDEX_LOCATION,
2628
MAX_UINT_64,
@@ -1119,6 +1121,44 @@ def test_sharding_codec_rejects_unknown_index_location() -> None:
11191121
ShardingCodec(**kwargs)
11201122

11211123

1124+
@pytest.mark.parametrize(
1125+
"index_codecs",
1126+
[
1127+
(BytesCodec(),),
1128+
(BytesCodec(), Crc32cCodec()),
1129+
(TransposeCodec(order=(1, 0)), BytesCodec(), Crc32cCodec()),
1130+
(BytesCodec(), CRC32()),
1131+
],
1132+
)
1133+
def test_sharding_fixed_size_index_codecs_roundtrip(index_codecs: tuple[Any, ...]) -> None:
1134+
"""
1135+
Any chain of fixed-size codecs is accepted as `index_codecs`, and data
1136+
written through it reads back unchanged.
1137+
"""
1138+
data = np.arange(16, dtype="uint16")
1139+
arr = zarr.create_array(
1140+
MemoryStore(),
1141+
shape=data.shape,
1142+
dtype=data.dtype,
1143+
chunks=(2,),
1144+
shards=(8,),
1145+
compressors=None,
1146+
serializer=ShardingCodec(chunk_shape=(2,), index_codecs=index_codecs),
1147+
)
1148+
arr[:] = data
1149+
np.testing.assert_array_equal(arr[:], data)
1150+
1151+
1152+
@pytest.mark.parametrize("compressor", [GzipCodec(), BloscCodec()])
1153+
def test_sharding_codec_rejects_variable_size_index_codecs(compressor: Any) -> None:
1154+
"""
1155+
ShardingCodec rejects an `index_codecs` chain containing a codec whose
1156+
encoded size is not fixed, as the spec requires.
1157+
"""
1158+
with pytest.raises(ValueError, match="must produce a fixed-size encoding"):
1159+
ShardingCodec(chunk_shape=(2,), index_codecs=(BytesCodec(), compressor))
1160+
1161+
11221162
def test_sharding_index_location_attribute_error_for_unknown_member() -> None:
11231163
"""
11241164
Attribute access for a name that is not a known member of the deprecated

‎tests/test_codecs/test_sharding_unit.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,7 @@ class _AsyncOnlyBytesCodec(ArrayBytesCodec):
732732
`_decode_shard_index`/`_encode_shard_index`.
733733
"""
734734

735+
is_fixed_size = True
735736
_inner = BytesCodec()
736737

737738
def to_dict(self) -> dict[str, Any]:

‎uv.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)