Skip to content

Commit 08d335d

Browse files
d-v-bclaude
andcommitted
fix(zarr-metadata): close five gaps in the composition rules
Found by adversarial review; each has a regression test. - scale_offset registers the identity spec transition the spec describes ("MUST be performed using the arithmetic semantics of the input array's data type"; astype was removed in v3 in favour of cast_value). Without it a no-op codec stopped propagation and stood down every later rule. The test exemption that recorded this is gone. - chain_initial_spec keeps the rank when it cannot keep the extents: an ArraySpec extent may now be None individually, so a rectilinear grid or a zero extent no longer hides a rank-mismatched transpose or shard. - Entity rules declare the configuration members they read, and run_entity_rules stands down only the rules that read an unusable member rather than the whole entity. This also makes the invariant the configuration["member"] accesses rely on explicit and checked at registration, where it was previously true only by inspection. - A rule reporting at the entity itself keeps that location instead of being re-based under a "configuration" node a bare-string entity does not have. - The endianness problem names the data type, so the shard-index case reads as uint64 rather than appearing to contradict the document. Assisted-by: ClaudeCode:claude-opus-5 Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent f52c154 commit 08d335d

16 files changed

Lines changed: 318 additions & 55 deletions

File tree

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
Five gaps in the composition rules, found by adversarial review of the
2+
rules layer:
3+
4+
- A modelled `array -> array` codec with no registered spec transition
5+
stopped spec propagation, standing down every rule after it. The
6+
`scale_offset` codec had none, so inserting a no-op `scale_offset`
7+
switched off the `bytes` codec's endianness requirement and every
8+
shard's geometry check. It now registers the identity transition the
9+
spec describes, and no codec is exempt from having one.
10+
- A chunk grid whose extents this package cannot read (a rectilinear
11+
grid, or a regular grid with a non-positive extent) discarded the
12+
*rank* along with the extents, so a rank-mismatched `transpose` order
13+
or inner chunk shape went unreported. Every chunk of an array has the
14+
array's rank whatever the grid, so the rank is kept and only the
15+
geometry checks stand down.
16+
- One unusable configuration member suppressed every other rule about
17+
the same entity — a misspelled `index_location` hid the problems in a
18+
shard's inner pipelines. Rules now declare the members they read and
19+
stand down individually, so the rest of the entity is still judged.
20+
- Problems about an entity as a whole pointed at a `configuration` node
21+
that a bare-string entity does not have (`("codecs", 0,
22+
"configuration")` for `"codecs": ["bytes"]`). They now point at the
23+
entity.
24+
- The endianness message named no data type, so inside a shard's
25+
`index_codecs` it appeared to be about the array's own type rather
26+
than the shard index's `uint64`. It names the type.

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/bytes_codec.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
_ARRAY_V3 = "zarr_v3_array"
2020

2121

22-
@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME)
22+
@entity_rule(_ARRAY_V3, CODECS, BYTES_CODEC_NAME, reads=frozenset({"endian"}))
2323
def data_type_has_a_raw_byte_representation(
2424
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
2525
) -> tuple[ValidationProblem, ...]:
@@ -29,8 +29,8 @@ def data_type_has_a_raw_byte_representation(
2929
if shape_verdict is not None and len(blocking_problems(shape_verdict)) != 0:
3030
return ()
3131
found = storage_class(incoming.data_type)
32+
name = data_type_name(incoming.data_type)
3233
if found == "variable_length":
33-
name = data_type_name(incoming.data_type)
3434
return (
3535
ValidationProblem(
3636
(),
@@ -39,10 +39,12 @@ def data_type_has_a_raw_byte_representation(
3939
),
4040
)
4141
if found == "multi_byte" and "endian" not in configuration:
42+
# Name the type: inside a shard's `index_codecs` the array is the
43+
# shard index, whose uint64 type appears nowhere in the document.
4244
return (
4345
ValidationProblem(
4446
("endian",),
45-
"endian is required for a data type containing multi-byte values",
47+
f"endian is required for data type {name!r}, which contains multi-byte values",
4648
"missing_key",
4749
),
4850
)

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/cast_value.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON
2929

3030

31-
@entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME)
31+
@entity_rule("zarr_v3_array", CODECS, CAST_VALUE_CODEC_NAME, reads=frozenset({"data_type"}))
3232
def target_data_type_obeys_its_rules(
3333
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
3434
) -> tuple[ValidationProblem, ...]:

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/gzip.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
_ARRAY_V3 = "zarr_v3_array"
1818

1919

20-
@entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME)
20+
@entity_rule(_ARRAY_V3, CODECS, GZIP_CODEC_NAME, reads=frozenset({"level"}))
2121
def level_is_in_range(
2222
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
2323
) -> tuple[ValidationProblem, ...]:

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/numpy_time.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
_ARRAY_V3 = "zarr_v3_array"
1919
_MAX_SCALE_FACTOR = 2**31 - 1
20+
_SCALE_FACTOR = frozenset({"scale_factor"})
2021

2122

2223
def _scale_factor_is_in_range(
@@ -34,5 +35,9 @@ def _scale_factor_is_in_range(
3435
)
3536

3637

37-
entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE_NAME)(_scale_factor_is_in_range)
38-
entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE_NAME)(_scale_factor_is_in_range)
38+
entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_DATETIME64_DATA_TYPE_NAME, reads=_SCALE_FACTOR)(
39+
_scale_factor_is_in_range
40+
)
41+
entity_rule(_ARRAY_V3, DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE_NAME, reads=_SCALE_FACTOR)(
42+
_scale_factor_is_in_range
43+
)

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/rectilinear_grid.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def _expanded_extent(spec: Sequence[object]) -> int | None:
4444
return total
4545

4646

47-
@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME)
47+
@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shapes"}))
4848
def chunk_extents_are_positive(
4949
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
5050
) -> tuple[ValidationProblem, ...]:
@@ -85,7 +85,13 @@ def chunk_extents_are_positive(
8585
return tuple(problems)
8686

8787

88-
@entity_rule(_ARRAY_V3, CHUNK_GRID, RECTILINEAR_CHUNK_GRID_NAME, requires=frozenset({"shape"}))
88+
@entity_rule(
89+
_ARRAY_V3,
90+
CHUNK_GRID,
91+
RECTILINEAR_CHUNK_GRID_NAME,
92+
requires=frozenset({"shape"}),
93+
reads=frozenset({"chunk_shapes"}),
94+
)
8995
def tiles_the_array(
9096
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
9197
) -> tuple[ValidationProblem, ...]:

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/regular_grid.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
_ARRAY_V3 = "zarr_v3_array"
1818

1919

20-
@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME)
20+
@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, reads=frozenset({"chunk_shape"}))
2121
def chunk_extents_are_positive(
2222
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
2323
) -> tuple[ValidationProblem, ...]:
@@ -39,7 +39,13 @@ def chunk_extents_are_positive(
3939
)
4040

4141

42-
@entity_rule(_ARRAY_V3, CHUNK_GRID, REGULAR_CHUNK_GRID_NAME, requires=frozenset({"shape"}))
42+
@entity_rule(
43+
_ARRAY_V3,
44+
CHUNK_GRID,
45+
REGULAR_CHUNK_GRID_NAME,
46+
requires=frozenset({"shape"}),
47+
reads=frozenset({"chunk_shape"}),
48+
)
4349
def chunks_every_dimension(
4450
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
4551
) -> tuple[ValidationProblem, ...]:
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""Spec transition for the `scale_offset` codec.
2+
3+
`scale_offset` subtracts an offset and multiplies by a scale, element by
4+
element, and the spec requires the result to be representable in the
5+
array's own data type: "The encoding and decoding transformations MUST be
6+
performed using the arithmetic semantics of the input array's data type.
7+
If any intermediate or final value produced during encoding or decoding
8+
is not representable in that data type, implementations MUST treat this
9+
as an error."
10+
11+
So it changes neither the shape nor the data type, and its transition is
12+
the identity. The spec is explicit that narrowing is somebody else's job:
13+
the codec's `astype` field "was removed from the `scale_offset` codec in
14+
favor of expressing data type conversion via a dedicated codec", because
15+
"in Zarr V3, a `dtype` field is not needed — the data type of the input
16+
to an array-array codec is determined by its location in the `codecs`
17+
metadata".
18+
19+
Registering this matters beyond tidiness. A modelled `array -> array`
20+
codec with no transition is treated as unknown, which stops propagation
21+
and silently stands down every rule downstream of it — so without this,
22+
inserting a no-op `scale_offset` would switch off the `bytes` codec's
23+
endianness requirement and every shard's geometry check.
24+
25+
https://github.com/zarr-developers/zarr-extensions/blob/main/codecs/scale_offset/README.md
26+
"""
27+
28+
from __future__ import annotations
29+
30+
from typing import TYPE_CHECKING
31+
32+
from zarr_metadata.rules._spec import spec_transition
33+
from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME
34+
35+
if TYPE_CHECKING:
36+
from collections.abc import Mapping
37+
38+
from zarr_metadata.rules._spec import ArraySpec
39+
40+
41+
@spec_transition(SCALE_OFFSET_CODEC_NAME)
42+
def preserves_the_array(configuration: Mapping[str, object], incoming: ArraySpec) -> ArraySpec:
43+
"""Element-wise arithmetic in the input type: same shape, same type."""
44+
return incoming

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/sharding.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,15 @@
3737
from collections.abc import Mapping
3838

3939
_ARRAY_V3 = "zarr_v3_array"
40+
_CHUNK_SHAPE = frozenset({"chunk_shape"})
41+
_PIPELINES = frozenset({"chunk_shape", "codecs", "index_codecs"})
42+
_INDEX_CODECS = frozenset({"index_codecs"})
4043
_VARIABLE_SIZE_CODECS = frozenset(
4144
{BLOSC_CODEC_NAME, GZIP_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME, ZSTD_CODEC_NAME}
4245
)
4346

4447

45-
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME)
48+
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE)
4649
def inner_chunk_extents_are_positive(
4750
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
4851
) -> tuple[ValidationProblem, ...]:
@@ -58,15 +61,16 @@ def inner_chunk_extents_are_positive(
5861
)
5962

6063

61-
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME)
64+
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_CHUNK_SHAPE)
6265
def inner_chunks_tile_the_incoming_array(
6366
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
6467
) -> tuple[ValidationProblem, ...]:
6568
"""The inner chunk must rank-match and evenly divide the array it receives.
6669
67-
Declines when the incoming shape is unknown — an unclassified codec
68-
upstream, or a non-regular grid at the top level — rather than
69-
guessing from the document.
70+
Declines when the incoming array is unknown entirely — an unclassified
71+
codec upstream — rather than guessing. A known rank with unknown
72+
extents (a chunk grid this package cannot read) still supports the
73+
rank check; only the divisibility check needs the extents.
7074
"""
7175
if incoming.shape is None:
7276
return ()
@@ -89,11 +93,11 @@ def inner_chunks_tile_the_incoming_array(
8993
"invalid_value",
9094
)
9195
for position, (outer_extent, inner_extent) in enumerate(zip(outer, inner, strict=True))
92-
if inner_extent >= 1 and outer_extent % inner_extent != 0
96+
if outer_extent is not None and inner_extent >= 1 and outer_extent % inner_extent != 0
9397
)
9498

9599

96-
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME)
100+
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_PIPELINES)
97101
def inner_pipelines_are_pipelines(
98102
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
99103
) -> tuple[ValidationProblem, ...]:
@@ -133,7 +137,7 @@ def inner_pipelines_are_pipelines(
133137
return tuple(problems)
134138

135139

136-
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME)
140+
@entity_rule(_ARRAY_V3, CODECS, SHARDING_INDEXED_CODEC_NAME, reads=_INDEX_CODECS)
137141
def index_codecs_have_fixed_encoded_size(
138142
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
139143
) -> tuple[ValidationProblem, ...]:

‎packages/zarr-metadata/src/zarr_metadata/rules/_entities/struct_dtype.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
_ARRAY_V3 = "zarr_v3_array"
3131

3232

33-
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME)
33+
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"}))
3434
def field_data_types_obey_their_rules(
3535
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
3636
) -> tuple[ValidationProblem, ...]:
@@ -69,7 +69,7 @@ def _field_names(configuration: Mapping[str, object]) -> tuple[tuple[int, str],
6969
return tuple(named)
7070

7171

72-
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME)
72+
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"}))
7373
def fields_are_non_empty(
7474
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
7575
) -> tuple[ValidationProblem, ...]:
@@ -79,7 +79,7 @@ def fields_are_non_empty(
7979
return (ValidationProblem(("fields",), "expected at least one struct field", "invalid_value"),)
8080

8181

82-
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME)
82+
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"}))
8383
def field_data_types_are_fixed_size(
8484
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
8585
) -> tuple[ValidationProblem, ...]:
@@ -100,7 +100,7 @@ def field_data_types_are_fixed_size(
100100
return tuple(problems)
101101

102102

103-
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME)
103+
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"}))
104104
def field_names_are_non_empty(
105105
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
106106
) -> tuple[ValidationProblem, ...]:
@@ -114,7 +114,7 @@ def field_names_are_non_empty(
114114
)
115115

116116

117-
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME)
117+
@entity_rule(_ARRAY_V3, DATA_TYPE, STRUCT_DATA_TYPE_NAME, reads=frozenset({"fields"}))
118118
def field_names_are_unique(
119119
configuration: Mapping[str, object], document: Mapping[str, object], incoming: ArraySpec
120120
) -> tuple[ValidationProblem, ...]:

0 commit comments

Comments
 (0)