Skip to content

Commit 8aef319

Browse files
d-v-bclaude
andcommitted
refactor(metadata): drop numpy and tuple widening from chunk grid metadata
The chunk grid metadata classes take typed Python values again: a regular chunk shape of `int`s, rectilinear dimensions of `int` or `list` in stored documents. `declares_chunk_edges` and the numpy and tuple handling in `expand_rle`, `_validate_chunk_shapes` and `RectilinearChunkGridMetadata.from_dict` are removed with their tests; numpy input belongs to the `chunks=` normalizer. The regular grid still rejects edge lists (zarr-developersgh-4374), now naming the offending type instead of echoing the value. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d17a848 commit 8aef319

4 files changed

Lines changed: 26 additions & 132 deletions

File tree

‎src/zarr/core/common.py‎

Lines changed: 8 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
Literal,
1313
NotRequired,
1414
TypedDict,
15-
TypeGuard,
1615
cast,
1716
overload,
1817
)
@@ -277,21 +276,7 @@ def _default_zarr_format() -> ZarrFormat:
277276
return cast("ZarrFormat", int(zarr_config.get("default_zarr_format", 3)))
278277

279278

280-
def declares_chunk_edges(value: object) -> TypeGuard[Iterable[Any]]:
281-
"""Whether a chunk specification element declares explicit chunk edge lengths.
282-
283-
True for any non-integer iterable, which is the form explicit edge lengths
284-
take, and False for a single integer size. The test is structural rather
285-
than an exact type check: edge lengths reach us as a list parsed from JSON,
286-
as a tuple from a metadata dict built in Python, or as a numpy array, and
287-
all three mean the same thing.
288-
"""
289-
if isinstance(value, int | np.integer):
290-
return False
291-
return isinstance(value, Iterable) and not isinstance(value, str | bytes)
292-
293-
294-
def expand_rle(data: Iterable[int | Sequence[int]]) -> list[int]:
279+
def expand_rle(data: Sequence[int | list[int]]) -> list[int]:
295280
"""Expand a mixed array of bare integers and RLE pairs.
296281
297282
Per the rectilinear chunk grid spec, each element can be:
@@ -300,21 +285,18 @@ def expand_rle(data: Iterable[int | Sequence[int]]) -> list[int]:
300285
"""
301286
result: list[int] = []
302287
for item in data:
303-
if declares_chunk_edges(item):
304-
pair = tuple(item)
305-
if len(pair) != 2:
306-
raise ValueError(f"RLE entries must be an integer or [size, count], got {item}")
307-
size, count = int(pair[0]), int(pair[1])
288+
if isinstance(item, (int, float)) and not isinstance(item, bool):
289+
val = int(item)
290+
if val < 1:
291+
raise ValueError(f"Chunk edge length must be >= 1, got {val}")
292+
result.append(val)
293+
elif isinstance(item, list) and len(item) == 2:
294+
size, count = int(item[0]), int(item[1])
308295
if size < 1:
309296
raise ValueError(f"Chunk edge length must be >= 1, got {size}")
310297
if count < 1:
311298
raise ValueError(f"RLE repeat count must be >= 1, got {count}")
312299
result.extend([size] * count)
313-
elif isinstance(item, int | np.integer | float) and not isinstance(item, bool):
314-
val = int(item)
315-
if val < 1:
316-
raise ValueError(f"Chunk edge length must be >= 1, got {val}")
317-
result.append(val)
318300
else:
319301
raise ValueError(f"RLE entries must be an integer or [size, count], got {item}")
320302
return result

‎src/zarr/core/metadata/v3.py‎

Lines changed: 14 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from dataclasses import dataclass, field, replace
66
from typing import TYPE_CHECKING, Any, Final, Literal, NotRequired, TypeGuard, cast
77

8-
import numpy as np
98
from typing_extensions import TypedDict
109

1110
from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec
@@ -26,7 +25,6 @@
2625
NamedConfig,
2726
NamedRequiredConfig,
2827
compress_rle,
29-
declares_chunk_edges,
3028
expand_rle,
3129
parse_named_configuration,
3230
parse_shapelike,
@@ -229,18 +227,15 @@ def _parse_chunk_shape(chunk_shape: Iterable[int]) -> tuple[int, ...]:
229227
let a rectilinear chunk shape be stored as a regular grid (gh-4374).
230228
"""
231229
parsed: list[int] = []
230+
# Typed as ints, but a stored document can hold anything here.
232231
for dim_idx, dim_spec in enumerate(cast("Iterable[object]", chunk_shape)):
233-
if not isinstance(dim_spec, int | np.integer):
232+
if not isinstance(dim_spec, int):
234233
raise TypeError(
235234
f"Dimension {dim_idx}: a regular chunk grid requires an integer chunk "
236-
f"edge length, got {dim_spec!r}. Lists of chunk edge lengths belong "
237-
"to a rectilinear chunk grid."
235+
f"edge length, got a {type(dim_spec).__name__}. Lists of chunk edge "
236+
"lengths belong to a rectilinear chunk grid."
238237
)
239-
parsed.append(
240-
parse_chunk_edge(
241-
int(dim_spec) if isinstance(dim_spec, np.integer) else dim_spec, dim_idx
242-
)
243-
)
238+
parsed.append(parse_chunk_edge(dim_spec, dim_idx))
244239
return tuple(parsed)
245240

246241

@@ -254,10 +249,10 @@ def _validate_chunk_shapes(
254249
"""
255250
result: list[int | tuple[int, ...]] = []
256251
for dim_idx, dim_spec in enumerate(chunk_shapes):
257-
if isinstance(dim_spec, int | np.integer):
258-
result.append(parse_chunk_edge(int(dim_spec), dim_idx))
252+
if isinstance(dim_spec, int):
253+
result.append(parse_chunk_edge(dim_spec, dim_idx))
259254
else:
260-
edges = tuple(int(edge) for edge in dim_spec)
255+
edges = tuple(dim_spec)
261256
if not edges:
262257
raise ValueError(f"Dimension {dim_idx} has no chunk edges.")
263258
bad = [i for i, e in enumerate(edges) if isinstance(e, bool) or e < 1]
@@ -298,8 +293,7 @@ def to_dict(self) -> RegularChunkGridMetadataJSON: # type: ignore[override]
298293
def from_dict(cls, data: RegularChunkGridMetadataJSON) -> Self: # type: ignore[override]
299294
parse_named_configuration(data, "regular") # validate name
300295
configuration = data["configuration"]
301-
# `__post_init__` parses, so this only has to hand over the dimensions.
302-
return cls(chunk_shape=tuple(configuration["chunk_shape"]))
296+
return cls(chunk_shape=_parse_chunk_shape(configuration["chunk_shape"]))
303297

304298

305299
@dataclass(frozen=True, kw_only=True)
@@ -380,17 +374,14 @@ def from_dict(cls, data: RectilinearChunkGridMetadataJSON) -> Self: # type: ign
380374
raw_shapes = configuration["chunk_shapes"]
381375
parsed: list[int | tuple[int, ...]] = []
382376
for dim_spec in raw_shapes:
383-
if declares_chunk_edges(dim_spec):
377+
if isinstance(dim_spec, int):
378+
# `__post_init__` range-checks it, naming the dimension.
379+
parsed.append(dim_spec)
380+
elif isinstance(dim_spec, list):
384381
parsed.append(tuple(expand_rle(dim_spec)))
385-
elif isinstance(dim_spec, int | np.integer):
386-
# Range checks belong to `_validate_chunk_shapes`, which
387-
# `__post_init__` runs over the result and which names the
388-
# offending dimension.
389-
parsed.append(int(dim_spec))
390382
else:
391383
raise TypeError(
392-
"Invalid chunk_shapes entry: expected an integer or a sequence of "
393-
f"chunk edge lengths, got {type(dim_spec)}"
384+
f"Invalid chunk_shapes entry: expected int or list, got {type(dim_spec)}"
394385
)
395386
return cls(chunk_shapes=tuple(parsed))
396387

‎tests/test_metadata/test_v3.py‎

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import json
66
from typing import TYPE_CHECKING
77

8-
import numpy as np
98
import pytest
109

1110
from tests.conftest import Expect, ExpectFail
@@ -20,7 +19,6 @@
2019
ARRAY_METADATA_KEYS,
2120
ArrayMetadataJSON_V3,
2221
ArrayV3Metadata,
23-
RectilinearChunkGridMetadata,
2422
RegularChunkGridMetadata,
2523
create_chunk_grid_metadata,
2624
parse_codecs,
@@ -32,7 +30,6 @@
3230
MetadataValidationError,
3331
NodeTypeValidationError,
3432
UnknownCodecError,
35-
ZarrUserWarning,
3633
)
3734

3835
if TYPE_CHECKING:
@@ -159,44 +156,6 @@ def test_create_chunk_grid_metadata_unknown_dimension_type() -> None:
159156
create_chunk_grid_metadata(grid)
160157

161158

162-
@pytest.mark.parametrize(
163-
"case",
164-
[
165-
Expect(input=(2, 3), output=(2, 3), id="python_ints"),
166-
Expect(input=(np.int64(2), np.uint8(3)), output=(2, 3), id="numpy_ints"),
167-
],
168-
ids=lambda case: case.id,
169-
)
170-
def test_regular_chunk_grid_normalizes_ints(case: Expect[tuple[Any, ...], tuple[int, ...]]) -> None:
171-
"""A regular chunk grid accepts Python and numpy integers and stores Python ints."""
172-
grid = RegularChunkGridMetadata(chunk_shape=case.input)
173-
assert grid.chunk_shape == case.output
174-
assert all(type(c) is int for c in grid.chunk_shape)
175-
176-
177-
@pytest.mark.parametrize(
178-
"case",
179-
[
180-
Expect(input=(2, (5, 10, 5)), output=(2, (5, 10, 5)), id="python_values"),
181-
Expect(
182-
input=(np.int64(2), np.array([5, 10, 5])), output=(2, (5, 10, 5)), id="numpy_values"
183-
),
184-
Expect(input=(2, [5, 10, 5]), output=(2, (5, 10, 5)), id="list_edges"),
185-
],
186-
ids=lambda case: case.id,
187-
)
188-
def test_rectilinear_chunk_grid_normalizes_ints(
189-
case: Expect[tuple[Any, ...], tuple[int | tuple[int, ...], ...]],
190-
) -> None:
191-
"""A rectilinear chunk grid accepts any sequence of Python or numpy integers
192-
as a dimension's edges and stores Python ints in tuples."""
193-
with config.set({"array.rectilinear_chunks": True}):
194-
grid = RectilinearChunkGridMetadata(chunk_shapes=case.input)
195-
assert grid.chunk_shapes == case.output
196-
flat = [e for dim in grid.chunk_shapes for e in (dim if isinstance(dim, tuple) else (dim,))]
197-
assert all(type(e) is int for e in flat)
198-
199-
200159
@pytest.mark.parametrize(
201160
"build",
202161
[
@@ -505,6 +464,7 @@ def test_group_metadata_to_dict(attributes: dict[str, Any] | None) -> None:
505464
def test_group_metadata_to_dict_consolidated(attributes: dict[str, Any] | None) -> None:
506465
"""GroupMetadata.to_dict includes consolidated_metadata when present."""
507466
from zarr import consolidate_metadata, create_group
467+
from zarr.errors import ZarrUserWarning
508468

509469
store: dict[str, object] = {}
510470
group = create_group(store, attributes=attributes, zarr_format=3)

‎tests/test_unified_chunk_grid.py‎

Lines changed: 3 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -512,8 +512,6 @@ def test_chunk_grid_iter() -> None:
512512
[
513513
([[10, 3]], [10, 10, 10]),
514514
([[10, 2], [20, 1]], [10, 10, 20]),
515-
pytest.param([(10, 3)], [10, 10, 10], id="tuple-pair"),
516-
pytest.param([np.int64(10), np.array([20, 2])], [10, 20, 20], id="numpy-values"),
517515
],
518516
)
519517
def test_rle_expand(compressed: list[Any], expected: list[int]) -> None:
@@ -551,8 +549,6 @@ def test_rle_roundtrip() -> None:
551549
([[-10, 2]], "Chunk edge length must be >= 1"),
552550
([[5, 0]], "RLE repeat count must be >= 1"),
553551
([[5, -1]], "RLE repeat count must be >= 1"),
554-
([(5, 2, 1)], "RLE entries must be an integer or"),
555-
(["5"], "RLE entries must be an integer or"),
556552
],
557553
ids=[
558554
"zero-edge",
@@ -561,8 +557,6 @@ def test_rle_roundtrip() -> None:
561557
"negative-rle-size",
562558
"zero-rle-count",
563559
"negative-rle-count",
564-
"rle-pair-wrong-length",
565-
"string-entry",
566560
],
567561
)
568562
def test_rle_expand_rejects_invalid(rle_input: list[Any], match: str) -> None:
@@ -3023,54 +3017,21 @@ def test_iter_chunk_regions_rectilinear() -> None:
30233017
},
30243018
(4, (10, 20)),
30253019
),
3026-
# A metadata dict built in Python may hold tuples and numpy values
3027-
# where parsed JSON holds lists and ints.
3028-
pytest.param(
3029-
{
3030-
"name": "rectilinear",
3031-
"configuration": {"kind": "inline", "chunk_shapes": (4, (10, 20))},
3032-
},
3033-
(4, (10, 20)),
3034-
id="tuple-dims",
3035-
),
3036-
pytest.param(
3037-
{
3038-
"name": "rectilinear",
3039-
"configuration": {"kind": "inline", "chunk_shapes": [((4, 3),), [10, 20]]},
3040-
},
3041-
((4, 4, 4), (10, 20)),
3042-
id="tuple-rle-pair",
3043-
),
3044-
pytest.param(
3045-
{
3046-
"name": "rectilinear",
3047-
"configuration": {
3048-
"kind": "inline",
3049-
"chunk_shapes": [np.int64(4), np.array([10, 20]), [np.array([5, 2])]],
3050-
},
3051-
},
3052-
(4, (10, 20), (5, 5)),
3053-
id="numpy-values",
3054-
),
30553020
],
30563021
)
30573022
def test_rectilinear_from_dict(
30583023
json_input: RectilinearChunkGridMetadataJSON,
30593024
expected_chunk_shapes: tuple[int | tuple[int, ...], ...],
30603025
) -> None:
3061-
"""RectilinearChunkGridMetadata.from_dict correctly parses all spec forms,
3062-
whatever sequence type holds a dimension's edges, and stores plain ints."""
3026+
"""RectilinearChunkGridMetadata.from_dict correctly parses all spec forms."""
30633027
grid = RectilinearChunkGridMetadata.from_dict(json_input)
30643028
assert grid.chunk_shapes == expected_chunk_shapes
3065-
flat = [e for dim in grid.chunk_shapes for e in (dim if isinstance(dim, tuple) else (dim,))]
3066-
assert all(type(e) is int for e in flat)
30673029

30683030

30693031
@pytest.mark.parametrize("dim_spec", [4.5, None, "10"], ids=["float", "none", "string"])
30703032
def test_rectilinear_from_dict_rejects_invalid_dim_spec(dim_spec: Any) -> None:
3071-
"""A dimension that is neither an integer nor a sequence of edges is rejected.
3072-
A string is iterable but is not a sequence of edges."""
3073-
with pytest.raises(TypeError, match="expected an integer or a sequence of chunk edge lengths"):
3033+
"""A dimension that is neither an integer nor a list of edges is rejected."""
3034+
with pytest.raises(TypeError, match="expected int or list"):
30743035
RectilinearChunkGridMetadata.from_dict(
30753036
{"name": "rectilinear", "configuration": {"kind": "inline", "chunk_shapes": [dim_spec]}}
30763037
)

0 commit comments

Comments
 (0)