Skip to content

Commit 9e9ee4e

Browse files
committed
fix(chunk-grids): make array creation O(1) in per-dimension chunk count
Chunk normalization now returns a ChunkGrid whose uniform dimensions are stored as FixedDimension (size + extent) instead of one array entry per chunk, so create_array(shape=(2**62,), chunks=(1,)) succeeds instantly instead of raising "array is too big", and chunks=(1, 1) on a (2**31, 2**31) array no longer allocates ~17 GB per dimension. Explicit per-chunk lists collapse to FixedDimension when they describe a regular grid; genuinely irregular lists become VaryingDimension. Both variants bind chunk sizes to their extent, so the two forms carry the same invariants. The intermediate ChunksTuple type and as_regular_shape helper are removed; create_chunk_grid_metadata consumes the grid directly and serializes uniform dimensions of mixed rectilinear grids as the spec's bare-int step-size shorthand. Creation-time counterpart of the zarr-developersgh-4174 indexing fix. Assisted-by: ClaudeCode:claude-fable-5
1 parent 7729d92 commit 9e9ee4e

7 files changed

Lines changed: 213 additions & 149 deletions

File tree

‎changes/4174.bugfix.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Array creation is now O(1) in the number of chunks per dimension. Chunk
2+
normalization returns a `ChunkGrid` whose uniform dimensions are stored as a
3+
size + extent pair (`FixedDimension`) instead of being expanded to one entry
4+
per chunk, so creating arrays like
5+
`zarr.create_array(store, shape=(2**62,), chunks=(1,), dtype='int32')` succeeds
6+
instantly instead of raising `ValueError` or allocating gigabytes of memory.
7+
The intermediate `ChunksTuple` representation was removed in the process.
8+
This is the creation-time counterpart of the indexing fix in #4172.

‎src/zarr/core/array.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
SHARDED_INNER_CHUNK_MAX_BYTES,
4444
ChunkGrid,
4545
_is_rectilinear_chunks,
46-
as_regular_shape,
4746
guess_chunks,
4847
normalize_chunks_nd,
4948
resolve_outer_and_inner_chunks,
@@ -523,7 +522,7 @@ async def _create(
523522
outer_chunks = guess_chunks(shape, item_size)
524523
else:
525524
outer_chunks = normalize_chunks_nd(_raw, shape)
526-
_chunks = as_regular_shape(outer_chunks)
525+
_chunks = outer_chunks.chunk_shape
527526

528527
if order is None:
529528
order_parsed = config_parsed.order
@@ -4472,7 +4471,7 @@ async def init_array(
44724471
"chunks=(inner_size, ...), shards=[[shard_sizes], ...]"
44734472
)
44744473

4475-
# Normalize the user's chunks into canonical ChunksTuple form
4474+
# Normalize the user's chunks into a canonical ChunkGrid
44764475

44774476
if chunks == "auto":
44784477
max_bytes = None if shards is None else SHARDED_INNER_CHUNK_MAX_BYTES
@@ -4515,7 +4514,7 @@ async def init_array(
45154514
meta = AsyncArray._create_metadata_v2(
45164515
shape=shape_parsed,
45174516
dtype=zdtype,
4518-
chunks=as_regular_shape(outer_chunks),
4517+
chunks=outer_chunks.chunk_shape,
45194518
dimension_separator=chunk_key_encoding_parsed.separator,
45204519
fill_value=fill_value,
45214520
order=order_parsed,
@@ -4534,7 +4533,7 @@ async def init_array(
45344533
grid = create_chunk_grid_metadata(outer_chunks)
45354534
codecs_out: tuple[Codec, ...]
45364535
if inner is not None:
4537-
inner_chunks_flat = as_regular_shape(inner.outer_chunks)
4536+
inner_chunks_flat = inner.outer_chunks.chunk_shape
45384537
index_location: IndexLocation = "end"
45394538
if isinstance(shards, dict):
45404539
index_location = cast("IndexLocation", shards.get("index_location", "end"))

‎src/zarr/core/chunk_grids.py‎

Lines changed: 59 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
TYPE_CHECKING,
1313
Any,
1414
NamedTuple,
15-
NewType,
1615
Protocol,
1716
TypeGuard,
1817
cast,
@@ -43,31 +42,6 @@
4342
is not `None`. Explicit chunk sizes are not affected by this value.
4443
"""
4544

46-
ChunksTuple = NewType("ChunksTuple", tuple[np.ndarray[tuple[int], np.dtype[np.int64]], ...])
47-
"""Normalized chunk specification: one 1D int64 array of chunk sizes per dimension.
48-
49-
Produced exclusively by `normalize_chunks_nd` and `guess_chunks`.
50-
Consumers should use this type to ensure they receive validated,
51-
canonical chunk specifications rather than raw user input.
52-
"""
53-
54-
55-
class ChunkLayout(NamedTuple):
56-
"""Result of resolving user `chunks`/`shards` into grid metadata inputs.
57-
58-
outer_chunks
59-
Chunk sizes for the chunk grid metadata. When sharding is active
60-
these are the shard sizes; otherwise they are the user's chunk sizes.
61-
inner
62-
Recursive sub-structure inside each chunk. `None` means the chunk is
63-
opaque (no sharding). When present, `inner.outer_chunks` gives the
64-
sub-chunk sizes passed to `ShardingCodec`, and `inner.inner` gives
65-
the next level of nesting (for nested sharding), or `None`.
66-
"""
67-
68-
outer_chunks: ChunksTuple
69-
inner: ChunkLayout | None = None
70-
7145

7246
@dataclass(frozen=True)
7347
class FixedDimension:
@@ -389,12 +363,6 @@ def is_regular_nd(
389363
return all(is_regular_1d(d) for d in chunks)
390364

391365

392-
def as_regular_shape(chunks: ChunksTuple) -> tuple[int, ...]:
393-
"""Flatten a regular ChunksTuple to one int per dimension."""
394-
assert is_regular_nd(chunks), f"expected regular chunks, got {chunks}"
395-
return tuple(int(dim[0]) for dim in chunks)
396-
397-
398366
@dataclass(frozen=True)
399367
class ChunkGrid:
400368
"""
@@ -489,6 +457,11 @@ def from_sizes(
489457

490458
# -- Properties --
491459

460+
@property
461+
def dimensions(self) -> tuple[DimensionGrid, ...]:
462+
"""The per-dimension grids (`FixedDimension` or `VaryingDimension`)."""
463+
return self._dimensions
464+
492465
@property
493466
def ndim(self) -> int:
494467
return len(self._dimensions)
@@ -641,6 +614,23 @@ def update_shape(self, new_shape: tuple[int, ...]) -> ChunkGrid:
641614
return ChunkGrid(dimensions=dims)
642615

643616

617+
class ChunkLayout(NamedTuple):
618+
"""Result of resolving user `chunks`/`shards` into grid metadata inputs.
619+
620+
outer_chunks
621+
Chunk grid for the chunk grid metadata. When sharding is active
622+
this holds the shard sizes; otherwise it holds the user's chunk sizes.
623+
inner
624+
Recursive sub-structure inside each chunk. `None` means the chunk is
625+
opaque (no sharding). When present, `inner.outer_chunks` gives the
626+
sub-chunk sizes passed to `ShardingCodec`, and `inner.inner` gives
627+
the next level of nesting (for nested sharding), or `None`.
628+
"""
629+
630+
outer_chunks: ChunkGrid
631+
inner: ChunkLayout | None = None
632+
633+
644634
def _guess_regular_chunks(
645635
shape: tuple[int, ...] | int,
646636
typesize: int,
@@ -717,30 +707,31 @@ def _guess_regular_chunks(
717707
return tuple(int(x) for x in chunks)
718708

719709

720-
def normalize_chunks_1d(
721-
chunks: int | Iterable[object], span: int
722-
) -> np.ndarray[tuple[int], np.dtype[np.int64]]:
710+
def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionGrid:
723711
"""
724-
Normalize a one-dimensional chunk specification into a 1D int64 array of
725-
chunk sizes that cover the span.
712+
Normalize a one-dimensional chunk specification into a dimension grid:
713+
`FixedDimension` for uniform chunk sizes, `VaryingDimension` for explicit
714+
per-chunk sizes that genuinely vary. Both variants bind the chunk sizes to
715+
the span, and the uniform form is O(1) in the number of chunks — a
716+
dimension with `2**62` chunks must not materialize one entry per chunk.
726717
727718
`-1` means "one chunk covering the entire span."
728-
For an integer chunk size, all chunks are uniform — the last chunk may
729-
overhang the span. The actual data extent of each chunk is determined
730-
by the chunk grid at runtime, not by this function.
719+
Explicit chunk size lists must sum to the span exactly; lists that describe
720+
a regular grid (all sizes equal, or equal with a smaller boundary chunk)
721+
collapse to `FixedDimension`. For uniform sizes the last chunk may overhang
722+
the span.
731723
"""
732724
# `numbers.Integral` rather than `int` so that numpy integer scalars (which are not
733725
# `int` subclasses) take the uniform-chunk path instead of being treated as a sequence.
726+
# The `-1` sentinel check lives inside this branch so that numpy-array chunk
727+
# specifications never hit an ambiguous-truth-value error on `chunks == -1`.
734728
if isinstance(chunks, numbers.Integral):
735729
chunk_size = int(chunks)
736730
if chunk_size == -1:
737-
return np.array([span], dtype=np.int64)
731+
return FixedDimension(size=span, extent=span)
738732
if chunk_size <= 0:
739733
raise ValueError(f"Chunk size must be positive, got {chunk_size}")
740-
if span == 0:
741-
return np.array([chunk_size], dtype=np.int64)
742-
n = ceildiv(span, chunk_size)
743-
return np.full(n, chunk_size, dtype=np.int64)
734+
return FixedDimension(size=chunk_size, extent=span)
744735
else:
745736
try:
746737
chunk_list = list(chunks) # type: ignore[arg-type]
@@ -766,23 +757,30 @@ def normalize_chunks_1d(
766757
raise ValueError(f"All chunk sizes must be positive, got {ints}")
767758
if sum(ints) != span:
768759
raise ValueError(f"Chunk sizes {ints} do not sum to span {span}")
769-
return np.asarray(ints, dtype=np.int64)
760+
if is_regular_1d(ints):
761+
return FixedDimension(size=ints[0], extent=span)
762+
return VaryingDimension(ints, extent=span)
770763

771764

772765
def normalize_chunks_nd(
773766
chunks: Any,
774767
shape: tuple[int, ...],
775-
) -> ChunksTuple:
768+
) -> ChunkGrid:
776769
"""
777-
Normalize a chunk specification into a `ChunksTuple`.
770+
Normalize a chunk specification into a `ChunkGrid`.
778771
779772
This is a mechanical transformation — no heuristics, no guessing.
780773
Handles `False` ("all data in one chunk"), scalar ints, `-1` sentinels (one chunk
781774
per dimension covering the full span), and explicit per-dimension lists
782775
of chunk sizes (regular or rectilinear).
783776
777+
This is the strict parser for user-supplied chunk specifications; use
778+
`ChunkGrid.from_sizes` / `ChunkGrid.from_metadata` for stored metadata,
779+
which is validated under more tolerant rules (e.g. trailing edges beyond
780+
the array extent).
781+
784782
For auto-chunking, use `guess_chunks` which returns a
785-
`ChunksTuple` directly. `chunks=None` and `chunks=True` are rejected
783+
`ChunkGrid` directly. `chunks=None` and `chunks=True` are rejected
786784
here — the caller is responsible for choosing between explicit sizes
787785
and auto-chunking.
788786
"""
@@ -793,7 +791,9 @@ def normalize_chunks_nd(
793791

794792
# handle no chunking
795793
if chunks is False:
796-
return ChunksTuple(tuple(np.array([s], dtype=np.int64) for s in shape))
794+
return ChunkGrid(
795+
dimensions=tuple(FixedDimension(size=int(s), extent=int(s)) for s in shape)
796+
)
797797

798798
# handle 1D convenience form. bool is excluded above so this only catches actual ints.
799799
if isinstance(chunks, numbers.Integral):
@@ -805,20 +805,20 @@ def normalize_chunks_nd(
805805
f"chunks has {len(chunks)} dimensions but shape has {len(shape)} dimensions"
806806
)
807807

808-
return ChunksTuple(
809-
tuple(normalize_chunks_1d(c, span=s) for c, s in zip(chunks, shape, strict=True))
808+
return ChunkGrid(
809+
dimensions=tuple(normalize_chunks_1d(c, span=s) for c, s in zip(chunks, shape, strict=True))
810810
)
811811

812812

813813
def guess_chunks(
814814
shape: tuple[int, ...], typesize: int, *, max_bytes: int | None = None
815-
) -> ChunksTuple:
815+
) -> ChunkGrid:
816816
"""
817817
Heuristically determine chunk sizes for an array.
818818
819819
This is the policy function — it makes opinionated choices about
820820
chunk sizes based on array shape and element size, and returns a
821-
normalized `ChunksTuple`.
821+
normalized `ChunkGrid`.
822822
823823
Parameters
824824
----------
@@ -877,7 +877,7 @@ def _guess_num_chunks_per_axis_shard(
877877
def resolve_outer_and_inner_chunks(
878878
*,
879879
array_shape: tuple[int, ...],
880-
chunks: ChunksTuple,
880+
chunks: ChunkGrid,
881881
shard_shape: ShardsLike | None,
882882
item_size: int,
883883
) -> ChunkLayout:
@@ -888,7 +888,7 @@ def resolve_outer_and_inner_chunks(
888888
array_shape
889889
The array shape.
890890
chunks
891-
Normalized chunk specification (the user's `chunks=`).
891+
Normalized chunk grid (the user's `chunks=`).
892892
shard_shape
893893
Raw shard specification (the user's `shards=`).
894894
`None` means no sharding, `"auto"` triggers heuristic inference,
@@ -900,7 +900,7 @@ def resolve_outer_and_inner_chunks(
900900
Returns
901901
-------
902902
ChunkLayout
903-
`outer_chunks` is the `ChunksTuple` for chunk grid
903+
`outer_chunks` is the `ChunkGrid` for chunk grid
904904
metadata. `inner` holds the sub-chunk structure for
905905
`ShardingCodec`, or is `None` when sharding is not active.
906906
"""
@@ -912,8 +912,8 @@ def resolve_outer_and_inner_chunks(
912912
outer = normalize_chunks_nd(shard_shape, array_shape)
913913
return ChunkLayout(outer_chunks=outer, inner=ChunkLayout(outer_chunks=chunks))
914914

915-
# Extract the flat chunk shape (first size per dimension) for arithmetic.
916-
chunk_shape_flat = as_regular_shape(chunks)
915+
# Extract the flat chunk shape (uniform size per dimension) for arithmetic.
916+
chunk_shape_flat = chunks.chunk_shape
917917

918918
if shard_shape == "auto":
919919
warnings.warn(

‎src/zarr/core/metadata/v3.py‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from zarr.core._json import json_to_buffer
1313
from zarr.core.array_spec import ArrayConfig, ArraySpec
1414
from zarr.core.buffer.core import default_buffer_prototype
15-
from zarr.core.chunk_grids import is_regular_nd
15+
from zarr.core.chunk_grids import FixedDimension, VaryingDimension
1616
from zarr.core.chunk_key_encodings import (
1717
ChunkKeyEncoding,
1818
ChunkKeyEncodingLike,
@@ -43,7 +43,7 @@
4343
from typing import Self
4444

4545
from zarr.core.buffer import Buffer, BufferPrototype
46-
from zarr.core.chunk_grids import ChunksTuple
46+
from zarr.core.chunk_grids import ChunkGrid
4747
from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar
4848

4949

@@ -373,32 +373,36 @@ def from_dict(cls, data: RectilinearChunkGridMetadataJSON) -> Self: # type: ign
373373

374374

375375
def create_chunk_grid_metadata(
376-
chunks: ChunksTuple,
376+
chunks: ChunkGrid,
377377
) -> ChunkGridMetadata:
378-
"""Construct a chunk grid metadata object from a normalized `ChunksTuple`.
378+
"""Construct a chunk grid metadata object from a normalized `ChunkGrid`.
379379
380-
Regular chunks produce a `RegularChunkGridMetadata`.
381-
Rectilinear chunks produce a `RectilinearChunkGridMetadata`.
380+
Regular grids produce a `RegularChunkGridMetadata`.
381+
Rectilinear grids produce a `RectilinearChunkGridMetadata`.
382382
383383
Parameters
384384
----------
385-
chunks : ChunksTuple
386-
Normalized chunk specification, as returned by
385+
chunks : ChunkGrid
386+
Normalized chunk grid, as returned by
387387
`normalize_chunks_nd` or `guess_chunks`.
388388
389389
See Also
390390
--------
391391
parse_chunk_grid : Deserialize a chunk grid from stored JSON metadata.
392392
"""
393-
if is_regular_nd(chunks):
394-
# If we know the chunks specification is regular, then we can take the first
395-
# chunk size for each dimension as the chunk shape.
396-
chunk_shape = tuple(int(dim_chunks[0]) for dim_chunks in chunks)
397-
return RegularChunkGridMetadata(chunk_shape=chunk_shape)
398-
else:
399-
return RectilinearChunkGridMetadata(
400-
chunk_shapes=tuple(tuple(int(x) for x in d) for d in chunks)
401-
)
393+
if chunks.is_regular:
394+
return RegularChunkGridMetadata(chunk_shape=chunks.chunk_shape)
395+
# Uniform dimensions stay bare ints — the rectilinear grid spec treats
396+
# a bare int as a step size repeating to cover the axis.
397+
chunk_shapes: list[int | tuple[int, ...]] = []
398+
for dim in chunks.dimensions:
399+
if isinstance(dim, FixedDimension):
400+
chunk_shapes.append(dim.size)
401+
elif isinstance(dim, VaryingDimension):
402+
chunk_shapes.append(dim.edges)
403+
else:
404+
raise TypeError(f"Unknown dimension grid type: {type(dim)}")
405+
return RectilinearChunkGridMetadata(chunk_shapes=tuple(chunk_shapes))
402406

403407

404408
def parse_chunk_grid(

‎tests/conftest.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
)
2626
from zarr.core.chunk_grids import (
2727
SHARDED_INNER_CHUNK_MAX_BYTES,
28-
as_regular_shape,
2928
guess_chunks,
3029
normalize_chunks_nd,
3130
resolve_outer_and_inner_chunks,
@@ -393,7 +392,7 @@ def create_array_metadata(
393392
return ArrayV2Metadata(
394393
shape=shape_parsed,
395394
dtype=dtype_parsed,
396-
chunks=as_regular_shape(outer_chunks),
395+
chunks=outer_chunks.chunk_shape,
397396
order=order_parsed,
398397
dimension_separator=chunk_key_encoding_parsed.separator,
399398
fill_value=fill_value,
@@ -412,7 +411,7 @@ def create_array_metadata(
412411
sub_codecs: tuple[Codec, ...] = (*array_array, array_bytes, *bytes_bytes)
413412
codecs_out: tuple[Codec, ...]
414413
if inner is not None:
415-
inner_chunks_flat = as_regular_shape(inner.outer_chunks)
414+
inner_chunks_flat = inner.outer_chunks.chunk_shape
416415
index_location: IndexLocation = "end"
417416
if isinstance(shards, dict):
418417
index_location = cast("IndexLocation", shards.get("index_location", "end"))

0 commit comments

Comments
 (0)