Skip to content

Commit d3dc9f5

Browse files
authored
fix: accept numpy integers as chunk sizes (zarr-developers#4257)
* fix: accept numpy integers as chunk sizes `normalize_chunks_nd` dispatches the scalar convenience form on `numbers.Integral`, but `normalize_chunks_1d` narrowed on `int`. Numpy integer scalars satisfy the former and not the latter, so a per-dimension numpy integer passed the outer dispatch and then fell into the branch meant for explicit per-dimension chunk sequences, where `list(chunks)` raised `TypeError: 'numpy.int64' object is not iterable`. Numpy integers arise naturally whenever a chunk shape is computed rather than written as a literal, since numpy reductions and elementwise ops yield numpy scalars. Narrow on `numbers.Integral` and coerce with `int()`, matching the caller and the sequence branch, which already accepted `Integral` elements. Move the `-1` sentinel check inside that branch. It previously ran on the raw input, so a numpy array chunk specification made `chunks == -1` return an array and raise an ambiguous-truth-value error; rectilinear specs given as numpy arrays now work. A chunk specification that is neither an integer nor iterable now names the offending value and its type instead of surfacing an opaque "object is not iterable" from `list(chunks)`. Fixes zarr-developers#4255 Assisted-by: ClaudeCode:claude-opus-5 * Rename 4255.bugfix.md to 4257.bugfix.md
1 parent 6ba6e89 commit d3dc9f5

4 files changed

Lines changed: 44 additions & 10 deletions

File tree

‎changes/4257.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Numpy integers are accepted as chunk sizes again. Since 3.3.0 a per-dimension chunk size that was a numpy integer (e.g. `chunks=(np.int64(2), np.int64(2))`, as produced by any computed chunk shape) raised `TypeError: 'numpy.int64' object is not iterable`, because the scalar chunk path narrowed on `int` while its caller dispatched on `numbers.Integral`. Numpy arrays are now also accepted as chunk specifications, and a chunk specification that is neither an integer nor iterable now reports the offending value instead of failing with an opaque iteration error.

‎src/zarr/core/chunk_grids.py‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -729,17 +729,26 @@ def normalize_chunks_1d(
729729
overhang the span. The actual data extent of each chunk is determined
730730
by the chunk grid at runtime, not by this function.
731731
"""
732-
if chunks == -1:
733-
return np.array([span], dtype=np.int64)
734-
if isinstance(chunks, int):
735-
if chunks <= 0:
736-
raise ValueError(f"Chunk size must be positive, got {chunks}")
732+
# `numbers.Integral` rather than `int` so that numpy integer scalars (which are not
733+
# `int` subclasses) take the uniform-chunk path instead of being treated as a sequence.
734+
if isinstance(chunks, numbers.Integral):
735+
chunk_size = int(chunks)
736+
if chunk_size == -1:
737+
return np.array([span], dtype=np.int64)
738+
if chunk_size <= 0:
739+
raise ValueError(f"Chunk size must be positive, got {chunk_size}")
737740
if span == 0:
738-
return np.array([chunks], dtype=np.int64)
739-
n = ceildiv(span, chunks)
740-
return np.full(n, chunks, dtype=np.int64)
741+
return np.array([chunk_size], dtype=np.int64)
742+
n = ceildiv(span, chunk_size)
743+
return np.full(n, chunk_size, dtype=np.int64)
741744
else:
742-
chunk_list = list(chunks)
745+
try:
746+
chunk_list = list(chunks) # type: ignore[arg-type]
747+
except TypeError:
748+
raise TypeError(
749+
f"Chunk specification must be an integer or an iterable of integers; got "
750+
f"{chunks!r} of type {type(chunks).__name__}."
751+
) from None
743752
if not chunk_list:
744753
raise ValueError("Chunk specification must not be empty")
745754
non_int = [

‎tests/test_api.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def test_create(memory_store: Store) -> None:
7979
z = create(shape=(400.5, 100), store=store, overwrite=True) # type: ignore[arg-type]
8080

8181
# create array with float chunk shape
82-
with pytest.raises(TypeError, match="'float' object is not iterable"):
82+
with pytest.raises(TypeError, match="Chunk specification must be an integer or an iterable"):
8383
z = create(shape=(400, 100), chunks=(16, 16.5), store=store, overwrite=True) # type: ignore[arg-type]
8484

8585

‎tests/test_chunk_grids.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,15 @@ def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None:
6868
(10, (0,), ((10,),)),
6969
((5, 10), (0, 100), ((5,), (10,) * 10)),
7070
((5, 10), (20, 0), ((5, 5, 5, 5), (10,))),
71+
# numpy integers are accepted anywhere a python int is, whether as the scalar
72+
# convenience form, as per-dimension entries, or as the `-1` sentinel.
73+
(np.int64(10), (100,), ((10,) * 10,)),
74+
((np.int64(2), np.int64(2)), (4, 4), ((2, 2), (2, 2))),
75+
((1, 3, np.int64(16), np.int64(16)), (1, 3, 32, 32), ((1,), (3,), (16, 16), (16, 16))),
76+
((np.int32(30), np.int64(-1)), (100, 20), ((30, 30, 30, 30), (20,))),
77+
(np.array([10, 10]), (100, 100), ((10,) * 10, (10,) * 10)),
78+
# rectilinear chunks given as numpy arrays
79+
((np.array([60, 40]), np.array([50, 50])), (100, 100), ((60, 40), (50, 50))),
7180
],
7281
)
7382
def test_normalize_chunks(
@@ -142,7 +151,22 @@ def test_chunk_layout_nested() -> None:
142151
id="negative-uniform",
143152
msg="Chunk size must be positive",
144153
),
154+
ExpectFail(
155+
input=(np.int64(0), 100),
156+
exception=ValueError,
157+
id="zero-uniform-numpy",
158+
msg="Chunk size must be positive",
159+
),
145160
ExpectFail(input=([], 100), exception=ValueError, id="empty-list", msg="must not be empty"),
161+
# Scalars that are neither integers nor iterable name themselves in the error,
162+
# rather than surfacing an opaque "object is not iterable" from `list(chunks)`.
163+
ExpectFail(
164+
input=(2.5, 100),
165+
exception=TypeError,
166+
id="non-iterable-scalar",
167+
msg="must be an integer or an iterable of integers; got 2.5 of type float",
168+
escape=True,
169+
),
146170
ExpectFail(
147171
input=([10, -1, 10], 100),
148172
exception=ValueError,

0 commit comments

Comments
 (0)