Skip to content

Commit 9186283

Browse files
committed
refactor(indexing): keep dense indexer attributes as deprecated properties
The compressed per-occupied-chunk cumsum introduced for zarr-developersgh-4174 changed the observable semantics of chunk_nitems_cumsum (and removed chunk_nitems on IntArrayDimIndexer). Although zarr.core is documented as private API, external code is known to introspect these indexers, so be conservative: store the compressed offsets under a new name (chunk_run_ends, aligned with chunk_rixs / dim_chunk_ixs) and restore chunk_nitems / chunk_nitems_cumsum as properties that lazily rebuild the original dense arrays, warning with ZarrDeprecationWarning. The O(nchunks) cost is now only paid if someone actually accesses them — which was the status quo before the fix. Assisted-by: ClaudeCode:claude-fable-5
1 parent aa6fd8a commit 9186283

3 files changed

Lines changed: 77 additions & 13 deletions

File tree

‎changes/4174.bugfix.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
Coordinate, mask, and orthogonal integer-array selections no longer allocate memory
22
proportional to the total number of chunks in the array. Previously a selection of a few
33
points on an array with billions of chunks could raise `MemoryError`; indexing cost now
4-
scales with the number of selected points instead.
4+
scales with the number of selected points instead. The internal indexers now store
5+
per-occupied-chunk run offsets (`chunk_run_ends`); the dense `chunk_nitems` /
6+
`chunk_nitems_cumsum` attributes of `CoordinateIndexer` and `IntArrayDimIndexer` are
7+
deprecated but keep their original semantics.

‎src/zarr/core/indexing.py‎

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import numpy as np
2222
import numpy.typing as npt
23+
from typing_extensions import deprecated
2324

2425
from zarr.core.chunk_grids import FixedDimension
2526
from zarr.core.common import ceildiv, product
@@ -30,6 +31,7 @@
3031
BoundsCheckError,
3132
NegativeStepError,
3233
VindexInvalidSelectionError,
34+
ZarrDeprecationWarning,
3335
)
3436

3537
if TYPE_CHECKING:
@@ -769,7 +771,7 @@ class IntArrayDimIndexer:
769771
dim_out_sel: npt.NDArray[np.intp]
770772
dim_chunk_ixs: npt.NDArray[np.intp]
771773
# end offset of each occupied chunk's run of selected items, aligned with dim_chunk_ixs
772-
chunk_nitems_cumsum: npt.NDArray[np.intp]
774+
chunk_run_ends: npt.NDArray[np.intp]
773775

774776
def __init__(
775777
self,
@@ -823,7 +825,7 @@ def __init__(
823825

824826
# the chunks to visit and, per occupied chunk, the end offset of its run of
825827
# selected items — O(nitems), never O(nchunks)
826-
dim_chunk_ixs, chunk_nitems_cumsum = sorted_run_ends(dim_sel_chunk_sorted)
828+
dim_chunk_ixs, chunk_run_ends = sorted_run_ends(dim_sel_chunk_sorted)
827829

828830
# store attributes
829831
object.__setattr__(self, "dim_len", dim_len)
@@ -834,7 +836,31 @@ def __init__(
834836
object.__setattr__(self, "dim_sel", dim_sel)
835837
object.__setattr__(self, "dim_out_sel", dim_out_sel)
836838
object.__setattr__(self, "dim_chunk_ixs", dim_chunk_ixs)
837-
object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum)
839+
object.__setattr__(self, "chunk_run_ends", chunk_run_ends)
840+
841+
@property
842+
@deprecated(
843+
"IntArrayDimIndexer.chunk_nitems is deprecated: it materializes a dense array with "
844+
"one entry per chunk along the dimension. Use dim_chunk_ixs and chunk_run_ends, "
845+
"which cover only the occupied chunks.",
846+
category=ZarrDeprecationWarning,
847+
)
848+
def chunk_nitems(self) -> npt.NDArray[np.intp]:
849+
dense = np.zeros(self.nchunks, dtype=np.intp)
850+
dense[self.dim_chunk_ixs] = np.diff(self.chunk_run_ends, prepend=0)
851+
return dense
852+
853+
@property
854+
@deprecated(
855+
"IntArrayDimIndexer.chunk_nitems_cumsum is deprecated: it materializes a dense array "
856+
"with one entry per chunk along the dimension. Use dim_chunk_ixs and chunk_run_ends, "
857+
"which cover only the occupied chunks.",
858+
category=ZarrDeprecationWarning,
859+
)
860+
def chunk_nitems_cumsum(self) -> npt.NDArray[np.intp]:
861+
dense = np.zeros(self.nchunks, dtype=np.intp)
862+
dense[self.dim_chunk_ixs] = np.diff(self.chunk_run_ends, prepend=0)
863+
return np.cumsum(dense)
838864

839865
def __iter__(self) -> Iterator[ChunkDimProjection]:
840866
g = self.dim_grid
@@ -845,8 +871,8 @@ def __iter__(self) -> Iterator[ChunkDimProjection]:
845871
if i == 0:
846872
start = 0
847873
else:
848-
start = self.chunk_nitems_cumsum[i - 1]
849-
stop = self.chunk_nitems_cumsum[i]
874+
start = self.chunk_run_ends[i - 1]
875+
stop = self.chunk_run_ends[i]
850876
if self.order == Order.INCREASING:
851877
dim_out_sel = slice(start, stop)
852878
else:
@@ -1184,8 +1210,9 @@ class CoordinateIndexer(Indexer):
11841210
sel_shape: tuple[int, ...]
11851211
selection: CoordinateSelectionNormalized
11861212
sel_sort: npt.NDArray[np.intp] | None
1213+
cdata_shape: tuple[int, ...]
11871214
# end offset of each occupied chunk's run of selected points, aligned with chunk_rixs
1188-
chunk_nitems_cumsum: npt.NDArray[np.intp]
1215+
chunk_run_ends: npt.NDArray[np.intp]
11891216
chunk_rixs: npt.NDArray[np.intp]
11901217
chunk_mixs: tuple[npt.NDArray[np.intp], ...]
11911218
shape: tuple[int, ...]
@@ -1259,12 +1286,13 @@ def __init__(
12591286
counts = np.diff(cuts, prepend=0, append=coords.size)
12601287
occupied = np.nonzero(counts)[0]
12611288
chunk_rixs = (first + occupied).astype(np.intp)
1262-
chunk_nitems_cumsum = np.cumsum(counts[occupied])
1289+
chunk_run_ends = np.cumsum(counts[occupied])
12631290

12641291
object.__setattr__(self, "sel_shape", coords.shape)
12651292
object.__setattr__(self, "selection", (coords,))
12661293
object.__setattr__(self, "sel_sort", None)
1267-
object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum)
1294+
object.__setattr__(self, "cdata_shape", cdata_shape)
1295+
object.__setattr__(self, "chunk_run_ends", chunk_run_ends)
12681296
object.__setattr__(self, "chunk_rixs", chunk_rixs)
12691297
object.__setattr__(self, "chunk_mixs", (chunk_rixs,))
12701298
object.__setattr__(self, "dim_grids", dim_grids)
@@ -1317,30 +1345,43 @@ def __init__(
13171345

13181346
# the chunks to visit and, per occupied chunk, the end offset of its run of
13191347
# selected points — O(npoints), never O(nchunks)
1320-
chunk_rixs, chunk_nitems_cumsum = sorted_run_ends(chunks_raveled_indices)
1348+
chunk_rixs, chunk_run_ends = sorted_run_ends(chunks_raveled_indices)
13211349

13221350
# unravel chunk indices
13231351
chunk_mixs = np.unravel_index(chunk_rixs, cdata_shape)
13241352

13251353
object.__setattr__(self, "sel_shape", sel_shape)
13261354
object.__setattr__(self, "selection", selection_broadcast)
13271355
object.__setattr__(self, "sel_sort", sel_sort)
1328-
object.__setattr__(self, "chunk_nitems_cumsum", chunk_nitems_cumsum)
1356+
object.__setattr__(self, "cdata_shape", cdata_shape)
1357+
object.__setattr__(self, "chunk_run_ends", chunk_run_ends)
13291358
object.__setattr__(self, "chunk_rixs", chunk_rixs)
13301359
object.__setattr__(self, "chunk_mixs", chunk_mixs)
13311360
object.__setattr__(self, "dim_grids", dim_grids)
13321361
object.__setattr__(self, "shape", shape)
13331362
object.__setattr__(self, "drop_axes", ())
13341363

1364+
@property
1365+
@deprecated(
1366+
"CoordinateIndexer.chunk_nitems_cumsum is deprecated: it materializes a dense array "
1367+
"with one entry per chunk in the array. Use chunk_rixs and chunk_run_ends, which "
1368+
"cover only the occupied chunks.",
1369+
category=ZarrDeprecationWarning,
1370+
)
1371+
def chunk_nitems_cumsum(self) -> npt.NDArray[np.intp]:
1372+
dense = np.zeros(product(self.cdata_shape), dtype=np.intp)
1373+
dense[self.chunk_rixs] = np.diff(self.chunk_run_ends, prepend=0)
1374+
return np.cumsum(dense)
1375+
13351376
def __iter__(self) -> Iterator[ChunkProjection]:
13361377
# iterate over chunks
13371378
for i in range(len(self.chunk_rixs)):
13381379
chunk_coords = tuple(m[i] for m in self.chunk_mixs)
13391380
if i == 0:
13401381
start = 0
13411382
else:
1342-
start = self.chunk_nitems_cumsum[i - 1]
1343-
stop = self.chunk_nitems_cumsum[i]
1383+
start = self.chunk_run_ends[i - 1]
1384+
stop = self.chunk_run_ends[i]
13441385
out_selection: slice | npt.NDArray[np.intp]
13451386
if self.sel_sort is None:
13461387
out_selection = slice(start, stop)

‎tests/test_indexing.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1236,6 +1236,26 @@ def unexpected_searchsorted(*args: Any, **kwargs: Any) -> None:
12361236
assert tuple(projection.chunk_coords for projection in projections) == ((0,), (99,))
12371237

12381238

1239+
def test_deprecated_dense_indexer_attributes() -> None:
1240+
"""`chunk_nitems` / `chunk_nitems_cumsum` keep their original dense per-chunk semantics
1241+
for external code that introspected indexers, but warn: they allocate O(nchunks) and the
1242+
indexers no longer use them internally (see gh-4174)."""
1243+
chunk_grid = ChunkGrid.from_sizes((20,), (3,)) # 7 chunks
1244+
coords = np.array([1, 4, 4, 19])
1245+
expected_nitems = np.bincount(coords // 3, minlength=7)
1246+
1247+
indexer = CoordinateIndexer((coords,), (20,), chunk_grid)
1248+
with pytest.warns(zarr.errors.ZarrDeprecationWarning):
1249+
assert_array_equal(indexer.chunk_nitems_cumsum, np.cumsum(expected_nitems))
1250+
1251+
(dim_grid,) = chunk_grid._dimensions
1252+
dim_indexer = IntArrayDimIndexer(coords, 20, dim_grid)
1253+
with pytest.warns(zarr.errors.ZarrDeprecationWarning):
1254+
assert_array_equal(dim_indexer.chunk_nitems, expected_nitems)
1255+
with pytest.warns(zarr.errors.ZarrDeprecationWarning):
1256+
assert_array_equal(dim_indexer.chunk_nitems_cumsum, np.cumsum(expected_nitems))
1257+
1258+
12391259
def test_sparse_selections_on_arrays_with_many_chunks(store: StorePath) -> None:
12401260
"""Coordinate and orthogonal selections must scale with the number of selected points,
12411261
not the total number of chunks in the array. This array has 2**44 chunks, so any dense

0 commit comments

Comments
 (0)