Skip to content

Commit cef4415

Browse files
committed
fix: gate bulk full-shard decode on identity reads; harden shard-index density check
The FusedCodecPipeline's vectorized whole-shard decode accepted any indexer without `sel_shape` whose output shape matched the shard shape. An OrthogonalIndexer from `arr[perm, :]` / `arr.oindex[...]` satisfies that, so reordering or duplicating fancy-index reads on uncompressed, crc-free sharded arrays silently returned the shard in natural order. The bulk path now requires an identity full read: one whole-dimension, step-1 slice per dimension (structural, not a BasicIndexer type check, since `arr[:]` arrives as an OrthogonalIndexer). Also: - decline structured dtypes in the bulk path, which lacks the Struct byte-order branch of BytesCodec._decode_sync (latent until zarr-developers#3054 is fixed); - `_ShardIndex.is_dense` now requires offsets to exactly tile the data section instead of merely being unique, so corrupt indexes with overlapping or out-of-range offsets (e.g. pointing into an index_location='start' index region) cannot be served as array data. Assisted-by: ClaudeCode:claude-fable-5
1 parent 43e4d9e commit cef4415

4 files changed

Lines changed: 358 additions & 63 deletions

File tree

‎changes/256.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed silent data corruption in the experimental `FusedCodecPipeline`: reordering or duplicating fancy-index reads (e.g. `arr[perm, :]`, `arr.oindex[[0, 0, 1], :]`) on uncompressed, crc-free sharded arrays could return the shard in natural order because the vectorized whole-shard decode accepted any selection whose output shape matched the shard shape. The bulk decode now fires only for identity full-shard reads, declines structured dtypes (whose byte-order handling it lacks), and requires shard-index offsets to exactly tile the data section, so corrupt indexes with overlapping or out-of-range offsets can no longer be served as array data.

‎src/zarr/codecs/sharding.py‎

Lines changed: 69 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,12 @@
5353
from zarr.core.config import config as zarr_config
5454
from zarr.core.dtype.common import HasEndianness
5555
from zarr.core.dtype.npy.int import UInt64
56+
from zarr.core.dtype.npy.structured import Struct
5657
from zarr.core.indexing import (
5758
BasicIndexer,
5859
ChunkProjection,
5960
SelectorTuple,
61+
SliceDimIndexer,
6062
_lexicographic_order,
6163
colexicographic_order_coords,
6264
get_indexer,
@@ -109,6 +111,32 @@ class ShardingCodecIndexLocation(metaclass=_DeprecatedStrEnumMeta):
109111
)
110112

111113

114+
def _is_identity_full_read(indexer: Any, shard_shape: tuple[int, ...]) -> bool:
115+
"""True when `indexer` selects every element of a `shard_shape` array in
116+
natural order: one whole-dimension, step-1 `SliceDimIndexer` per dimension.
117+
118+
Structural on purpose, not `isinstance(indexer, BasicIndexer)`: a full
119+
`arr[:]` read reaches the shard as an `OrthogonalIndexer`, so a type gate
120+
would silently disable the bulk fast path for the most common case. Any
121+
gather (integer-array / boolean / coordinate selection), subset, strided,
122+
or integer-scalar selection fails the per-dimension check — output shape
123+
alone is not enough, because a reordering or duplicating selection can have
124+
the same shape as the shard while requiring `chunk_selection` /
125+
`out_selection` to be honored.
126+
"""
127+
dim_indexers = getattr(indexer, "dim_indexers", None)
128+
if dim_indexers is None or len(dim_indexers) != len(shard_shape):
129+
return False
130+
return all(
131+
isinstance(dim_indexer, SliceDimIndexer)
132+
and dim_indexer.dim_len == dim_len
133+
and dim_indexer.start == 0
134+
and dim_indexer.stop == dim_len
135+
and dim_indexer.step == 1
136+
for dim_indexer, dim_len in zip(dim_indexers, shard_shape, strict=True)
137+
)
138+
139+
112140
def _parse_index_location(data: object) -> IndexLocation:
113141
if isinstance(data, str) and data in INDEX_LOCATION:
114142
return data # type: ignore[return-value]
@@ -192,12 +220,17 @@ def is_all_empty(self) -> bool:
192220
def get_full_chunk_map(self) -> npt.NDArray[np.bool_]:
193221
return np.not_equal(self.offsets_and_lengths[..., 0], MAX_UINT_64)
194222

195-
def is_dense(self, chunk_byte_length: int) -> bool:
196-
"""True when every chunk is present, fixed-length, and uniquely placed.
197-
198-
Used to gate the vectorized whole-shard decode: a dense fixed-size shard
199-
is a regular grid of equal-length payloads, so it can be reshaped/scattered
200-
in bulk rather than decoded chunk-by-chunk.
223+
def is_dense(self, chunk_byte_length: int, *, data_section_start: int) -> bool:
224+
"""True when the chunk payloads exactly tile the shard's data section.
225+
226+
Every chunk must be present with length `chunk_byte_length`, and the
227+
sorted offsets must be exactly `data_section_start + i * chunk_byte_length`
228+
for `i` in `0..n_chunks-1`: no gaps, no overlaps, and nothing outside the
229+
data section (a corrupt index could otherwise point chunks into the
230+
index region or out of the blob). Used to gate the vectorized
231+
whole-shard decode: a dense fixed-size shard is a regular grid of
232+
equal-length payloads, so it can be reshaped/scattered in bulk rather
233+
than decoded chunk-by-chunk.
201234
"""
202235
offsets = self.offsets_and_lengths[..., 0].reshape(-1)
203236
lengths = self.offsets_and_lengths[..., 1].reshape(-1)
@@ -207,8 +240,10 @@ def is_dense(self, chunk_byte_length: int) -> bool:
207240
# all the same fixed length
208241
if not bool(np.all(lengths == chunk_byte_length)):
209242
return False
210-
# offsets unique (no two chunks share a slot)
211-
return int(np.unique(offsets).size) == int(offsets.size)
243+
expected = np.uint64(data_section_start) + np.arange(
244+
offsets.size, dtype=np.uint64
245+
) * np.uint64(chunk_byte_length)
246+
return bool(np.array_equal(np.sort(offsets), expected))
212247

213248
def get_chunk_slice(self, chunk_coords: tuple[int, ...]) -> tuple[int, int] | None:
214249
localized_chunk = self._localize_chunk(chunk_coords)
@@ -1090,8 +1125,12 @@ def _decode_full_shard_bulk_if_uncompressed(
10901125
dtype/endian view with no reordering. A trailing crc32c is NOT accepted
10911126
(the bulk path can't verify per-chunk checksums, so crc shards keep the
10921127
per-chunk path's corruption detection);
1128+
- the data type is not structured (the byte-order handling below has no
1129+
`Struct` branch);
1130+
- `indexer` is an identity full-shard read (`_is_identity_full_read`);
10931131
- the stored index is dense (every chunk present, equal fixed length,
1094-
contiguous) so the data section is a regular grid of chunk payloads.
1132+
exactly tiling the data section) so the data section is a regular
1133+
grid of chunk payloads.
10951134
10961135
Chunk positions are read from the stored index, so this is correct for
10971136
any `subchunk_write_order` (morton / lexicographic / colexicographic /
@@ -1111,27 +1150,29 @@ def _decode_full_shard_bulk_if_uncompressed(
11111150
return None
11121151
ab_codec = self.codecs[0]
11131152

1153+
# The byte-order handling below lacks the structured-dtype branch of
1154+
# `BytesCodec._decode_sync` (which applies `newbyteorder` to multi-byte
1155+
# struct fields), so structured dtypes must take the per-chunk path.
1156+
if isinstance(shard_spec.dtype, Struct):
1157+
return None
1158+
11141159
chunks_per_shard = self._get_chunks_per_shard(shard_spec)
11151160
chunk_spec = self._get_chunk_spec(shard_spec)
11161161
n_chunks = product(chunks_per_shard)
11171162
if n_chunks == 0:
11181163
return None
11191164

1120-
# Only valid for a plain contiguous full-shard read, where each chunk
1121-
# lands at its natural grid position. The `sel_shape` check is
1122-
# load-bearing: a gather indexer (CoordinateIndexer, from vindex / an
1123-
# oindex with an integer-array selection) reorders points and exposes
1124-
# `sel_shape`, but its `.shape` is the FLATTENED point count, which can
1125-
# equal the shard shape by coincidence (trivially in 1-D). Gating on
1126-
# shape alone lets such a selection through, and the bulk path then
1127-
# returns the shard in natural order, silently dropping the reordering.
1128-
# A contiguous full read (BasicIndexer, or a non-gathering
1129-
# OrthogonalIndexer from `arr[:]`) has no `sel_shape` and is served here.
1130-
# Anything that gathers must fall through to the per-chunk path so
1165+
# Only valid for an identity full-shard read, where each chunk lands at
1166+
# its natural grid position. The per-dimension check is load-bearing:
1167+
# a gather selection (an `OrthogonalIndexer` with an integer-array or
1168+
# boolean dimension, from `arr[perm, :]` / `arr.oindex[...]`, or a
1169+
# `CoordinateIndexer` from vindex) can have an output `.shape` equal to
1170+
# the shard shape while reordering or duplicating points — serving it
1171+
# from the bulk path would return the shard in natural order, silently
1172+
# dropping the reordering. Anything that is not a full-slice-per-
1173+
# dimension read falls through to the per-chunk path so
11311174
# chunk_selection / out_selection are honored.
1132-
if getattr(indexer, "sel_shape", None) is not None:
1133-
return None
1134-
if tuple(indexer.shape) != tuple(shard_spec.shape):
1175+
if not _is_identity_full_read(indexer, shard_spec.shape):
11351176
return None
11361177
chunk_byte_length = self._inner_chunk_byte_length(chunk_spec)
11371178

@@ -1145,7 +1186,8 @@ def _decode_full_shard_bulk_if_uncompressed(
11451186
else:
11461187
index_bytes = shard_bytes[-shard_index_size:]
11471188
index = self._decode_shard_index_sync(index_bytes, chunks_per_shard)
1148-
if not index.is_dense(chunk_byte_length):
1189+
data_section_start = shard_index_size if self.index_location == "start" else 0
1190+
if not index.is_dense(chunk_byte_length, data_section_start=data_section_start):
11491191
return None
11501192

11511193
# --- bulk reconstruct ---
@@ -1231,9 +1273,9 @@ def _decode_partial_sync(
12311273
return None
12321274
bulk = self._decode_full_shard_bulk_if_uncompressed(shard_bytes, shard_spec, indexer)
12331275
if bulk is not None:
1234-
# The bulk path only fires for a contiguous full-shard read (it
1235-
# returns None for any gather indexer that exposes `sel_shape`),
1236-
# so the result is already shard-shaped — no reshape needed.
1276+
# The bulk path only fires for an identity full-shard read
1277+
# (`_is_identity_full_read`), so the result is already
1278+
# shard-shaped and in natural order — no reshape needed.
12371279
return bulk
12381280
shard_reader = self._shard_reader_from_bytes_sync(shard_bytes, chunks_per_shard)
12391281
shard_dict: ShardMapping = shard_reader

0 commit comments

Comments
 (0)