Skip to content

Commit bf81831

Browse files
authored
fix: gate bulk full-shard decode on identity reads; harden shard-index density check (#4203)
The FusedCodecPipeline's vectorized whole-shard decode accepted any indexer without `sel_shape` whose output shape matched the shard shape. An OrthogonalIndexer from `arr[perm, :]` / `arr.oindex[...]` satisfies that, so reordering or duplicating fancy-index reads on uncompressed, crc-free sharded arrays silently returned the shard in natural order. The bulk path now requires an identity full read: one whole-dimension, step-1 slice per dimension (structural, not a BasicIndexer type check, since `arr[:]` arrives as an OrthogonalIndexer). Also: - decline structured dtypes in the bulk path, which lacks the Struct byte-order branch of BytesCodec._decode_sync (latent until #3054 is fixed); - `_ShardIndex.is_dense` now requires offsets to exactly tile the data section instead of merely being unique, so corrupt indexes with overlapping or out-of-range offsets (e.g. pointing into an index_location='start' index region) cannot be served as array data. Assisted-by: ClaudeCode:claude-fable-5
1 parent ba83236 commit bf81831

4 files changed

Lines changed: 358 additions & 63 deletions

File tree

‎changes/4203.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed silent data corruption in the experimental `FusedCodecPipeline`: reordering or duplicating fancy-index reads (e.g. `arr[perm, :]`, `arr.oindex[[0, 0, 1], :]`) on uncompressed, crc-free sharded arrays could return the shard in natural order because the vectorized whole-shard decode accepted any selection whose output shape matched the shard shape. The bulk decode now fires only for identity full-shard reads, declines structured dtypes (whose byte-order handling it lacks), and requires shard-index offsets to exactly tile the data section, so corrupt indexes with overlapping or out-of-range offsets can no longer be served as array data.

‎src/zarr/codecs/sharding.py‎

Lines changed: 69 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,12 @@
5353
from zarr.core.config import config as zarr_config
5454
from zarr.core.dtype.common import HasEndianness
5555
from zarr.core.dtype.npy.int import UInt64
56+
from zarr.core.dtype.npy.structured import Struct
5657
from zarr.core.indexing import (
5758
BasicIndexer,
5859
ChunkProjection,
5960
SelectorTuple,
61+
SliceDimIndexer,
6062
_lexicographic_order,
6163
colexicographic_order_coords,
6264
get_indexer,
@@ -109,6 +111,32 @@ class ShardingCodecIndexLocation(metaclass=_DeprecatedStrEnumMeta):
109111
)
110112

111113

114+
def _is_identity_full_read(indexer: Any, shard_shape: tuple[int, ...]) -> bool:
115+
"""True when `indexer` selects every element of a `shard_shape` array in
116+
natural order: one whole-dimension, step-1 `SliceDimIndexer` per dimension.
117+
118+
Structural on purpose, not `isinstance(indexer, BasicIndexer)`: a full
119+
`arr[:]` read reaches the shard as an `OrthogonalIndexer`, so a type gate
120+
would silently disable the bulk fast path for the most common case. Any
121+
gather (integer-array / boolean / coordinate selection), subset, strided,
122+
or integer-scalar selection fails the per-dimension check — output shape
123+
alone is not enough, because a reordering or duplicating selection can have
124+
the same shape as the shard while requiring `chunk_selection` /
125+
`out_selection` to be honored.
126+
"""
127+
dim_indexers = getattr(indexer, "dim_indexers", None)
128+
if dim_indexers is None or len(dim_indexers) != len(shard_shape):
129+
return False
130+
return all(
131+
isinstance(dim_indexer, SliceDimIndexer)
132+
and dim_indexer.dim_len == dim_len
133+
and dim_indexer.start == 0
134+
and dim_indexer.stop == dim_len
135+
and dim_indexer.step == 1
136+
for dim_indexer, dim_len in zip(dim_indexers, shard_shape, strict=True)
137+
)
138+
139+
112140
def _parse_index_location(data: object) -> IndexLocation:
113141
if isinstance(data, str) and data in INDEX_LOCATION:
114142
return data # type: ignore[return-value]
@@ -192,12 +220,17 @@ def is_all_empty(self) -> bool:
192220
def get_full_chunk_map(self) -> npt.NDArray[np.bool_]:
193221
return np.not_equal(self.offsets_and_lengths[..., 0], MAX_UINT_64)
194222

195-
def is_dense(self, chunk_byte_length: int) -> bool:
196-
"""True when every chunk is present, fixed-length, and uniquely placed.
197-
198-
Used to gate the vectorized whole-shard decode: a dense fixed-size shard
199-
is a regular grid of equal-length payloads, so it can be reshaped/scattered
200-
in bulk rather than decoded chunk-by-chunk.
223+
def is_dense(self, chunk_byte_length: int, *, data_section_start: int) -> bool:
224+
"""True when the chunk payloads exactly tile the shard's data section.
225+
226+
Every chunk must be present with length `chunk_byte_length`, and the
227+
sorted offsets must be exactly `data_section_start + i * chunk_byte_length`
228+
for `i` in `0..n_chunks-1`: no gaps, no overlaps, and nothing outside the
229+
data section (a corrupt index could otherwise point chunks into the
230+
index region or out of the blob). Used to gate the vectorized
231+
whole-shard decode: a dense fixed-size shard is a regular grid of
232+
equal-length payloads, so it can be reshaped/scattered in bulk rather
233+
than decoded chunk-by-chunk.
201234
"""
202235
offsets = self.offsets_and_lengths[..., 0].reshape(-1)
203236
lengths = self.offsets_and_lengths[..., 1].reshape(-1)
@@ -207,8 +240,10 @@ def is_dense(self, chunk_byte_length: int) -> bool:
207240
# all the same fixed length
208241
if not bool(np.all(lengths == chunk_byte_length)):
209242
return False
210-
# offsets unique (no two chunks share a slot)
211-
return int(np.unique(offsets).size) == int(offsets.size)
243+
expected = np.uint64(data_section_start) + np.arange(
244+
offsets.size, dtype=np.uint64
245+
) * np.uint64(chunk_byte_length)
246+
return bool(np.array_equal(np.sort(offsets), expected))
212247

213248
def get_chunk_slice(self, chunk_coords: tuple[int, ...]) -> tuple[int, int] | None:
214249
localized_chunk = self._localize_chunk(chunk_coords)
@@ -1086,8 +1121,12 @@ def _decode_full_shard_bulk_if_uncompressed(
10861121
dtype/endian view with no reordering. A trailing crc32c is NOT accepted
10871122
(the bulk path can't verify per-chunk checksums, so crc shards keep the
10881123
per-chunk path's corruption detection);
1124+
- the data type is not structured (the byte-order handling below has no
1125+
`Struct` branch);
1126+
- `indexer` is an identity full-shard read (`_is_identity_full_read`);
10891127
- the stored index is dense (every chunk present, equal fixed length,
1090-
contiguous) so the data section is a regular grid of chunk payloads.
1128+
exactly tiling the data section) so the data section is a regular
1129+
grid of chunk payloads.
10911130
10921131
Chunk positions are read from the stored index, so this is correct for
10931132
any `subchunk_write_order` (morton / lexicographic / colexicographic /
@@ -1107,27 +1146,29 @@ def _decode_full_shard_bulk_if_uncompressed(
11071146
return None
11081147
ab_codec = self.codecs[0]
11091148

1149+
# The byte-order handling below lacks the structured-dtype branch of
1150+
# `BytesCodec._decode_sync` (which applies `newbyteorder` to multi-byte
1151+
# struct fields), so structured dtypes must take the per-chunk path.
1152+
if isinstance(shard_spec.dtype, Struct):
1153+
return None
1154+
11101155
chunks_per_shard = self._get_chunks_per_shard(shard_spec)
11111156
chunk_spec = self._get_chunk_spec(shard_spec)
11121157
n_chunks = product(chunks_per_shard)
11131158
if n_chunks == 0:
11141159
return None
11151160

1116-
# Only valid for a plain contiguous full-shard read, where each chunk
1117-
# lands at its natural grid position. The `sel_shape` check is
1118-
# load-bearing: a gather indexer (CoordinateIndexer, from vindex / an
1119-
# oindex with an integer-array selection) reorders points and exposes
1120-
# `sel_shape`, but its `.shape` is the FLATTENED point count, which can
1121-
# equal the shard shape by coincidence (trivially in 1-D). Gating on
1122-
# shape alone lets such a selection through, and the bulk path then
1123-
# returns the shard in natural order, silently dropping the reordering.
1124-
# A contiguous full read (BasicIndexer, or a non-gathering
1125-
# OrthogonalIndexer from `arr[:]`) has no `sel_shape` and is served here.
1126-
# Anything that gathers must fall through to the per-chunk path so
1161+
# Only valid for an identity full-shard read, where each chunk lands at
1162+
# its natural grid position. The per-dimension check is load-bearing:
1163+
# a gather selection (an `OrthogonalIndexer` with an integer-array or
1164+
# boolean dimension, from `arr[perm, :]` / `arr.oindex[...]`, or a
1165+
# `CoordinateIndexer` from vindex) can have an output `.shape` equal to
1166+
# the shard shape while reordering or duplicating points — serving it
1167+
# from the bulk path would return the shard in natural order, silently
1168+
# dropping the reordering. Anything that is not a full-slice-per-
1169+
# dimension read falls through to the per-chunk path so
11271170
# chunk_selection / out_selection are honored.
1128-
if getattr(indexer, "sel_shape", None) is not None:
1129-
return None
1130-
if tuple(indexer.shape) != tuple(shard_spec.shape):
1171+
if not _is_identity_full_read(indexer, shard_spec.shape):
11311172
return None
11321173
chunk_byte_length = self._inner_chunk_byte_length(chunk_spec)
11331174

@@ -1141,7 +1182,8 @@ def _decode_full_shard_bulk_if_uncompressed(
11411182
else:
11421183
index_bytes = shard_bytes[-shard_index_size:]
11431184
index = self._decode_shard_index_sync(index_bytes, chunks_per_shard)
1144-
if not index.is_dense(chunk_byte_length):
1185+
data_section_start = shard_index_size if self.index_location == "start" else 0
1186+
if not index.is_dense(chunk_byte_length, data_section_start=data_section_start):
11451187
return None
11461188

11471189
# --- bulk reconstruct ---
@@ -1227,9 +1269,9 @@ def _decode_partial_sync(
12271269
return None
12281270
bulk = self._decode_full_shard_bulk_if_uncompressed(shard_bytes, shard_spec, indexer)
12291271
if bulk is not None:
1230-
# The bulk path only fires for a contiguous full-shard read (it
1231-
# returns None for any gather indexer that exposes `sel_shape`),
1232-
# so the result is already shard-shaped — no reshape needed.
1272+
# The bulk path only fires for an identity full-shard read
1273+
# (`_is_identity_full_read`), so the result is already
1274+
# shard-shaped and in natural order — no reshape needed.
12331275
return bulk
12341276
shard_reader = self._shard_reader_from_bytes_sync(shard_bytes, chunks_per_shard)
12351277
shard_dict: ShardMapping = shard_reader

0 commit comments

Comments
 (0)