5353from zarr .core .config import config as zarr_config
5454from zarr .core .dtype .common import HasEndianness
5555from zarr .core .dtype .npy .int import UInt64
56+ from zarr .core .dtype .npy .structured import Struct
5657from zarr .core .indexing import (
5758 BasicIndexer ,
5859 ChunkProjection ,
5960 SelectorTuple ,
61+ SliceDimIndexer ,
6062 _lexicographic_order ,
6163 colexicographic_order_coords ,
6264 get_indexer ,
@@ -109,6 +111,32 @@ class ShardingCodecIndexLocation(metaclass=_DeprecatedStrEnumMeta):
109111)
110112
111113
114+ def _is_identity_full_read (indexer : Any , shard_shape : tuple [int , ...]) -> bool :
115+ """True when `indexer` selects every element of a `shard_shape` array in
116+ natural order: one whole-dimension, step-1 `SliceDimIndexer` per dimension.
117+
118+ Structural on purpose, not `isinstance(indexer, BasicIndexer)`: a full
119+ `arr[:]` read reaches the shard as an `OrthogonalIndexer`, so a type gate
120+ would silently disable the bulk fast path for the most common case. Any
121+ gather (integer-array / boolean / coordinate selection), subset, strided,
122+ or integer-scalar selection fails the per-dimension check — output shape
123+ alone is not enough, because a reordering or duplicating selection can have
124+ the same shape as the shard while requiring `chunk_selection` /
125+ `out_selection` to be honored.
126+ """
127+ dim_indexers = getattr (indexer , "dim_indexers" , None )
128+ if dim_indexers is None or len (dim_indexers ) != len (shard_shape ):
129+ return False
130+ return all (
131+ isinstance (dim_indexer , SliceDimIndexer )
132+ and dim_indexer .dim_len == dim_len
133+ and dim_indexer .start == 0
134+ and dim_indexer .stop == dim_len
135+ and dim_indexer .step == 1
136+ for dim_indexer , dim_len in zip (dim_indexers , shard_shape , strict = True )
137+ )
138+
139+
112140def _parse_index_location (data : object ) -> IndexLocation :
113141 if isinstance (data , str ) and data in INDEX_LOCATION :
114142 return data # type: ignore[return-value]
@@ -192,12 +220,17 @@ def is_all_empty(self) -> bool:
192220 def get_full_chunk_map (self ) -> npt .NDArray [np .bool_ ]:
193221 return np .not_equal (self .offsets_and_lengths [..., 0 ], MAX_UINT_64 )
194222
195- def is_dense (self , chunk_byte_length : int ) -> bool :
196- """True when every chunk is present, fixed-length, and uniquely placed.
197-
198- Used to gate the vectorized whole-shard decode: a dense fixed-size shard
199- is a regular grid of equal-length payloads, so it can be reshaped/scattered
200- in bulk rather than decoded chunk-by-chunk.
223+ def is_dense (self , chunk_byte_length : int , * , data_section_start : int ) -> bool :
224+ """True when the chunk payloads exactly tile the shard's data section.
225+
226+ Every chunk must be present with length `chunk_byte_length`, and the
227+ sorted offsets must be exactly `data_section_start + i * chunk_byte_length`
228+ for `i` in `0..n_chunks-1`: no gaps, no overlaps, and nothing outside the
229+ data section (a corrupt index could otherwise point chunks into the
230+ index region or out of the blob). Used to gate the vectorized
231+ whole-shard decode: a dense fixed-size shard is a regular grid of
232+ equal-length payloads, so it can be reshaped/scattered in bulk rather
233+ than decoded chunk-by-chunk.
201234 """
202235 offsets = self .offsets_and_lengths [..., 0 ].reshape (- 1 )
203236 lengths = self .offsets_and_lengths [..., 1 ].reshape (- 1 )
@@ -207,8 +240,10 @@ def is_dense(self, chunk_byte_length: int) -> bool:
207240 # all the same fixed length
208241 if not bool (np .all (lengths == chunk_byte_length )):
209242 return False
210- # offsets unique (no two chunks share a slot)
211- return int (np .unique (offsets ).size ) == int (offsets .size )
243+ expected = np .uint64 (data_section_start ) + np .arange (
244+ offsets .size , dtype = np .uint64
245+ ) * np .uint64 (chunk_byte_length )
246+ return bool (np .array_equal (np .sort (offsets ), expected ))
212247
213248 def get_chunk_slice (self , chunk_coords : tuple [int , ...]) -> tuple [int , int ] | None :
214249 localized_chunk = self ._localize_chunk (chunk_coords )
@@ -1086,8 +1121,12 @@ def _decode_full_shard_bulk_if_uncompressed(
10861121 dtype/endian view with no reordering. A trailing crc32c is NOT accepted
10871122 (the bulk path can't verify per-chunk checksums, so crc shards keep the
10881123 per-chunk path's corruption detection);
1124+ - the data type is not structured (the byte-order handling below has no
1125+ `Struct` branch);
1126+ - `indexer` is an identity full-shard read (`_is_identity_full_read`);
10891127 - the stored index is dense (every chunk present, equal fixed length,
1090- contiguous) so the data section is a regular grid of chunk payloads.
1128+ exactly tiling the data section) so the data section is a regular
1129+ grid of chunk payloads.
10911130
10921131 Chunk positions are read from the stored index, so this is correct for
10931132 any `subchunk_write_order` (morton / lexicographic / colexicographic /
@@ -1107,27 +1146,29 @@ def _decode_full_shard_bulk_if_uncompressed(
11071146 return None
11081147 ab_codec = self .codecs [0 ]
11091148
1149+ # The byte-order handling below lacks the structured-dtype branch of
1150+ # `BytesCodec._decode_sync` (which applies `newbyteorder` to multi-byte
1151+ # struct fields), so structured dtypes must take the per-chunk path.
1152+ if isinstance (shard_spec .dtype , Struct ):
1153+ return None
1154+
11101155 chunks_per_shard = self ._get_chunks_per_shard (shard_spec )
11111156 chunk_spec = self ._get_chunk_spec (shard_spec )
11121157 n_chunks = product (chunks_per_shard )
11131158 if n_chunks == 0 :
11141159 return None
11151160
1116- # Only valid for a plain contiguous full-shard read, where each chunk
1117- # lands at its natural grid position. The `sel_shape` check is
1118- # load-bearing: a gather indexer (CoordinateIndexer, from vindex / an
1119- # oindex with an integer-array selection) reorders points and exposes
1120- # `sel_shape`, but its `.shape` is the FLATTENED point count, which can
1121- # equal the shard shape by coincidence (trivially in 1-D). Gating on
1122- # shape alone lets such a selection through, and the bulk path then
1123- # returns the shard in natural order, silently dropping the reordering.
1124- # A contiguous full read (BasicIndexer, or a non-gathering
1125- # OrthogonalIndexer from `arr[:]`) has no `sel_shape` and is served here.
1126- # Anything that gathers must fall through to the per-chunk path so
1161+ # Only valid for an identity full-shard read, where each chunk lands at
1162+ # its natural grid position. The per-dimension check is load-bearing:
1163+ # a gather selection (an `OrthogonalIndexer` with an integer-array or
1164+ # boolean dimension, from `arr[perm, :]` / `arr.oindex[...]`, or a
1165+ # `CoordinateIndexer` from vindex) can have an output `.shape` equal to
1166+ # the shard shape while reordering or duplicating points — serving it
1167+ # from the bulk path would return the shard in natural order, silently
1168+ # dropping the reordering. Anything that is not a full-slice-per-
1169+ # dimension read falls through to the per-chunk path so
11271170 # chunk_selection / out_selection are honored.
1128- if getattr (indexer , "sel_shape" , None ) is not None :
1129- return None
1130- if tuple (indexer .shape ) != tuple (shard_spec .shape ):
1171+ if not _is_identity_full_read (indexer , shard_spec .shape ):
11311172 return None
11321173 chunk_byte_length = self ._inner_chunk_byte_length (chunk_spec )
11331174
@@ -1141,7 +1182,8 @@ def _decode_full_shard_bulk_if_uncompressed(
11411182 else :
11421183 index_bytes = shard_bytes [- shard_index_size :]
11431184 index = self ._decode_shard_index_sync (index_bytes , chunks_per_shard )
1144- if not index .is_dense (chunk_byte_length ):
1185+ data_section_start = shard_index_size if self .index_location == "start" else 0
1186+ if not index .is_dense (chunk_byte_length , data_section_start = data_section_start ):
11451187 return None
11461188
11471189 # --- bulk reconstruct ---
@@ -1227,9 +1269,9 @@ def _decode_partial_sync(
12271269 return None
12281270 bulk = self ._decode_full_shard_bulk_if_uncompressed (shard_bytes , shard_spec , indexer )
12291271 if bulk is not None :
1230- # The bulk path only fires for a contiguous full-shard read (it
1231- # returns None for any gather indexer that exposes `sel_shape`),
1232- # so the result is already shard-shaped — no reshape needed.
1272+ # The bulk path only fires for an identity full-shard read
1273+ # (`_is_identity_full_read`), so the result is already
1274+ # shard-shaped and in natural order — no reshape needed.
12331275 return bulk
12341276 shard_reader = self ._shard_reader_from_bytes_sync (shard_bytes , chunks_per_shard )
12351277 shard_dict : ShardMapping = shard_reader
0 commit comments