Skip to content

Commit 9f0ae4e

Browse files
authored
fix: replace runtime assert statements with explicit checks (#4363)
* fix: replace runtime assert statements with explicit checks Asserts are stripped under `python -O`, so checks that matter for correctness or type narrowing must be explicit. Two were load-bearing: `GroupMetadata.from_dict` asserted on node_type and the array-to-group fallback in `zarr.api.asynchronous.open` caught the AssertionError, and `make_store` asserted on mode ahead of the real validation. Redundant asserts are deleted, narrowing asserts are restructured so mypy narrows on its own, and the rest become explicit raises. Ruff S101 is enabled with `tests/` and `src/zarr/testing/` excluded. Assisted-by: ClaudeCode:claude-fable-5-1 * docs: add changelog fragment for #4363 Assisted-by: ClaudeCode:claude-fable-5-1 * ci: scope the no-assert lint to runtime code across the monorepo The S101 per-file ignores only covered the root tests/ and src/zarr/testing/. The packages under packages/ inherit the root ruff config, so the rule fired on 1524 asserts in their tests, examples, and test-support modules and broke the ruff, Lint, pre-commit.ci, and zarr-http-server jobs. Widen the ignores to **/tests/**, **/examples/**, and **/testing/**, and exclude zarr-indexing's runtime source for now; its own asserts are tracked as a separate change. Assisted-by: ClaudeCode:claude-fable-5-1
1 parent b6c592e commit 9f0ae4e

21 files changed

Lines changed: 158 additions & 103 deletions

‎changes/4363.bugfix.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Removed every `assert` statement from runtime code and enabled ruff's `S101`
2+
rule to keep them out. Asserts are stripped under `python -O`, and two of them
3+
were load-bearing: `GroupMetadata.from_dict` asserted on `node_type` and the
4+
array-to-group fallback in `zarr.open` relied on catching the resulting
5+
`AssertionError`, and `make_store` asserted on `mode` before the real
6+
validation. Both now raise proper errors (`NodeTypeValidationError` and
7+
`ValueError`). Redundant asserts were deleted, type-narrowing asserts were
8+
restructured so mypy narrows without them, and the rest became explicit raises.

‎pyproject.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ extend-select = [
361361
"RET", # flake8-return
362362
"RSE", # flake8-raise
363363
"RUF",
364+
"S101", # flake8-bandit: no assert in runtime code
364365
"SIM", # flake8-simplify
365366
"SLOT", # flake8-slots
366367
"TC", # flake8-type-checking
@@ -395,6 +396,14 @@ ignore = [
395396

396397
[tool.ruff.lint.extend-per-file-ignores]
397398
"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"]
399+
# S101 (no assert) is for runtime code only. Tests, examples, and test-support
400+
# packages (store conformance suite, hypothesis strategies) assert on purpose.
401+
# The globs cover the root package and every package under packages/.
402+
"**/tests/**" = ["S101"]
403+
"**/examples/**" = ["S101"]
404+
"**/testing/**" = ["S101"]
405+
# zarr-indexing still has asserts in runtime code; tracked as a separate change.
406+
"packages/zarr-indexing/src/**" = ["S101"]
398407

399408
[tool.mypy]
400409
files = ["src", "tests"]

‎src/zarr/__init__.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,10 @@
3939
from zarr.core.group import AsyncGroup, Group
4040

4141
# in case setuptools scm screw up and find version to be 0.0.0
42-
assert not __version__.startswith("0.0.0")
42+
if __version__.startswith("0.0.0"):
43+
raise RuntimeError(
44+
f"zarr resolved its own version to {__version__!r}; the build metadata is broken."
45+
)
4346

4447
_logger = logging.getLogger(__name__)
4548

‎src/zarr/api/asynchronous.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,7 @@ async def open(
403403
return AsyncArray(
404404
store_path=store_path, metadata=_metadata_dict, config=kwargs.get("config")
405405
)
406-
except (AssertionError, FileNotFoundError, NodeTypeValidationError):
406+
except (FileNotFoundError, NodeTypeValidationError):
407407
pass
408408
return await open_group(store=store_path, zarr_format=zarr_format, mode=mode, **kwargs)
409409

‎src/zarr/codecs/bytes.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
from zarr.abc.codec import ArrayBytesCodec
99
from zarr.codecs._deprecated_enum import _coerce_enum_input, _DeprecatedStrEnumMeta
10-
from zarr.core.buffer import Buffer, NDBuffer
1110
from zarr.core.common import JSON, parse_named_configuration
1211
from zarr.core.dtype.common import HasEndianness
1312
from zarr.core.dtype.npy.structured import Struct
@@ -16,6 +15,7 @@
1615
from typing import Self
1716

1817
from zarr.core.array_spec import ArraySpec
18+
from zarr.core.buffer import Buffer, NDBuffer
1919

2020

2121
EndianLiteral = Literal["little", "big"]
@@ -142,7 +142,6 @@ def _encode_sync(
142142
chunk_array: NDBuffer,
143143
chunk_spec: ArraySpec,
144144
) -> Buffer | None:
145-
assert isinstance(chunk_array, NDBuffer)
146145
if chunk_array.dtype.itemsize > 1 and self.endian is not None:
147146
# Compare full dtypes rather than the top-level byteorder: numpy reports
148147
# byteorder '|' for structured dtypes even when their fields are

‎src/zarr/codecs/cast_value.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,9 @@ def _validate_scalar_map(
311311
target_zdtype: ZDType[TBaseDType, TBaseScalar],
312312
) -> None:
313313
"""Validate that scalar map entries are compatible with source/target dtypes."""
314-
assert self.scalar_map is not None
314+
scalar_map = self.scalar_map
315+
if scalar_map is None:
316+
return
315317
# For encode: keys are source values, values are target values.
316318
# For decode: keys are target values, values are source values.
317319
direction_dtypes: dict[
@@ -321,9 +323,9 @@ def _validate_scalar_map(
321323
"decode": (target_zdtype, source_zdtype),
322324
}
323325
for direction, (key_zdtype, val_zdtype) in direction_dtypes.items():
324-
if direction not in self.scalar_map:
326+
if direction not in scalar_map:
325327
continue
326-
sub_map = self.scalar_map[direction] # type: ignore[literal-required]
328+
sub_map = scalar_map[direction] # type: ignore[literal-required]
327329
for k, v in sub_map.items():
328330
_check_representable(k, key_zdtype, f"scalar_map {direction} key")
329331
_check_representable(v, val_zdtype, f"scalar_map {direction} value")

‎src/zarr/codecs/sharding.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,11 @@ class _ShardingByteGetter(ByteGetter):
160160
def get_sync(
161161
self, prototype: BufferPrototype | None = None, byte_range: ByteRequest | None = None
162162
) -> Buffer | None:
163-
assert prototype is None or prototype == default_buffer_prototype(), (
164-
f"prototype is not supported within shards currently. diff: {prototype} != {default_buffer_prototype()}"
165-
)
163+
if prototype is not None and prototype != default_buffer_prototype():
164+
raise ValueError(
165+
"Non-default buffer prototypes are not supported within shards. "
166+
f"Got {prototype}, expected {default_buffer_prototype()}."
167+
)
166168
value = self.shard_dict.get(self.chunk_coords)
167169
if value is None:
168170
return None
@@ -193,7 +195,8 @@ def delete_sync(self) -> None:
193195
del self.shard_dict[self.chunk_coords]
194196

195197
async def set(self, value: Buffer, byte_range: ByteRequest | None = None) -> None:
196-
assert byte_range is None, "byte_range is not supported within shards"
198+
if byte_range is not None:
199+
raise NotImplementedError("byte_range is not supported within shards.")
197200
self.set_sync(value)
198201

199202
async def delete(self) -> None:
@@ -656,7 +659,8 @@ def _encode_shard_index_sync(self, index: _ShardIndex) -> Buffer:
656659
index_spec = self._get_index_chunk_spec(index.chunks_per_shard)
657660
index_nd = get_ndbuffer_class().from_numpy_array(index.offsets_and_lengths)
658661
result: Buffer | None = index_transform.encode_chunk(index_nd, index_spec)
659-
assert result is not None
662+
if result is None:
663+
raise RuntimeError("Encoding the shard index produced no bytes.")
660664
return result
661665

662666
def _shard_reader_from_bytes_sync(
@@ -1531,7 +1535,8 @@ async def _decode_shard_index(
15311535
.decode([(index_bytes, self._get_index_chunk_spec(chunks_per_shard))])
15321536
)
15331537
)
1534-
assert index_array is not None # the bytes are already in hand
1538+
if index_array is None:
1539+
raise RuntimeError("Decoding the shard index produced no array.")
15351540
return _ShardIndex(chunks_per_shard, index_array.as_numpy_array())
15361541

15371542
async def _encode_shard_index(self, index: _ShardIndex) -> Buffer:
@@ -1553,7 +1558,8 @@ async def _encode_shard_index(self, index: _ShardIndex) -> Buffer:
15531558
)
15541559
)
15551560
)
1556-
assert index_bytes is not None
1561+
if index_bytes is None:
1562+
raise RuntimeError("Encoding the shard index produced no bytes.")
15571563
return index_bytes
15581564

15591565
def _shard_index_size(self, chunks_per_shard: tuple[int, ...]) -> int:

‎src/zarr/codecs/vlen_utf8.py‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@
88

99
from zarr._compat import _reshape_view
1010
from zarr.abc.codec import ArrayBytesCodec
11-
from zarr.core.buffer import Buffer, NDBuffer
1211
from zarr.core.common import JSON, parse_named_configuration
1312

1413
if TYPE_CHECKING:
1514
from typing import Self
1615

1716
from zarr.core.array_spec import ArraySpec
17+
from zarr.core.buffer import Buffer, NDBuffer
1818

1919

2020
# can use a global because there are no parameters
@@ -45,11 +45,8 @@ def _decode_sync(
4545
chunk_bytes: Buffer,
4646
chunk_spec: ArraySpec,
4747
) -> NDBuffer:
48-
assert isinstance(chunk_bytes, Buffer)
49-
5048
raw_bytes = chunk_bytes.as_array_like()
5149
decoded = _vlen_utf8_codec.decode(raw_bytes)
52-
assert decoded.dtype == np.object_
5350
decoded = _reshape_view(decoded, chunk_spec.shape)
5451
as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False)
5552
return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype)
@@ -66,7 +63,6 @@ def _encode_sync(
6663
chunk_array: NDBuffer,
6764
chunk_spec: ArraySpec,
6865
) -> Buffer | None:
69-
assert isinstance(chunk_array, NDBuffer)
7066
# numcodecs vlen codecs flatten with order="A", so an F-contiguous chunk
7167
# would be encoded in transposed element order (gh-3558)
7268
return chunk_spec.prototype.buffer.from_bytes(
@@ -106,11 +102,8 @@ def _decode_sync(
106102
chunk_bytes: Buffer,
107103
chunk_spec: ArraySpec,
108104
) -> NDBuffer:
109-
assert isinstance(chunk_bytes, Buffer)
110-
111105
raw_bytes = chunk_bytes.as_array_like()
112106
decoded = _vlen_bytes_codec.decode(raw_bytes)
113-
assert decoded.dtype == np.object_
114107
decoded = _reshape_view(decoded, chunk_spec.shape)
115108
return chunk_spec.prototype.nd_buffer.from_numpy_array(decoded)
116109

@@ -126,7 +119,6 @@ def _encode_sync(
126119
chunk_array: NDBuffer,
127120
chunk_spec: ArraySpec,
128121
) -> Buffer | None:
129-
assert isinstance(chunk_array, NDBuffer)
130122
# numcodecs vlen codecs flatten with order="A", so an F-contiguous chunk
131123
# would be encoded in transposed element order (gh-3558)
132124
return chunk_spec.prototype.buffer.from_bytes(

‎src/zarr/core/array.py‎

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,7 @@
158158
from zarr.abc.codec import CodecPipeline
159159
from zarr.abc.store import Store
160160
from zarr.codecs.sharding import IndexLocation, ShardingCodec
161+
from zarr.core.buffer import Buffer
161162
from zarr.core.dtype.wrapper import TBaseDType, TBaseScalar
162163
from zarr.storage import StoreLike
163164
from zarr.types import AnyArray, AnyAsyncArray, ArrayV2, ArrayV3, AsyncArrayV2, AsyncArrayV3
@@ -282,6 +283,7 @@ async def get_array_metadata(
282283
f"{store_path.store!r} at path {store_path.path!r}."
283284
)
284285
raise ArrayNotFoundError(msg)
286+
return _array_metadata_dict_v2(zarray_bytes, zattrs_bytes)
285287
elif zarr_format == 3:
286288
zarr_json_bytes = await (store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype)
287289
if zarr_json_bytes is None:
@@ -290,6 +292,7 @@ async def get_array_metadata(
290292
f"{store_path.store!r} at path {store_path.path!r}."
291293
)
292294
raise ArrayNotFoundError(msg)
295+
return _array_metadata_dict_v3(zarr_json_bytes)
293296
elif zarr_format is None:
294297
zarr_json_bytes, zarray_bytes, zattrs_bytes = await gather(
295298
(store_path / ZARR_JSON).get(prototype=cpu_buffer_prototype),
@@ -300,35 +303,34 @@ async def get_array_metadata(
300303
# warn and favor v3
301304
msg = f"Both zarr.json (Zarr format 3) and .zarray (Zarr format 2) metadata objects exist at {store_path}. Zarr v3 will be used."
302305
warnings.warn(msg, category=ZarrUserWarning, stacklevel=1)
303-
if zarr_json_bytes is None and zarray_bytes is None:
304-
msg = (
305-
f"Neither Zarr V3 nor Zarr V2 array metadata documents "
306-
f"were found in store {store_path.store!r} at path {store_path.path!r}."
307-
)
308-
raise ArrayNotFoundError(msg)
309-
# set zarr_format based on which keys were found
306+
# favor v3 when both are present
310307
if zarr_json_bytes is not None:
311-
zarr_format = 3
312-
else:
313-
zarr_format = 2
308+
return _array_metadata_dict_v3(zarr_json_bytes)
309+
if zarray_bytes is not None:
310+
return _array_metadata_dict_v2(zarray_bytes, zattrs_bytes)
311+
msg = (
312+
f"Neither Zarr V3 nor Zarr V2 array metadata documents "
313+
f"were found in store {store_path.store!r} at path {store_path.path!r}."
314+
)
315+
raise ArrayNotFoundError(msg)
314316
else:
315317
msg = f"Invalid value for 'zarr_format'. Expected 2, 3, or None. Got '{zarr_format}'." # type: ignore[unreachable]
316318
raise MetadataValidationError(msg)
317319

318-
metadata_dict: dict[str, JSON]
319-
if zarr_format == 2:
320-
# V2 arrays are comprised of a .zarray and .zattrs objects
321-
assert zarray_bytes is not None
322-
metadata_dict = buffer_to_json_object(zarray_bytes)
323-
zattrs_dict = buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {}
324-
metadata_dict["attributes"] = zattrs_dict
325-
else:
326-
# V3 arrays are comprised of a zarr.json object
327-
assert zarr_json_bytes is not None
328-
metadata_dict = buffer_to_json_object(zarr_json_bytes)
329320

330-
parse_node_type_array(metadata_dict.get("node_type"))
321+
def _array_metadata_dict_v2(zarray_bytes: Buffer, zattrs_bytes: Buffer | None) -> dict[str, JSON]:
322+
"""Combine a `.zarray` document and an optional `.zattrs` document into one metadata dict."""
323+
metadata_dict: dict[str, JSON] = buffer_to_json_object(zarray_bytes)
324+
metadata_dict["attributes"] = (
325+
buffer_to_json_object(zattrs_bytes) if zattrs_bytes is not None else {}
326+
)
327+
return metadata_dict
331328

329+
330+
def _array_metadata_dict_v3(zarr_json_bytes: Buffer) -> dict[str, JSON]:
331+
"""Parse a `zarr.json` document, checking that it describes an array."""
332+
metadata_dict: dict[str, JSON] = buffer_to_json_object(zarr_json_bytes)
333+
parse_node_type_array(metadata_dict.get("node_type"))
332334
return metadata_dict
333335

334336

@@ -5896,7 +5898,11 @@ async def _resize(
58965898
If False, the data in those chunks will be preserved.
58975899
"""
58985900
new_shape = parse_shapelike(new_shape)
5899-
assert len(new_shape) == len(array.metadata.shape)
5901+
if len(new_shape) != len(array.metadata.shape):
5902+
raise ValueError(
5903+
f"The new shape must have the same number of dimensions as the array. "
5904+
f"Got {len(new_shape)} dimension(s), expected {len(array.metadata.shape)}."
5905+
)
59005906

59015907
new_metadata = array.metadata.update_shape(new_shape)
59025908
new_chunk_grid = ChunkGrid.from_metadata(new_metadata)

‎src/zarr/core/buffer/cpu.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@ def combine(self, others: Iterable[core.Buffer]) -> Self:
111111
data = [np.asanyarray(self._data)]
112112
for buf in others:
113113
other_array = buf.as_array_like()
114-
assert other_array.dtype == np.dtype("B")
115114
data.append(np.asanyarray(other_array))
116115
return self.__class__(np.concatenate(data))
117116

0 commit comments

Comments
 (0)