Skip to content

Commit 23231e6

Browse files
committed
fix(array): encode new array metadata before deleting an existing node
With the rectilinear flag checked when metadata is stored rather than when it is built, `zarr.create(..., overwrite=True)` with rectilinear chunks and the flag off deleted the existing array and then raised; zarr 3.4.0 raised before deleting anything. `AsyncArray._create_v2`, `_create_v3` and `init_array` now build and encode the new metadata before `_prepare_overwrite`, so metadata that cannot be stored fails with the store untouched. This also covers `create_array(..., overwrite=True)`, which deleted the existing array before validating its arguments. Assisted-by: ClaudeCode:claude-opus-5-5
1 parent 1a91532 commit 23231e6

3 files changed

Lines changed: 29 additions & 9 deletions

File tree

‎changes/4375.bugfix.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
Arrays whose stored `regular` chunk grid mixes chunk sizes with lists of chunk edge lengths, such as `"chunk_shape": [2, [5, 10, 5]]` (written by zarr 3.2.0 and 3.2.1 for `chunks=(2, (5, 10, 5))`, and as `[2, [5.0, 10.0, 5.0]]` for float edges), can be read again, without enabling `array.rectilinear_chunks`. The grid is read as the rectilinear chunk grid it describes, with a `ZarrUserWarning`; re-saving the array's metadata (or writing chunks to it) stores that rectilinear grid, which requires the flag. A group's consolidated metadata keeps such an array's metadata as it was stored. A regular chunk grid given edge lists is now rejected, so this metadata is no longer written.
22

3-
The `array.rectilinear_chunks` flag now gates reading and storing array metadata that declares a rectilinear chunk grid, including in a group's consolidated metadata, instead of constructing `RectilinearChunkGridMetadata`. `Array.resize`, deleting a group member, and `create_hierarchy(..., overwrite=True)` now encode the metadata they will store before deleting anything, so metadata that cannot be stored (for example, a rectilinear chunk grid with the flag off) fails with the store untouched. That error names the array when it is a member of a group's consolidated metadata.
3+
The `array.rectilinear_chunks` flag now gates reading and storing array metadata that declares a rectilinear chunk grid, including in a group's consolidated metadata, instead of constructing `RectilinearChunkGridMetadata`. `Array.resize`, deleting a group member, `create_hierarchy(..., overwrite=True)`, and creating an array with `overwrite=True` (`zarr.create`, `zarr.create_array` and the functions built on them) now encode the metadata they will store before deleting anything, so metadata that cannot be stored (for example, a rectilinear chunk grid with the flag off) fails with the store untouched. That error names the array when it is a member of a group's consolidated metadata.

‎src/zarr/core/array.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -646,8 +646,6 @@ async def _create_v3(
646646
attributes: dict[str, JSON] | None = None,
647647
overwrite: bool = False,
648648
) -> AsyncArrayV3:
649-
await _prepare_overwrite(store_path, zarr_format=3, overwrite=overwrite)
650-
651649
if isinstance(chunk_key_encoding, tuple):
652650
chunk_key_encoding = (
653651
V2ChunkKeyEncoding(separator=chunk_key_encoding[1])
@@ -665,6 +663,10 @@ async def _create_v3(
665663
dimension_names=dimension_names,
666664
attributes=attributes,
667665
)
666+
# Encode first, so metadata that cannot be stored fails before any existing
667+
# node is deleted.
668+
encode_documents(store_path, metadata)
669+
await _prepare_overwrite(store_path, zarr_format=3, overwrite=overwrite)
668670

669671
array = cls(metadata=metadata, store_path=store_path, config=config)
670672
await array._save_metadata(metadata, ensure_parents=True)
@@ -721,8 +723,6 @@ async def _create_v2(
721723
attributes: dict[str, JSON] | None = None,
722724
overwrite: bool = False,
723725
) -> AsyncArrayV2:
724-
await _prepare_overwrite(store_path, zarr_format=2, overwrite=overwrite)
725-
726726
compressor_parsed: CompressorLikev2
727727
if compressor == "auto":
728728
compressor_parsed = default_compressor_v2(dtype)
@@ -748,6 +748,10 @@ async def _create_v2(
748748
compressor=compressor_parsed,
749749
attributes=attributes,
750750
)
751+
# Encode first, so metadata that cannot be stored fails before any existing
752+
# node is deleted.
753+
encode_documents(store_path, metadata)
754+
await _prepare_overwrite(store_path, zarr_format=2, overwrite=overwrite)
751755

752756
array = cls(metadata=metadata, store_path=store_path, config=config)
753757
await array._save_metadata(metadata, ensure_parents=True)
@@ -4591,8 +4595,6 @@ async def init_array(
45914595
chunk_key_encoding, zarr_format=zarr_format
45924596
)
45934597

4594-
await _prepare_overwrite(store_path, zarr_format=zarr_format, overwrite=overwrite)
4595-
45964598
# Validate rectilinear chunks constraints
45974599
if _is_rectilinear_chunks(chunks):
45984600
if zarr_format == 2:
@@ -4705,6 +4707,10 @@ async def init_array(
47054707
attributes=attributes,
47064708
)
47074709

4710+
# Encode first, so metadata that cannot be stored fails before any existing node is
4711+
# deleted.
4712+
encode_documents(store_path, meta)
4713+
await _prepare_overwrite(store_path, zarr_format=zarr_format, overwrite=overwrite)
47084714
arr = AsyncArray(metadata=meta, store_path=store_path, config=config)
47094715
await arr._save_metadata(meta, ensure_parents=True)
47104716
return arr

‎tests/test_metadata/test_upgrades.py‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1471,6 +1471,14 @@ def _overwrite_hierarchy(path: Path) -> None:
14711471
list(zarr.create_hierarchy(store=LocalStore(path), nodes={"n": mixed.metadata}, overwrite=True))
14721472

14731473

1474+
def _overwrite_with_create(path: Path) -> None:
1475+
zarr.create(shape=(4,), chunks=[[2, 2]], dtype="int64", store=path / "n", overwrite=True)
1476+
1477+
1478+
def _overwrite_with_create_array(path: Path) -> None:
1479+
zarr.create_array(path / "n", shape=(4,), chunks=[[2, 2]], dtype="int64", overwrite=True)
1480+
1481+
14741482
def _set_group_attribute(path: Path) -> None:
14751483
zarr.open_group(path, mode="a").attrs["x"] = 1
14761484

@@ -1481,8 +1489,14 @@ def _set_group_attribute(path: Path) -> None:
14811489
@pytest.mark.filterwarnings("ignore:Consolidated metadata is currently not part:UserWarning")
14821490
@pytest.mark.parametrize(
14831491
"action",
1484-
[_resize, _write_chunks, _overwrite_hierarchy],
1485-
ids=["resize", "write", "overwrite-hierarchy"],
1492+
[
1493+
_resize,
1494+
_write_chunks,
1495+
_overwrite_hierarchy,
1496+
_overwrite_with_create,
1497+
_overwrite_with_create_array,
1498+
],
1499+
ids=["resize", "write", "overwrite-hierarchy", "overwrite-create", "overwrite-create-array"],
14861500
)
14871501
def test_store_untouched_without_flag(tmp_path: Path, action: Callable[[Path], None]) -> None:
14881502
"""An operation that would store the rectilinear chunk grid read from the verbatim

0 commit comments

Comments
 (0)