Skip to content

Commit fc0e8cf

Browse files
author
Angus
committed
warn (not raise) on reserved node names in v3 consolidated listings
1 parent 7b01078 commit fc0e8cf

3 files changed

Lines changed: 77 additions & 16 deletions

File tree

‎changes/4485.bugfix.md‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1-
Reject invalid consolidated metadata listings at parse time. Listing keys
2-
with empty or reserved path segments (``.``, ``..``, ``zarr.json``,
3-
``__``-prefixed) now raise ``MetadataValidationError``, as do listings
4-
whose parent group entry is missing or that place a node below an array.
5-
Both previously surfaced as ``KeyError`` or left the group unopenable.
1+
Validate consolidated metadata listings at parse time. In v3 listings,
2+
keys containing a reserved node name (`.`, `..`, `zarr.json`, or a
3+
`__`-prefixed segment) now emit a `ZarrDeprecationWarning` instead of
4+
being accepted silently; stores written before these names were rejected
5+
still open. Keys with empty path segments still raise
6+
`MetadataValidationError`, as do listings whose parent group entry is
7+
missing or that place a node below an array; those cases previously
8+
surfaced as `KeyError` or left the group unopenable. The reserved-name
9+
check does not apply to v2 listings, whose specification does not
10+
reserve node names.

‎src/zarr/core/group.py‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
GroupNotFoundError,
6565
MetadataValidationError,
6666
NodeTypeValidationError,
67+
ZarrDeprecationWarning,
6768
ZarrUserWarning,
6869
)
6970
from zarr.storage import StoreLike, StorePath
@@ -140,14 +141,26 @@ def _parse_async_node(
140141
raise TypeError(f"Unknown node type, got {type(node)}")
141142

142143

143-
def _validate_consolidated_key(key: str) -> None:
144-
"""A consolidated metadata key is a node path: it must be non-empty, and every
145-
segment must be a valid node name (non-empty, not ``.``/``..``/``zarr.json``,
146-
not ``__``-prefixed)."""
144+
def _validate_consolidated_key(key: str, zarr_format: ZarrFormat) -> None:
145+
"""A consolidated metadata key is a node path: every segment must be non-empty.
146+
147+
The v3 specification also reserves the names ``.``, ``..``, ``zarr.json``, and
148+
every ``__``-prefixed name. Stores written before those names were rejected on
149+
creation still open today, so for now such keys only warn. The v2 specification
150+
does not reserve any node names, so no name checks apply to v2 listings."""
147151
segments = key.split("/")
148-
if any(not seg or seg in (".", "..", ZARR_JSON) or seg.startswith("__") for seg in segments):
152+
if any(not seg for seg in segments):
149153
msg = f"Invalid node path in consolidated metadata: {key!r}"
150154
raise MetadataValidationError(msg)
155+
if zarr_format == 3 and any(
156+
seg in (".", "..", ZARR_JSON) or seg.startswith("__") for seg in segments
157+
):
158+
msg = (
159+
f"Consolidated metadata lists {key!r}, which contains a name reserved "
160+
"by the v3 specification. Rename the node, as reading such listings "
161+
"will become an error."
162+
)
163+
warnings.warn(msg, category=ZarrDeprecationWarning, stacklevel=1)
151164

152165

153166
def _validate_consolidated_hierarchy(
@@ -225,9 +238,6 @@ def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> Consoli
225238

226239
metadata: dict[str, ArrayV2Metadata | ArrayV3Metadata | GroupMetadata] = {}
227240
if raw_metadata:
228-
for k in raw_metadata:
229-
_validate_consolidated_key(k)
230-
231241
for k, v in raw_metadata.items():
232242
member = k if path is None else _join_paths([path, k])
233243
if not isinstance(v, dict):
@@ -237,6 +247,7 @@ def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> Consoli
237247

238248
# zarr_format is present in v2 and v3.
239249
zarr_format = parse_zarr_format(v["zarr_format"])
250+
_validate_consolidated_key(k, zarr_format)
240251

241252
if zarr_format == 3:
242253
node_type = parse_node_type(v.get("node_type", None))

‎tests/test_metadata/test_consolidated.py‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import json
4+
import warnings
45
from typing import TYPE_CHECKING, Any
56

67
import numpy as np
@@ -22,7 +23,7 @@
2223
from zarr.core.group import ConsolidatedMetadata, GroupMetadata
2324
from zarr.core.metadata import ArrayV3Metadata
2425
from zarr.core.metadata.v2 import ArrayV2Metadata
25-
from zarr.errors import MetadataValidationError, ZarrUserWarning
26+
from zarr.errors import MetadataValidationError, ZarrDeprecationWarning, ZarrUserWarning
2627
from zarr.storage import StorePath
2728

2829
if TYPE_CHECKING:
@@ -918,6 +919,7 @@ async def test_consolidated_metadata_case_differing_siblings(memory_store: Store
918919
"codecs": [{"name": "bytes", "configuration": {"endian": "little"}}],
919920
}
920921
_GROUP_V3 = {"zarr_format": 3, "node_type": "group", "attributes": {}}
922+
_GROUP_V2 = {"zarr_format": 2}
921923

922924

923925
def _consolidated_doc(metadata: dict[str, Any]) -> dict[str, Any]:
@@ -926,14 +928,57 @@ def _consolidated_doc(metadata: dict[str, Any]) -> dict[str, Any]:
926928

927929
@pytest.mark.parametrize(
928930
"key",
929-
["", ".", "..", "zarr.json", "__a", "a/zarr.json", "a/../b", "a//b"],
931+
["", "a//b", "/a", "a/"],
930932
)
931933
def test_consolidated_metadata_rejects_invalid_keys(key: str) -> None:
932-
"""Listing keys are node paths; every segment must be a valid node name."""
934+
"""Listing keys are node paths; every segment must be non-empty."""
933935
with pytest.raises(MetadataValidationError, match="Invalid node path"):
934936
ConsolidatedMetadata.from_dict(_consolidated_doc({key: _GROUP_V3}))
935937

936938

939+
@pytest.mark.parametrize(
940+
"key",
941+
[".", "..", "zarr.json", "__a", "a/zarr.json", "a/__x"],
942+
)
943+
def test_consolidated_metadata_warns_on_reserved_v3_keys(key: str) -> None:
944+
"""v3 reserves '.', '..', 'zarr.json', and '__'-prefixed names, but stores
945+
written before the names were rejected still open, so they only warn."""
946+
with pytest.warns(ZarrDeprecationWarning, match="reserved"):
947+
consolidated = ConsolidatedMetadata.from_dict(
948+
_consolidated_doc({key: _GROUP_V3, "a": _GROUP_V3})
949+
)
950+
if "/" in key:
951+
parent, leaf = key.split("/")
952+
nested = consolidated.metadata[parent].consolidated_metadata
953+
assert nested is not None
954+
assert leaf in nested.metadata
955+
else:
956+
assert key in consolidated.metadata
957+
958+
959+
def test_consolidated_metadata_reserved_name_below_missing_parent() -> None:
960+
"""A reserved name warns before the missing-parent error is raised."""
961+
with pytest.warns(ZarrDeprecationWarning, match="reserved"):
962+
with pytest.raises(MetadataValidationError, match="without a listing"):
963+
ConsolidatedMetadata.from_dict(
964+
_consolidated_doc({"a": _GROUP_V3, "a/../b": _GROUP_V3})
965+
)
966+
967+
968+
@pytest.mark.parametrize(
969+
"key",
970+
[".", "..", "zarr.json", "__a"],
971+
)
972+
def test_consolidated_metadata_ignores_reserved_v2_keys(key: str) -> None:
973+
"""The v2 specification does not reserve node names."""
974+
with warnings.catch_warnings():
975+
warnings.simplefilter("error")
976+
consolidated = ConsolidatedMetadata.from_dict(
977+
_consolidated_doc({key: _GROUP_V2})
978+
)
979+
assert key in consolidated.metadata
980+
981+
937982
def test_consolidated_metadata_rejects_child_without_parent() -> None:
938983
"""A listing for 'a/b' with no 'a' listing made the root group unopenable."""
939984
with pytest.raises(MetadataValidationError, match="without a listing"):

0 commit comments

Comments
 (0)