Skip to content

Commit 24f9ad1

Browse files
authored
fix: make consolidated metadata nesting independent of persisted key order (zarr-developers#4227)
* fix: make consolidated metadata nesting independent of persisted key order `ConsolidatedMetadata._flat_to_nested` grouped the flat keys with `itertools.groupby` over keys sorted by depth alone. `groupby` only groups *consecutive* runs, so when a parent's children were not adjacent it emitted several runs for the same parent and the surrounding dict comprehension kept only the last one. Every child in the earlier runs was silently never re-parented, and lingered as a bogus slash-containing key at the top level, making it unreachable through the consolidated metadata. The persisted key order is arbitrary, so nesting must not depend on it. Group by parent with an accumulating mapping instead. This is reachable from zarr-python itself: `to_dict` sorts keys by `(depth, NFKC-casefold(key))`, so sibling subtrees whose names differ only by case interleave and trigger exactly this pattern. Fixes zarr-developers#4226 Assisted-by: ClaudeCode:claude-opus-5 * docs: add changelog entry for 273 Assisted-by: ClaudeCode:claude-opus-5 * docs: renumber changelog fragment to the upstream PR number Assisted-by: ClaudeCode:claude-opus-5
1 parent a994a4f commit 24f9ad1

3 files changed

Lines changed: 69 additions & 5 deletions

File tree

‎changes/4227.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Consolidated metadata is now reconstructed independently of the order the keys appear in on disk. Previously, sibling subtrees whose keys were not adjacent in the persisted mapping lost their children, which made nodes unreachable through consolidated metadata -- most visibly for sibling groups whose names differ only by case.

‎src/zarr/core/group.py‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import asyncio
4-
import itertools
54
import logging
65
import unicodedata
76
import warnings
@@ -237,10 +236,12 @@ def _flat_to_nested(
237236
# In the example, the group at `/a/b` will have consolidated metadata
238237
# for its children `array-0` and `array-1`.
239238

240-
keys = sorted(metadata, key=lambda k: k.count("/"))
241-
grouped = {
242-
k: list(v) for k, v in itertools.groupby(keys, key=lambda k: k.rsplit("/", 1)[0])
243-
}
239+
# Group keys by their parent path. This must not rely on same-parent keys
240+
# being adjacent: the persisted key order is arbitrary, so accumulate
241+
# instead of using itertools.groupby, which only groups consecutive runs.
242+
grouped: dict[str, list[str]] = defaultdict(list)
243+
for k in sorted(metadata, key=lambda k: k.count("/")):
244+
grouped[k.rsplit("/", 1)[0]].append(k)
244245

245246
# we go top down and directly manipulate metadata.
246247
for key, children_keys in grouped.items():

‎tests/test_metadata/test_consolidated.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -839,3 +839,65 @@ async def test_open_group_in_non_consolidating_stores() -> None:
839839
# Opening a group with use_consolidated=True should fail
840840
with pytest.raises(ValueError, match="doesn't support consolidated metadata"):
841841
await AsyncGroup.open(memory_store, use_consolidated=True)
842+
843+
844+
@pytest.mark.parametrize(
845+
"order",
846+
[
847+
# keys grouped by parent, the order zarr-python used to write before it
848+
# started sorting the persisted keys
849+
["a", "b", "a/x", "a/y", "b/x", "b/y"],
850+
# sibling subtrees interleaved, which is what the (depth, casefold) sort
851+
# produces for names differing only by case
852+
["a", "b", "a/x", "b/x", "a/y", "b/y"],
853+
# reversed, to cover a parent appearing after its children in the mapping
854+
["b/y", "b/x", "a/y", "a/x", "b", "a"],
855+
],
856+
)
857+
def test_flat_to_nested_is_order_independent(order: list[str]) -> None:
858+
"""The persisted key order is arbitrary, so nesting must not depend on it."""
859+
group_metadata: dict[str, JSON] = {"zarr_format": 3, "node_type": "group", "attributes": {}}
860+
consolidated = ConsolidatedMetadata.from_dict(
861+
{
862+
"kind": "inline",
863+
"must_understand": False,
864+
"metadata": dict.fromkeys(order, group_metadata),
865+
}
866+
)
867+
868+
assert sorted(consolidated.metadata) == ["a", "b"]
869+
for name in ("a", "b"):
870+
child = consolidated.metadata[name]
871+
assert isinstance(child, GroupMetadata)
872+
assert child.consolidated_metadata is not None
873+
assert sorted(child.consolidated_metadata.metadata) == ["x", "y"]
874+
875+
876+
async def test_consolidated_metadata_case_differing_siblings(memory_store: Store) -> None:
877+
"""Sibling nodes whose names differ only by case each keep their own children.
878+
879+
Regression test for https://github.com/zarr-developers/zarr-python/issues/4226
880+
"""
881+
root = await zarr.api.asynchronous.create_group(store=memory_store)
882+
for name in ("Study", "study"):
883+
child = await root.create_group(f"obs/{name}")
884+
await child.create_array(name="categories", shape=(2,), dtype="uint8")
885+
await child.create_array(name="codes", shape=(2,), dtype="uint8")
886+
887+
with pytest.warns(
888+
ZarrUserWarning,
889+
match="Consolidated metadata is currently not part in the Zarr format 3 specification.",
890+
):
891+
await consolidate_metadata(memory_store)
892+
893+
consolidated = await open_consolidated(store=memory_store)
894+
result = sorted([key async for key, _ in consolidated.members(max_depth=None)])
895+
assert result == [
896+
"obs",
897+
"obs/Study",
898+
"obs/Study/categories",
899+
"obs/Study/codes",
900+
"obs/study",
901+
"obs/study/categories",
902+
"obs/study/codes",
903+
]

0 commit comments

Comments
 (0)