Skip to content

Commit eff4c60

Browse files
d-v-bclaude
andcommitted
fix(group,sync): honor use_consolidated_for_children, merge async attribute updates, cancel timed-out sync coroutines
- `AsyncGroup.members`, `Group.members` and `_iter_members_deep` now honor and forward `use_consolidated_for_children`, instead of unconditionally short-circuiting to consolidated metadata when the root has it. - `Group.update_attributes_async` merges new attributes into the existing ones, matching `Group.update_attributes` and `AsyncGroup.update_attributes`. - `sync()` cancels the pending future when it times out instead of leaving the coroutine running on the IO thread. - The `zarr.storage.default_compressor` deprecation warning no longer points at the removed `array.v2_default_compressor` config key. - GPU `NDBuffer.create` zero-fills when no fill value is given, matching the CPU buffer. Addresses B11, B12, B14, B15 and B16 of #181. B13 (`require_array(dtype=None)`) was dropped: zarr-developers#4189 deliberately kept the float64 default. Assisted-by: ClaudeCode:claude-fable-5-1 Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent ba883a5 commit eff4c60

7 files changed

Lines changed: 155 additions & 23 deletions

File tree

‎changes/190.bugfix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix several small bugs in `Group` and `sync`: `Group.members` and `AsyncGroup.members` now honor `use_consolidated_for_children=False` when the group itself has consolidated metadata, and forward the flag when recursing; `Group.update_attributes_async` now merges new attributes into the existing ones instead of replacing them, matching `Group.update_attributes`; `sync()` now cancels the pending coroutine when it times out instead of leaving it running on the IO thread; the deprecation warning for `zarr.storage.default_compressor` no longer points at a removed config key; and the GPU `NDBuffer.create` zero-fills when no fill value is given, matching the CPU buffer.

‎src/zarr/core/buffer/gpu.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,9 +176,12 @@ def create(
176176
order: Literal["C", "F"] = "C",
177177
fill_value: Any | None = None,
178178
) -> Self:
179+
# Mirror CPU semantics: zero-fill when fill_value is None (cp.zeros is
180+
# faster than cp.full and matches numpy's behaviour for the common case).
181+
if fill_value is None or (isinstance(fill_value, int) and fill_value == 0):
182+
return cls(cp.zeros(shape=tuple(shape), dtype=dtype, order=order))
179183
ret = cls(cp.empty(shape=tuple(shape), dtype=dtype, order=order))
180-
if fill_value is not None:
181-
ret.fill(fill_value)
184+
ret.fill(fill_value)
182185
return ret
183186

184187
@classmethod

‎src/zarr/core/group.py‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1358,10 +1358,9 @@ async def members(
13581358
``max_depth=None`` to include all nodes, and some positive integer
13591359
to consider children within that many levels of the root Group.
13601360
use_consolidated_for_children : bool, default True
1361-
Whether to use the consolidated metadata of child groups loaded
1362-
from the store. Note that this only affects groups loaded from the
1363-
store. If the current Group already has consolidated metadata, it
1364-
will always be used.
1361+
Whether to use consolidated metadata when listing members. If
1362+
`False`, members are listed from the store even when this group
1363+
or any child group already has consolidated metadata loaded.
13651364
13661365
Returns
13671366
-------
@@ -1414,7 +1413,7 @@ async def _members(
14141413
else:
14151414
raise ValueError(f"Unknown Zarr format: {self.metadata.zarr_format}")
14161415

1417-
if self.metadata.consolidated_metadata is not None:
1416+
if self.metadata.consolidated_metadata is not None and use_consolidated_for_children:
14181417
members = self._members_consolidated(max_depth=max_depth)
14191418
for member in members:
14201419
yield member
@@ -2086,20 +2085,25 @@ def __repr__(self) -> str:
20862085
async def update_attributes_async(self, new_attributes: dict[str, Any]) -> Group:
20872086
"""Update the attributes of this group.
20882087
2088+
Existing attributes are preserved; `new_attributes` are merged on top
2089+
of them (same semantics as `Group.update_attributes`).
2090+
20892091
Examples
20902092
--------
20912093
>>> async def example():
20922094
... import zarr
20932095
...
20942096
... group = zarr.group()
2095-
... new_group = await group.update_attributes_async({"foo": "bar"})
2097+
... group = group.update_attributes({"foo": "bar"})
2098+
... new_group = await group.update_attributes_async({"baz": "qux"})
20962099
... return new_group.attrs.asdict()
20972100
20982101
>>> import asyncio
20992102
>>> asyncio.run(example())
2100-
{'foo': 'bar'}
2103+
{'foo': 'bar', 'baz': 'qux'}
21012104
"""
2102-
new_metadata = replace(self.metadata, attributes=new_attributes)
2105+
merged = {**self.metadata.attributes, **new_attributes}
2106+
new_metadata = replace(self.metadata, attributes=merged)
21032107

21042108
# Write new metadata
21052109
to_save = new_metadata.to_buffer_dict(default_buffer_prototype())
@@ -2237,10 +2241,9 @@ def members(
22372241
``max_depth=None`` to include all nodes, and some positive integer
22382242
to consider children within that many levels of the root Group.
22392243
use_consolidated_for_children : bool, default True
2240-
Whether to use the consolidated metadata of child groups loaded
2241-
from the store. Note that this only affects groups loaded from the
2242-
store. If the current Group already has consolidated metadata, it
2243-
will always be used.
2244+
Whether to use consolidated metadata when listing members. If
2245+
`False`, members are listed from the store even when this group
2246+
or any child group already has consolidated metadata loaded.
22442247
22452248
Returns
22462249
-------
@@ -2249,7 +2252,12 @@ def members(
22492252
value: AsyncArray or AsyncGroup
22502253
The AsyncArray or AsyncGroup that is a child of ``self``.
22512254
"""
2252-
_members = self._sync_iter(self._async_group.members(max_depth=max_depth))
2255+
_members = self._sync_iter(
2256+
self._async_group.members(
2257+
max_depth=max_depth,
2258+
use_consolidated_for_children=use_consolidated_for_children,
2259+
)
2260+
)
22532261

22542262
return tuple((kv[0], _parse_async_node(kv[1])) for kv in _members)
22552263

@@ -3446,10 +3454,9 @@ async def _iter_members_deep(
34463454
semaphore : asyncio.Semaphore | None
34473455
An optional semaphore to use for concurrency control.
34483456
use_consolidated_for_children : bool, default True
3449-
Whether to use the consolidated metadata of child groups loaded
3450-
from the store. Note that this only affects groups loaded from the
3451-
store. If the current Group already has consolidated metadata, it
3452-
will always be used.
3457+
Whether to use consolidated metadata when listing members. If
3458+
`False`, members are listed from the store even when this group
3459+
or any child group already has consolidated metadata loaded.
34533460
34543461
Yields
34553462
------
@@ -3478,7 +3485,11 @@ async def _iter_members_deep(
34783485
if is_group and do_recursion:
34793486
node = cast("AsyncGroup", node)
34803487
to_recurse[name] = _iter_members_deep(
3481-
node, max_depth=new_depth, skip_keys=skip_keys, semaphore=semaphore
3488+
node,
3489+
max_depth=new_depth,
3490+
skip_keys=skip_keys,
3491+
semaphore=semaphore,
3492+
use_consolidated_for_children=use_consolidated_for_children,
34823493
)
34833494

34843495
for prefix, subgroup_iter in to_recurse.items():

‎src/zarr/core/sync.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ def sync[T](
148148

149149
finished, unfinished = wait([future], return_when=asyncio.ALL_COMPLETED, timeout=timeout)
150150
if len(unfinished) > 0:
151+
future.cancel()
151152
raise TimeoutError(f"Coroutine {coro} failed to finish within {timeout} s")
152153
assert len(finished) == 1
153154
return_result = next(iter(finished)).result()

‎src/zarr/storage/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ def __setattr__(self, attr: str, value: Any) -> None:
3333
if attr == "default_compressor":
3434
warnings.warn(
3535
"setting zarr.storage.default_compressor is deprecated, use "
36-
"zarr.config to configure array.v2_default_compressor "
37-
"e.g. config.set({'codecs.zstd':'numcodecs.Zstd', 'array.v2_default_compressor.numeric': 'zstd'})",
36+
"the `compressors` argument to zarr.create_array() instead "
37+
"e.g. zarr.create_array(..., compressors=[numcodecs.Zstd()])",
3838
ZarrDeprecationWarning,
3939
stacklevel=1,
4040
)

‎tests/test_group.py‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -770,6 +770,10 @@ def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None:
770770
async def test_group_update_attributes_async(store: Store, zarr_format: ZarrFormat) -> None:
771771
"""
772772
Test the behavior of `Group.update_attributes_async`
773+
774+
update_attributes_async must *merge* new attributes with existing ones,
775+
matching the semantics of the synchronous update_attributes path.
776+
Regression test for B12.
773777
"""
774778
attrs = {"foo": 100}
775779
group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs)
@@ -780,7 +784,9 @@ async def test_group_update_attributes_async(store: Store, zarr_format: ZarrForm
780784
new_group = await group.update_attributes_async(new_attrs)
781785
else:
782786
new_group = await group.update_attributes_async(new_attrs)
783-
assert new_group.attrs == new_attrs
787+
# Both the original and the new key must be present (merge, not overwrite).
788+
expected_attrs = {**attrs, **new_attrs}
789+
assert new_group.attrs == expected_attrs
784790

785791

786792
@pytest.mark.parametrize("name", ["a", "/a"])
@@ -2378,3 +2384,83 @@ def test_open_array_as_group():
23782384
z = zarr.create_array(shape=(40, 50), chunks=(10, 10), dtype="f8", store={})
23792385
with pytest.raises(ContainsArrayError):
23802386
zarr.open_group(z.store)
2387+
2388+
2389+
# ---------------------------------------------------------------------------
2390+
# Regression tests for B11-B15 (bugs identified in 2026-06)
2391+
# ---------------------------------------------------------------------------
2392+
2393+
2394+
@pytest.mark.filterwarnings("ignore:Consolidated metadata")
2395+
async def test_iter_members_deep_use_consolidated_flag_propagates(
2396+
store: Store, zarr_format: ZarrFormat
2397+
) -> None:
2398+
"""B11 — _iter_members_deep must pass use_consolidated_for_children on recursion.
2399+
2400+
Build root/a/b/c, consolidate at a/b so that a/b carries stale
2401+
consolidated metadata, then iterate root with
2402+
use_consolidated_for_children=False and assert that the grandchild
2403+
group (a/b) has consolidated_metadata=None during iteration — meaning
2404+
the fresh-read path was taken rather than the cached one.
2405+
"""
2406+
root = zarr.group(store=store, zarr_format=zarr_format)
2407+
a = root.create_group("a")
2408+
ab = a.create_group("b")
2409+
ab.create_group("c")
2410+
2411+
# Consolidate so that a/b.metadata.consolidated_metadata is populated.
2412+
if isinstance(store, ZipStore):
2413+
with pytest.warns(UserWarning, match="Duplicate name"):
2414+
zarr.consolidate_metadata(store=store, zarr_format=zarr_format)
2415+
else:
2416+
zarr.consolidate_metadata(store=store, zarr_format=zarr_format)
2417+
2418+
root = zarr.open_consolidated(store=store, zarr_format=zarr_format)
2419+
2420+
# Collect members yielded by the deep iterator without using consolidated
2421+
# metadata for children.
2422+
members: dict[str, Group | Array] = dict(
2423+
root.members(max_depth=None, use_consolidated_for_children=False)
2424+
)
2425+
2426+
# The "a/b" entry must have been stripped of its consolidated_metadata.
2427+
ab_node = members.get("a/b")
2428+
assert ab_node is not None
2429+
assert isinstance(ab_node, Group)
2430+
assert ab_node.metadata.consolidated_metadata is None, (
2431+
"consolidated_metadata must be cleared when use_consolidated_for_children=False "
2432+
"propagates to recursive calls (B11)"
2433+
)
2434+
2435+
2436+
async def test_update_attributes_async_merges(store: Store, zarr_format: ZarrFormat) -> None:
2437+
"""B12 — Group.update_attributes_async must merge, not overwrite.
2438+
2439+
Both the synchronous update_attributes and the async variant must
2440+
preserve keys that are not in the new_attributes dict.
2441+
"""
2442+
initial_attrs: dict[str, Any] = {"existing_key": 42}
2443+
grp = zarr.group(store=store, zarr_format=zarr_format, attributes=initial_attrs)
2444+
2445+
# Synchronous path already merged; verify for completeness.
2446+
if isinstance(store, ZipStore):
2447+
with pytest.warns(UserWarning, match="Duplicate name"):
2448+
grp_after_sync = grp.update_attributes({"sync_key": 1})
2449+
else:
2450+
grp_after_sync = grp.update_attributes({"sync_key": 1})
2451+
assert grp_after_sync.attrs["existing_key"] == 42
2452+
assert grp_after_sync.attrs["sync_key"] == 1
2453+
2454+
# Async path — this was the buggy path (B12).
2455+
if isinstance(store, ZipStore):
2456+
with pytest.warns(UserWarning, match="Duplicate name"):
2457+
grp_after_async = await grp_after_sync.update_attributes_async({"async_key": 2})
2458+
else:
2459+
grp_after_async = await grp_after_sync.update_attributes_async({"async_key": 2})
2460+
assert grp_after_async.attrs["existing_key"] == 42, (
2461+
"existing_key must survive update_attributes_async (B12)"
2462+
)
2463+
assert grp_after_async.attrs["sync_key"] == 1, (
2464+
"sync_key must survive update_attributes_async (B12)"
2465+
)
2466+
assert grp_after_async.attrs["async_key"] == 2

‎tests/test_sync.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,3 +163,33 @@ def test_cleanup_resources_idempotent() -> None:
163163
_get_executor() # trigger resource creation (iothread, loop, thread-pool)
164164
cleanup_resources()
165165
cleanup_resources()
166+
167+
168+
def test_sync_timeout_cancels_coroutine() -> None:
169+
"""B15 — sync() must cancel the underlying future on timeout.
170+
171+
Before the fix, the coroutine kept running on the IO thread after
172+
TimeoutError was raised, because future.cancel() was never called.
173+
We verify cancellation by having the coroutine set a flag in its
174+
``finally`` block, which we observe shortly after the timeout.
175+
"""
176+
import threading
177+
178+
cancelled_flag: list[bool] = [False]
179+
lock = threading.Event()
180+
181+
async def slow_coro() -> None:
182+
try:
183+
await asyncio.sleep(10)
184+
except asyncio.CancelledError:
185+
cancelled_flag[0] = True
186+
raise
187+
finally:
188+
lock.set()
189+
190+
with pytest.raises(TimeoutError):
191+
sync(slow_coro(), timeout=0.02)
192+
193+
# Give the IO loop a moment to process the cancellation.
194+
lock.wait(timeout=1.0)
195+
assert cancelled_flag[0], "coroutine must be cancelled after sync() timeout (B15)"

0 commit comments

Comments
 (0)