Skip to content

Commit a88f889

Browse files
authored
fix: make ManagedMemoryStore/GpuMemoryStore sync methods parity-safe (#4204)
ManagedMemoryStore inherited get_sync/set_sync/delete_sync from MemoryStore, which use the raw key, while every async method prefixed keys with self.path. Any code taking the sync fast path (e.g. FusedCodecPipeline) wrote/read chunks outside the store's path prefix, so a fresh handle re-reading through the prefix silently got fill values. Override the three sync methods to prefix like their async counterparts. GpuMemoryStore.set_sync gets the same treatment: it now converts its value to a gpu.Buffer like set does, preserving the store's all-values-are-gpu invariant for the sync API. Also fix ManagedMemoryStore.get_partial_values, which applied self.path twice whenever path was non-empty (it pre-prefixed keys, then delegated to MemoryStore.get_partial_values, which itself dispatches through the already-overridden self.get) -- this made it return None for every key. Discovered via the strengthened test fixture below. Add sync/async parity laws to the shared StoreTests suite so every store subclass exercises this invariant: set through one API and read through the other (including byte_range variants), and confirm delete_sync is visible to async get. These are the tests that would have caught the ManagedMemoryStore bug. TestManagedMemoryStore's raw set/get test helpers now respect self.path, and store_kwargs uses a non-empty path, so prefix handling is actually exercised instead of passing vacuously. Add an end-to-end regression with FusedCodecPipeline writing to a ManagedMemoryStore(path=...) sharing a dict with a fresh handle. Add a LocalStore.delete_sync directory-branch test. Assisted-by: ClaudeCode:claude-sonnet-5
1 parent 020e5c3 commit a88f889

5 files changed

Lines changed: 202 additions & 13 deletions

File tree

‎changes/4204.bugfix.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
`ManagedMemoryStore.get_sync`/`set_sync`/`delete_sync` now apply the store's
2+
`path` prefix, matching the async `get`/`set`/`delete` methods. Previously the
3+
sync methods were inherited unchanged from `MemoryStore` and used the raw key,
4+
so code that takes the sync fast path (e.g. `FusedCodecPipeline`) would read
5+
and write chunks outside the store's `path` prefix, silently returning fill
6+
values when the data was re-read through a fresh handle. `GpuMemoryStore.set_sync`
7+
now converts its value to a `gpu.Buffer`, matching `set`, so writes through the
8+
sync API keep the store's all-values-are-GPU invariant. Also fixed
9+
`ManagedMemoryStore.get_partial_values` applying its `path` prefix twice
10+
whenever `path` is non-empty, which made it always return `None` for every
11+
requested key.
12+
13+
The shared store test suite (`zarr.testing.store.StoreTests`) gained
14+
sync/async parity checks — comparing sync and async observations of the same
15+
key on the same store instance, including with a `byte_range` — so every
16+
store subclass now exercises this invariant.

‎src/zarr/storage/_memory.py‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,19 @@ async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None
314314
gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value)
315315
await super().set(key, gpu_value, byte_range=byte_range)
316316

317+
def set_sync(self, key: str, value: Buffer) -> None:
318+
# docstring inherited
319+
self._check_writable()
320+
assert isinstance(key, str)
321+
if not isinstance(value, Buffer):
322+
raise TypeError(
323+
f"GpuMemoryStore.set(): `value` must be a Buffer instance. Got an instance of {type(value)} instead."
324+
)
325+
# Convert to gpu.Buffer, mirroring `set` above: every value in this store's
326+
# backing dict must be a gpu.Buffer, regardless of which API wrote it.
327+
gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value)
328+
super().set_sync(key, gpu_value)
329+
317330

318331
# -----------------------------------------------------------------------------
319332
# ManagedMemoryStore and its registry
@@ -572,25 +585,40 @@ def from_url(cls, url: str, *, read_only: bool = False) -> ManagedMemoryStore:
572585

573586
# Override MemoryStore methods to use path prefix and check process
574587

575-
async def get(
588+
def get_sync(
576589
self,
577590
key: str,
591+
*,
578592
prototype: BufferPrototype | None = None,
579593
byte_range: ByteRequest | None = None,
580594
) -> Buffer | None:
581595
# docstring inherited
582-
return await super().get(
596+
return super().get_sync(
583597
_join_paths([self.path, key]), prototype=prototype, byte_range=byte_range
584598
)
585599

586-
async def get_partial_values(
600+
def set_sync(self, key: str, value: Buffer) -> None:
601+
# docstring inherited
602+
super().set_sync(_join_paths([self.path, key]), value)
603+
604+
def delete_sync(self, key: str) -> None:
605+
# docstring inherited
606+
super().delete_sync(_join_paths([self.path, key]))
607+
608+
async def get(
587609
self,
588-
prototype: BufferPrototype,
589-
key_ranges: Iterable[tuple[str, ByteRequest | None]],
590-
) -> list[Buffer | None]:
610+
key: str,
611+
prototype: BufferPrototype | None = None,
612+
byte_range: ByteRequest | None = None,
613+
) -> Buffer | None:
591614
# docstring inherited
592-
key_ranges = [(_join_paths([self.path, key]), byte_range) for key, byte_range in key_ranges]
593-
return await super().get_partial_values(prototype, key_ranges)
615+
return await super().get(
616+
_join_paths([self.path, key]), prototype=prototype, byte_range=byte_range
617+
)
618+
619+
# get_partial_values is intentionally NOT overridden here: MemoryStore.get_partial_values
620+
# dispatches per-key through `self.get`, which already resolves to the override above.
621+
# Re-prefixing the keys here as well would apply `self.path` twice.
594622

595623
async def exists(self, key: str) -> bool:
596624
# docstring inherited

‎src/zarr/testing/store.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,71 @@ def test_delete_sync_missing(self, store: S) -> None:
618618
# should not raise
619619
deleter.delete_sync("nonexistent_sync")
620620

621+
# -------------------------------------------------------------------
622+
# Sync/async parity laws
623+
# -------------------------------------------------------------------
624+
# A store's sync and async methods must observe the same key the same
625+
# way. This is stronger than the individual test_get_sync/test_set_sync/
626+
# test_delete_sync tests above: those write and read back through the
627+
# *same* API (sync-only or, via `self.set`/`self.get`, bypassing the
628+
# store entirely), so a sync method that skips logic the async method
629+
# applies (e.g. a path prefix) can still pass them. These laws write
630+
# through one API and observe through the other.
631+
632+
@pytest.mark.parametrize("direction", ["set_async_get_sync", "set_sync_get_async"])
633+
async def test_sync_async_set_get_parity(self, store: S, direction: str) -> None:
634+
setter = self._require_set_sync(store)
635+
getter = self._require_get_sync(store)
636+
data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04")
637+
key = "parity_set_get"
638+
if direction == "set_async_get_sync":
639+
await store.set(key, data_buf)
640+
result = getter.get_sync(key)
641+
else:
642+
setter.set_sync(key, data_buf)
643+
result = await store.get(key, prototype=default_buffer_prototype())
644+
assert result is not None
645+
assert_bytes_equal(result, data_buf)
646+
647+
async def test_delete_sync_visible_to_async_get(self, store: S) -> None:
648+
deleter = self._require_delete_sync(store)
649+
if not store.supports_deletes:
650+
pytest.skip("store does not support deletes")
651+
data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04")
652+
key = "parity_delete"
653+
await store.set(key, data_buf)
654+
deleter.delete_sync(key)
655+
result = await store.get(key, prototype=default_buffer_prototype())
656+
assert result is None
657+
658+
@pytest.mark.parametrize(
659+
"byte_range",
660+
[
661+
None,
662+
RangeByteRequest(1, 4),
663+
OffsetByteRequest(1),
664+
SuffixByteRequest(1),
665+
RangeByteRequest(10, 20),
666+
],
667+
ids=["none", "range", "offset", "suffix", "range-past-eof"],
668+
)
669+
async def test_get_sync_byte_range_parity(
670+
self, store: S, byte_range: ByteRequest | None
671+
) -> None:
672+
getter = self._require_get_sync(store)
673+
data_buf = self.buffer_cls.from_bytes(b"\x01\x02\x03\x04")
674+
key = "parity_byte_range"
675+
await store.set(key, data_buf)
676+
sync_result = getter.get_sync(key, byte_range=byte_range)
677+
async_result = await store.get(
678+
key, prototype=default_buffer_prototype(), byte_range=byte_range
679+
)
680+
if async_result is None:
681+
assert sync_result is None
682+
else:
683+
assert sync_result is not None
684+
assert_bytes_equal(sync_result, async_result)
685+
621686

622687
class LatencyStore(WrapperStore[Store]):
623688
"""

‎tests/test_store/test_local.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,20 @@ async def test_empty_with_empty_subdir(self, store: LocalStore) -> None:
4646
(store.root / "foo/bar").mkdir(parents=True)
4747
assert await store.is_empty("")
4848

49+
def test_delete_sync_directory(self, store: LocalStore) -> None:
50+
"""`delete_sync` on a key that is a directory must remove the whole tree.
51+
52+
Mirrors the async `delete_dir` behavior: deleting `"foo"` where
53+
`"foo"` is a directory containing further nested paths should remove
54+
everything under it, not just fail or delete a single file.
55+
"""
56+
(store.root / "foo" / "bar").mkdir(parents=True)
57+
(store.root / "foo" / "bar" / "baz").write_bytes(b"data")
58+
59+
store.delete_sync("foo")
60+
61+
assert not (store.root / "foo").exists()
62+
4963
def test_creates_new_directory(self, tmp_path: pathlib.Path) -> None:
5064
target = tmp_path.joinpath("a", "b", "c")
5165
assert not target.exists()

‎tests/test_store/test_memory.py‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from zarr.core.buffer import Buffer, cpu, default_buffer_prototype, gpu
1212
from zarr.errors import ZarrUserWarning
1313
from zarr.storage import GpuMemoryStore, ManagedMemoryStore, MemoryStore
14+
from zarr.storage._utils import _join_paths
1415
from zarr.testing.store import StoreTests
1516
from zarr.testing.utils import gpu_test
1617

@@ -233,31 +234,48 @@ def test_from_dict(self) -> None:
233234
for v in result._store_dict.values():
234235
assert type(v) is gpu.Buffer
235236

237+
def test_set_sync_converts_to_gpu_buffer(self, store: GpuMemoryStore) -> None:
238+
"""`set_sync` must convert its value to a `gpu.Buffer`, mirroring `set`.
239+
240+
`GpuMemoryStore`'s invariant is that every stored value is a
241+
`gpu.Buffer`. Without this override, the inherited `MemoryStore.set_sync`
242+
would store the CPU buffer it was given as-is, breaking that invariant
243+
for whichever code path (e.g. the fused pipeline) uses the sync API.
244+
"""
245+
cpu_value = cpu.Buffer.from_bytes(b"aaaa")
246+
msg = "Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__ for zero-copy transfers, falling back to slow copy based path"
247+
with pytest.warns(ZarrUserWarning, match=msg):
248+
store.set_sync("k", cpu_value)
249+
assert type(store._store_dict["k"]) is gpu.Buffer
250+
236251

237252
class TestManagedMemoryStore(StoreTests[ManagedMemoryStore, cpu.Buffer]):
238253
store_cls = ManagedMemoryStore
239254
buffer_cls = cpu.Buffer
240255

241256
async def set(self, store: ManagedMemoryStore, key: str, value: Buffer) -> None:
242-
store._store_dict[key] = value
257+
store._store_dict[_join_paths([store.path, key])] = value
243258

244259
async def get(self, store: ManagedMemoryStore, key: str) -> Buffer:
245-
return store._store_dict[key]
260+
return store._store_dict[_join_paths([store.path, key])]
246261

247262
@pytest.fixture
248263
def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]:
249264
# Use a unique name per test to avoid sharing state between tests
250265
# but ensure the name is deterministic for equality tests
251266
# Replace '/' with '-' since store names cannot contain '/'
267+
# A non-empty path exercises prefix handling; a store with an
268+
# unprefixed key in its backing dict would pass these tests
269+
# vacuously with path="".
252270
sanitized_name = request.node.name.replace("/", "-")
253-
return {"name": f"test-{sanitized_name}"}
271+
return {"name": f"test-{sanitized_name}", "path": "prefix"}
254272

255273
@pytest.fixture
256274
async def store(self, store_kwargs: dict[str, Any]) -> ManagedMemoryStore:
257275
return self.store_cls(**store_kwargs)
258276

259277
def test_store_repr(self, store: ManagedMemoryStore) -> None:
260-
assert str(store) == f"memory://{store.name}"
278+
assert str(store) == _join_paths([f"memory://{store.name}", store.path])
261279

262280
async def test_serializable_store(self, store: ManagedMemoryStore) -> None:
263281
"""
@@ -383,7 +401,10 @@ def test_from_url(self, store: ManagedMemoryStore) -> None:
383401

384402
def test_from_url_with_path(self, store: ManagedMemoryStore) -> None:
385403
"""Test that from_url extracts path component from URL."""
386-
url = f"{store}/some/path"
404+
# Reconnect to the fixture's dict via its name, but with an empty
405+
# path, so appending "/some/path" below yields exactly that path.
406+
base = ManagedMemoryStore(name=store.name)
407+
url = f"{base}/some/path"
387408
store2 = ManagedMemoryStore.from_url(url)
388409
assert store2._store_dict is store._store_dict
389410
assert store2.path == "some/path"
@@ -512,3 +533,48 @@ def test_garbage_collection(self) -> None:
512533
# URL should no longer resolve
513534
with pytest.raises(ValueError, match="garbage collected"):
514535
ManagedMemoryStore.from_url(url)
536+
537+
def test_sync_methods_respect_path_prefix(self) -> None:
538+
"""`get_sync`/`set_sync`/`delete_sync` must prefix keys with `self.path`,
539+
exactly like the async `get`/`set`/`delete` methods.
540+
541+
`ManagedMemoryStore` used to inherit these from `MemoryStore`, which
542+
writes/reads the raw key. Two stores sharing a dict with different
543+
`path` values would then cross-talk through the sync API.
544+
"""
545+
store = ManagedMemoryStore(name="sync-prefix-test", path="subdir")
546+
data_buf = self.buffer_cls.from_bytes(b"value")
547+
548+
store.set_sync("key", data_buf)
549+
assert "subdir/key" in store._store_dict
550+
assert "key" not in store._store_dict
551+
552+
result = store.get_sync("key")
553+
assert result is not None
554+
assert result.to_bytes() == b"value"
555+
556+
store.delete_sync("key")
557+
assert "subdir/key" not in store._store_dict
558+
559+
def test_fused_pipeline_respects_path_prefix(self) -> None:
560+
"""End-to-end regression: the fused pipeline's sync store fast path must
561+
write chunks under the store's path prefix.
562+
563+
`FusedCodecPipeline` uses `set_sync`/`get_sync` when a store implements
564+
the sync protocols. If those methods skip the prefix that the async
565+
methods apply, chunk data lands outside `self.path` and a fresh handle
566+
re-reading through the prefix silently sees fill values instead.
567+
"""
568+
with zarr.config.set(
569+
{"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"}
570+
):
571+
store = ManagedMemoryStore(name="fused-prefix-test", path="subdir")
572+
arr = zarr.create_array(store, shape=(4,), chunks=(4,), dtype="uint8", zarr_format=3)
573+
arr[:] = np.arange(4, dtype="uint8")
574+
575+
bad_keys = [k for k in store._store_dict if not k.startswith("subdir/")]
576+
assert bad_keys == [], f"keys written outside the store's path prefix: {bad_keys}"
577+
578+
store2 = ManagedMemoryStore.from_url("memory://fused-prefix-test/subdir")
579+
arr2 = zarr.open_array(store2, mode="r")
580+
np.testing.assert_array_equal(arr2[:], np.arange(4, dtype="uint8"))

0 commit comments

Comments
 (0)