|
11 | 11 | from zarr.core.buffer import Buffer, cpu, default_buffer_prototype, gpu |
12 | 12 | from zarr.errors import ZarrUserWarning |
13 | 13 | from zarr.storage import GpuMemoryStore, ManagedMemoryStore, MemoryStore |
| 14 | +from zarr.storage._utils import _join_paths |
14 | 15 | from zarr.testing.store import StoreTests |
15 | 16 | from zarr.testing.utils import gpu_test |
16 | 17 |
|
@@ -233,31 +234,48 @@ def test_from_dict(self) -> None: |
233 | 234 | for v in result._store_dict.values(): |
234 | 235 | assert type(v) is gpu.Buffer |
235 | 236 |
|
| 237 | + def test_set_sync_converts_to_gpu_buffer(self, store: GpuMemoryStore) -> None: |
| 238 | + """`set_sync` must convert its value to a `gpu.Buffer`, mirroring `set`. |
| 239 | +
|
| 240 | + `GpuMemoryStore`'s invariant is that every stored value is a |
| 241 | + `gpu.Buffer`. Without this override, the inherited `MemoryStore.set_sync` |
| 242 | + would store the CPU buffer it was given as-is, breaking that invariant |
| 243 | + for whichever code path (e.g. the fused pipeline) uses the sync API. |
| 244 | + """ |
| 245 | + cpu_value = cpu.Buffer.from_bytes(b"aaaa") |
| 246 | + msg = "Creating a zarr.buffer.gpu.Buffer with an array that does not support the __cuda_array_interface__ for zero-copy transfers, falling back to slow copy based path" |
| 247 | + with pytest.warns(ZarrUserWarning, match=msg): |
| 248 | + store.set_sync("k", cpu_value) |
| 249 | + assert type(store._store_dict["k"]) is gpu.Buffer |
| 250 | + |
236 | 251 |
|
237 | 252 | class TestManagedMemoryStore(StoreTests[ManagedMemoryStore, cpu.Buffer]): |
238 | 253 | store_cls = ManagedMemoryStore |
239 | 254 | buffer_cls = cpu.Buffer |
240 | 255 |
|
241 | 256 | async def set(self, store: ManagedMemoryStore, key: str, value: Buffer) -> None: |
242 | | - store._store_dict[key] = value |
| 257 | + store._store_dict[_join_paths([store.path, key])] = value |
243 | 258 |
|
244 | 259 | async def get(self, store: ManagedMemoryStore, key: str) -> Buffer: |
245 | | - return store._store_dict[key] |
| 260 | + return store._store_dict[_join_paths([store.path, key])] |
246 | 261 |
|
247 | 262 | @pytest.fixture |
248 | 263 | def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, Any]: |
249 | 264 | # Use a unique name per test to avoid sharing state between tests |
250 | 265 | # but ensure the name is deterministic for equality tests |
251 | 266 | # Replace '/' with '-' since store names cannot contain '/' |
| 267 | + # A non-empty path exercises prefix handling; a store with an |
| 268 | + # unprefixed key in its backing dict would pass these tests |
| 269 | + # vacuously with path="". |
252 | 270 | sanitized_name = request.node.name.replace("/", "-") |
253 | | - return {"name": f"test-{sanitized_name}"} |
| 271 | + return {"name": f"test-{sanitized_name}", "path": "prefix"} |
254 | 272 |
|
255 | 273 | @pytest.fixture |
256 | 274 | async def store(self, store_kwargs: dict[str, Any]) -> ManagedMemoryStore: |
257 | 275 | return self.store_cls(**store_kwargs) |
258 | 276 |
|
259 | 277 | def test_store_repr(self, store: ManagedMemoryStore) -> None: |
260 | | - assert str(store) == f"memory://{store.name}" |
| 278 | + assert str(store) == _join_paths([f"memory://{store.name}", store.path]) |
261 | 279 |
|
262 | 280 | async def test_serializable_store(self, store: ManagedMemoryStore) -> None: |
263 | 281 | """ |
@@ -383,7 +401,10 @@ def test_from_url(self, store: ManagedMemoryStore) -> None: |
383 | 401 |
|
384 | 402 | def test_from_url_with_path(self, store: ManagedMemoryStore) -> None: |
385 | 403 | """Test that from_url extracts path component from URL.""" |
386 | | - url = f"{store}/some/path" |
| 404 | + # Reconnect to the fixture's dict via its name, but with an empty |
| 405 | + # path, so appending "/some/path" below yields exactly that path. |
| 406 | + base = ManagedMemoryStore(name=store.name) |
| 407 | + url = f"{base}/some/path" |
387 | 408 | store2 = ManagedMemoryStore.from_url(url) |
388 | 409 | assert store2._store_dict is store._store_dict |
389 | 410 | assert store2.path == "some/path" |
@@ -512,3 +533,48 @@ def test_garbage_collection(self) -> None: |
512 | 533 | # URL should no longer resolve |
513 | 534 | with pytest.raises(ValueError, match="garbage collected"): |
514 | 535 | ManagedMemoryStore.from_url(url) |
| 536 | + |
| 537 | + def test_sync_methods_respect_path_prefix(self) -> None: |
| 538 | + """`get_sync`/`set_sync`/`delete_sync` must prefix keys with `self.path`, |
| 539 | + exactly like the async `get`/`set`/`delete` methods. |
| 540 | +
|
| 541 | + `ManagedMemoryStore` used to inherit these from `MemoryStore`, which |
| 542 | + writes/reads the raw key. Two stores sharing a dict with different |
| 543 | + `path` values would then cross-talk through the sync API. |
| 544 | + """ |
| 545 | + store = ManagedMemoryStore(name="sync-prefix-test", path="subdir") |
| 546 | + data_buf = self.buffer_cls.from_bytes(b"value") |
| 547 | + |
| 548 | + store.set_sync("key", data_buf) |
| 549 | + assert "subdir/key" in store._store_dict |
| 550 | + assert "key" not in store._store_dict |
| 551 | + |
| 552 | + result = store.get_sync("key") |
| 553 | + assert result is not None |
| 554 | + assert result.to_bytes() == b"value" |
| 555 | + |
| 556 | + store.delete_sync("key") |
| 557 | + assert "subdir/key" not in store._store_dict |
| 558 | + |
| 559 | + def test_fused_pipeline_respects_path_prefix(self) -> None: |
| 560 | + """End-to-end regression: the fused pipeline's sync store fast path must |
| 561 | + write chunks under the store's path prefix. |
| 562 | +
|
| 563 | + `FusedCodecPipeline` uses `set_sync`/`get_sync` when a store implements |
| 564 | + the sync protocols. If those methods skip the prefix that the async |
| 565 | + methods apply, chunk data lands outside `self.path` and a fresh handle |
| 566 | + re-reading through the prefix silently sees fill values instead. |
| 567 | + """ |
| 568 | + with zarr.config.set( |
| 569 | + {"codec_pipeline.path": "zarr.core.codec_pipeline.FusedCodecPipeline"} |
| 570 | + ): |
| 571 | + store = ManagedMemoryStore(name="fused-prefix-test", path="subdir") |
| 572 | + arr = zarr.create_array(store, shape=(4,), chunks=(4,), dtype="uint8", zarr_format=3) |
| 573 | + arr[:] = np.arange(4, dtype="uint8") |
| 574 | + |
| 575 | + bad_keys = [k for k in store._store_dict if not k.startswith("subdir/")] |
| 576 | + assert bad_keys == [], f"keys written outside the store's path prefix: {bad_keys}" |
| 577 | + |
| 578 | + store2 = ManagedMemoryStore.from_url("memory://fused-prefix-test/subdir") |
| 579 | + arr2 = zarr.open_array(store2, mode="r") |
| 580 | + np.testing.assert_array_equal(arr2[:], np.arange(4, dtype="uint8")) |
0 commit comments