Skip to content

Commit ecef745

Browse files
committed
Address list_dir prefix marker review
Signed-off-by: ABrandes <[email protected]>
1 parent 00c7c4e commit ecef745

6 files changed

Lines changed: 22 additions & 19 deletions

File tree

‎src/zarr/core/array.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3994,6 +3994,7 @@ async def _shards_initialized(
39943994
]
39953995
store_contents_relative = [
39963996
_relativize_path(path=key, prefix=array.store_path.path) for key in store_contents
3997+
if not array.store_path.path or key != array.store_path.path
39973998
]
39983999
return tuple(
39994000
chunk_key for chunk_key in array._iter_shard_keys() if chunk_key in store_contents_relative

‎src/zarr/storage/_memory.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]:
219219
keys_unique = {
220220
key.removeprefix(f"{prefix}/").split("/")[0]
221221
for key in self._store_dict
222-
if key.startswith(f"{prefix}/") and key != prefix
222+
if key.startswith(f"{prefix}/") and key not in {prefix, f"{prefix}/"}
223223
}
224224

225225
for key in keys_unique:

‎src/zarr/storage/_obstore.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ async def _transform_list_dir(
267267
for path in chain(
268268
list_result["common_prefixes"], map(itemgetter("path"), list_result["objects"])
269269
):
270-
if prefix and path in {prefix, f"{prefix}/"}:
270+
if prefix and path == prefix:
271271
continue
272272
relpath = _relativize_path(path=path, prefix=prefix)
273273
if relpath:

‎src/zarr/testing/store.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,21 @@ async def test_list_dir(self, store: S) -> None:
554554
keys_observed = await _collect_aiterator(store.list_dir(f"{root}/"))
555555
assert sorted(keys_expected) == sorted(keys_observed)
556556

557+
async def test_list_dir_ignores_prefix_marker(self, store: S) -> None:
558+
data = self.buffer_cls.from_bytes(b"")
559+
root = "foo"
560+
try:
561+
await self.set(store, f"{root}/", data)
562+
await self.set(store, f"{root}/child", data)
563+
except OSError as exc:
564+
pytest.skip(f"store cannot represent directory marker objects: {exc}")
565+
566+
keys_observed = await _collect_aiterator(store.list_dir(root))
567+
assert keys_observed == ("child",)
568+
569+
keys_observed = await _collect_aiterator(store.list_dir(f"{root}/"))
570+
assert keys_observed == ("child",)
571+
557572
async def test_set_if_not_exists(self, store: S) -> None:
558573
key = "k"
559574
data_buf = self.buffer_cls.from_bytes(b"0000")

‎tests/test_array.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
default_serializer_v3,
4646
)
4747
from zarr.core.array_spec import ArrayConfig, ArrayConfigParams
48-
from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, default_buffer_prototype
48+
from zarr.core.buffer import NDArrayLike, NDArrayLikeOrScalar, cpu, default_buffer_prototype
4949
from zarr.core.chunk_grids import (
5050
SHARDED_INNER_CHUNK_MAX_BYTES,
5151
guess_chunks,
@@ -447,6 +447,8 @@ async def test_chunks_initialized(
447447
arr = zarr.create_array(
448448
store, name=path, shape=shape, shards=shard_shape, chunks=chunk_shape, dtype="i1"
449449
)
450+
if path:
451+
await store.set(path, cpu.Buffer.from_bytes(b""))
450452

451453
chunks_accumulated = tuple(
452454
accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_shard_keys()))

‎tests/test_store/test_object.py‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# ruff: noqa: E402
22
from pathlib import Path
3-
from typing import Any, TypedDict
3+
from typing import TypedDict
44

55
import pytest
66

@@ -13,7 +13,6 @@
1313

1414
from zarr.core.buffer import Buffer, cpu
1515
from zarr.storage import ObjectStore
16-
from zarr.storage._obstore import _transform_list_dir
1716
from zarr.testing.stateful import ZarrHierarchyStateMachine
1817
from zarr.testing.store import StoreTests
1918

@@ -97,20 +96,6 @@ async def test_store_getsize_prefix(self, store: ObjectStore[LocalStore]) -> Non
9796
total_size = await store.getsize_prefix("c")
9897
assert total_size == len(buf) * 2
9998

100-
101-
@pytest.mark.parametrize("prefix", ["g", "g/"])
102-
async def test_transform_list_dir_skips_prefix_marker(prefix: str) -> None:
103-
async def list_result() -> Any:
104-
return {
105-
"common_prefixes": ["g/child/"],
106-
"objects": [{"path": "g"}, {"path": "g/"}, {"path": "g/item"}],
107-
}
108-
109-
result = [path async for path in _transform_list_dir(list_result(), prefix)]
110-
111-
assert result == ["child/", "item"]
112-
113-
11499
@pytest.mark.slow_hypothesis
115100
def test_zarr_hierarchy() -> None:
116101
sync_store = ObjectStore(MemoryStore())

0 commit comments

Comments
 (0)