Skip to content

Commit 34b7c3d

Browse files
authored
fix(storage): yield only direct children from FsspecStore.list_dir (zarr-developers#4402)
* fix(storage): yield only direct children from FsspecStore.list_dir An HTTP listing is scraped from the links on an HTML index page. A link to the site root ("/") becomes a self-link that list_dir reduced to "", so a group listed itself as a member named "". In-page anchors ("#usage") became keys whose metadata fetch returned the HTML page, and directories carried a trailing "/". list_dir now skips http(s) entries with a fragment or query, strips the trailing "/", drops empty or nested names, and yields each name once. Closes zarr-developers#3575 Assisted-by: ClaudeCode:claude-opus-5-5 * docs: add changelog fragment for zarr-developers#4402 Assisted-by: ClaudeCode:claude-opus-5-5 * fix(storage): scope HTTP link filtering to the child name in list_dir The "#"/"?" check ran on the whole URL, so a store rooted at a URL containing either character would list nothing. It now runs on the name left after stripping the prefix. HTTP names are also normalized with posixpath.normpath, so "./a" lists as "a" and "./" no longer lists as ".". Assisted-by: ClaudeCode:claude-opus-5-5
1 parent eb2ce59 commit 34b7c3d

3 files changed

Lines changed: 75 additions & 2 deletions

File tree

‎changes/4402.bugfix.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Fixed `FsspecStore.list_dir` over HTTP returning entries that are not children
2+
of the listed directory. A link to the site root on the index page made a group
3+
list itself as a member named `""`, in-page anchor links such as `#usage` were
4+
returned as keys, and directory names kept a trailing `/`. `list_dir` now
5+
returns each direct child once, by its bare name.

‎src/zarr/storage/_fsspec.py‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from __future__ import annotations
22

3+
import posixpath
34
import warnings
45
from contextlib import suppress
56
from typing import TYPE_CHECKING, Any
@@ -431,8 +432,23 @@ async def list_dir(self, prefix: str) -> AsyncIterator[str]:
431432
allfiles = await self.fs._ls(prefix, detail=False)
432433
except FileNotFoundError:
433434
return
434-
for onefile in (a.replace(f"{prefix}/", "") for a in allfiles):
435-
yield onefile.removeprefix(self.path).removeprefix("/")
435+
seen: set[str] = set()
436+
for onefile in allfiles:
437+
name = onefile.replace(f"{prefix}/", "").removeprefix(self.path).removeprefix("/")
438+
if onefile.startswith(("http://", "https://")):
439+
# An HTTP listing is scraped from the links on an HTML page. In-page anchors
440+
# and queries name the page itself, not an object, and relative links such
441+
# as "./a" need normalizing.
442+
if "#" in name or "?" in name:
443+
continue
444+
name = posixpath.normpath(name)
445+
# Only direct children, each once: an HTTP listing marks directories with a
446+
# trailing "/", and a link back to the listed directory reduces to "" or ".".
447+
name = name.rstrip("/")
448+
if name in ("", ".", "..") or "/" in name or name in seen:
449+
continue
450+
seen.add(name)
451+
yield name
436452

437453
async def list_prefix(self, prefix: str) -> AsyncIterator[str]:
438454
# docstring inherited

‎tests/test_store/test_fsspec.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
from __future__ import annotations
22

3+
import functools
4+
import http.server
35
import json
46
import re
7+
import threading
58
import warnings
69
from typing import TYPE_CHECKING, Any
710

@@ -561,6 +564,55 @@ def test_open_s3map_raises(endpoint_url: str) -> None:
561564
zarr.open(store=mapper, storage_options={"anon": True}, mode="w", shape=(3, 3))
562565

563566

567+
async def test_list_dir_http_yields_only_children(tmp_path: pathlib.Path) -> None:
568+
"""list_dir over HTTP yields each direct child once, by bare name.
569+
570+
An HTTP listing is scraped from an HTML index page, whose links include the site
571+
root, the parent, in-page anchors, queries, "./" relative links, and directories
572+
with a trailing "/".
573+
Regression test for https://github.com/zarr-developers/zarr-python/issues/3575,
574+
where the site-root link surfaced as a group member named "".
575+
"""
576+
pytest.importorskip("aiohttp")
577+
links = [
578+
"/",
579+
"../",
580+
"./",
581+
"#",
582+
"#usage",
583+
"?sort=name",
584+
"a/",
585+
"a",
586+
"b",
587+
"./d",
588+
".zgroup",
589+
"/group/c/",
590+
]
591+
group = tmp_path / "group"
592+
group.mkdir()
593+
(group / "index.html").write_text(
594+
"<html><body>"
595+
+ "".join(f'<a href="{link}">{link}</a>' for link in links)
596+
+ "</body></html>"
597+
)
598+
599+
class QuietHandler(http.server.SimpleHTTPRequestHandler):
600+
def log_message(self, format: str, *args: Any) -> None:
601+
pass
602+
603+
handler = functools.partial(QuietHandler, directory=str(tmp_path))
604+
with http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) as server:
605+
threading.Thread(target=server.serve_forever, daemon=True).start()
606+
store = FsspecStore.from_url(f"http://127.0.0.1:{server.server_port}/group")
607+
try:
608+
observed = await _collect_aiterator(store.list_dir(""))
609+
finally:
610+
store.close()
611+
server.shutdown()
612+
613+
assert sorted(observed) == [".zgroup", "a", "b", "c", "d"]
614+
615+
564616
async def test_close_does_not_close_filesystem_session() -> None:
565617
"""close() must not touch the filesystem's session.
566618

0 commit comments

Comments
 (0)