Skip to content

Commit 96d786a

Browse files
d-v-bclaude
andcommitted
fix(FsspecStore): propagate fs ownership through with_read_only
Addresses roborev review findings on the filesystem-ownership change: - with_read_only() transferred fs ownership: it built the derived store with _owns_fs=False while sharing the same fs. In the common from_url(...).with_read_only() chain the only owner (the unreferenced source) was GC'd without close(), reintroducing the unclosed-session leak. Ownership now transfers to the surviving store and is cleared on the source to avoid a double-close. Covered by a new test. - close() now logs at debug before suppressing, so a regression in the close path stays observable instead of silently reverting to leaking. - Documented that set_session() lazily creates a session, so closing a store that never did I/O may instantiate one purely to close it (accepted best-effort behavior). Co-Authored-By: Claude Opus 4.7 <[email protected]>
1 parent f4d4517 commit 96d786a

2 files changed

Lines changed: 47 additions & 2 deletions

File tree

‎src/zarr/storage/_fsspec.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import json
44
import warnings
55
from contextlib import suppress
6+
from logging import getLogger
67
from typing import TYPE_CHECKING, Any
78

89
from packaging.version import parse as parse_version
@@ -18,6 +19,8 @@
1819
from zarr.errors import ZarrUserWarning
1920
from zarr.storage._utils import _dereference_path
2021

22+
logger = getLogger(__name__)
23+
2124
if TYPE_CHECKING:
2225
from collections.abc import AsyncIterator, Iterable
2326

@@ -44,6 +47,11 @@ async def _close_fs(fs: AsyncFileSystem) -> None:
4447
"Unclosed client session" ``ResourceWarning``s from aiohttp. For all
4548
other filesystem types the call is a no-op (not every implementation
4649
manages an HTTP session directly).
50+
51+
Note that ``set_session()`` lazily creates a session if none exists yet, so
52+
closing a store that never performed any I/O may instantiate a session
53+
purely to close it. This is accepted best-effort behavior; fsspec does not
54+
expose a stable, cross-implementation way to test for an existing session.
4755
"""
4856
if hasattr(fs, "set_session"):
4957
session = await fs.set_session()
@@ -270,20 +278,32 @@ def from_url(
270278

271279
def with_read_only(self, read_only: bool = False) -> FsspecStore:
272280
# docstring inherited
273-
return type(self)(
281+
new_store = type(self)(
274282
fs=self.fs,
275283
path=self.path,
276284
allowed_exceptions=self.allowed_exceptions,
277285
read_only=read_only,
278286
)
287+
# The derived store shares the same fs. Transfer ownership so the
288+
# surviving store closes it, and clear ours to avoid a double-close.
289+
# Otherwise the common ``from_url(...).with_read_only()`` chain would
290+
# drop the only owner (the unreferenced source) and leak the session.
291+
new_store._owns_fs = self._owns_fs
292+
self._owns_fs = False
293+
return new_store
279294

280295
def close(self) -> None:
281296
# docstring inherited
282297
if self._owns_fs:
283298
from zarr.core.sync import sync as zarr_sync
284299

285-
with suppress(Exception):
300+
# Best-effort: a failure to release the session must not block close(),
301+
# but log it so a genuine regression in the close path stays observable
302+
# rather than silently reverting to the leaking behavior.
303+
try:
286304
zarr_sync(_close_fs(self.fs))
305+
except Exception:
306+
logger.debug("Failed to close owned filesystem %r", self.fs, exc_info=True)
287307
super().close()
288308

289309
async def clear(self) -> None:

‎tests/test_store/test_fsspec.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -634,6 +634,31 @@ def test_from_mapper_owns_wrapped_sync_filesystem(tmp_path: pathlib.Path) -> Non
634634
assert store._owns_fs
635635

636636

637+
@pytest.mark.skipif(
638+
parse_version(fsspec.__version__) < parse_version("2024.12.0"),
639+
reason="No AsyncFileSystemWrapper",
640+
)
641+
def test_with_read_only_transfers_filesystem_ownership(tmp_path: pathlib.Path) -> None:
642+
"""
643+
with_read_only() must transfer fs ownership to the derived store and clear
644+
it on the source, so the surviving store closes the shared fs exactly once.
645+
646+
In the common ``from_url(...).with_read_only()`` chain the source store is
647+
immediately unreferenced; if ownership were not transferred, the only owner
648+
would be garbage-collected without close() and the session would leak.
649+
"""
650+
source = FsspecStore.from_url(f"file://{tmp_path}", storage_options={"auto_mkdir": False})
651+
assert source._owns_fs
652+
653+
derived = source.with_read_only(read_only=True)
654+
655+
# Ownership moved to the survivor; the source no longer owns it (no double-close).
656+
assert derived._owns_fs
657+
assert not source._owns_fs
658+
# The derived store shares the same underlying fs.
659+
assert derived.fs is source.fs
660+
661+
637662
@pytest.mark.parametrize("asynchronous", [True, False])
638663
def test_make_async(asynchronous: bool) -> None:
639664
s3_filesystem = s3fs.S3FileSystem(

0 commit comments

Comments
 (0)