Skip to content

Commit 0a2d81c

Browse files
dschwoererpre-commit-ci[bot]shoyerjsignell
authored
Ensure netcdf4 is locked while closing (pydata#10788)
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Stephan Hoyer <[email protected]> Co-authored-by: Julia Signell <[email protected]>
1 parent 43edfa3 commit 0a2d81c

2 files changed

Lines changed: 31 additions & 12 deletions

File tree

‎xarray/backends/file_manager.py‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -355,13 +355,15 @@ def __init__(
355355
opener: Callable[..., T_File],
356356
*args: Any,
357357
mode: Any = _OMIT_MODE,
358+
lock: Lock | None | Literal[False] = None,
358359
kwargs: Mapping[str, Any] | None = None,
359360
):
360361
kwargs = {} if kwargs is None else dict(kwargs)
361362
self._opener = opener
362363
self._args = args
363364
self._mode = "a" if mode == "w" else mode
364365
self._kwargs = kwargs
366+
self._lock = lock
365367

366368
# Note: No need for locking with PickleableFileManager, because all
367369
# opening of files happens in the constructor.
@@ -391,10 +393,13 @@ def acquire_context(self, needs_lock: bool = True) -> Iterator[T_File]:
391393
yield self._get_unclosed_file()
392394

393395
def close(self, needs_lock: bool = True) -> None:
394-
del needs_lock # unused
395396
if not self._closed:
396397
file = self._get_unclosed_file()
397-
file.close()
398+
if needs_lock and self._lock:
399+
with self._lock:
400+
file.close()
401+
else:
402+
file.close()
398403
self._file = None
399404
# Remove all references to opener arguments, so they can be garbage
400405
# collected.
@@ -417,11 +422,11 @@ def __del__(self) -> None:
417422
def __getstate__(self):
418423
# file is intentionally omitted: we want to open it again
419424
opener = _get_none if self._closed else self._opener
420-
return (opener, self._args, self._mode, self._kwargs)
425+
return (opener, self._args, self._mode, self._lock, self._kwargs)
421426

422427
def __setstate__(self, state) -> None:
423-
opener, args, mode, kwargs = state
424-
self.__init__(opener, *args, mode=mode, kwargs=kwargs) # type: ignore[misc]
428+
opener, args, mode, lock, kwargs = state
429+
self.__init__(opener, *args, mode=mode, lock=lock, kwargs=kwargs) # type: ignore[misc]
425430

426431
def __repr__(self) -> str:
427432
if self._closed:
@@ -448,9 +453,16 @@ def _remove_del_methods():
448453
class DummyFileManager(FileManager[T_File]):
449454
"""FileManager that simply wraps an open file in the FileManager interface."""
450455

451-
def __init__(self, value: T_File, *, close: Callable[[], None] | None = None):
456+
def __init__(
457+
self,
458+
value: T_File,
459+
*,
460+
close: Callable[[], None] | None = None,
461+
lock: Lock | None | Literal[False] = None,
462+
):
452463
if close is None:
453464
close = value.close
465+
self._lock = lock
454466
self._value = value
455467
self._close = close
456468

@@ -464,5 +476,8 @@ def acquire_context(self, needs_lock: bool = True) -> Iterator[T_File]:
464476
yield self._value
465477

466478
def close(self, needs_lock: bool = True) -> None:
467-
del needs_lock # unused
468-
self._close()
479+
if needs_lock and self._lock:
480+
with self._lock:
481+
self._close()
482+
else:
483+
self._close()

‎xarray/backends/netCDF4_.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ def __init__(
421421
"argument is provided"
422422
)
423423
root = manager
424-
manager = DummyFileManager(root)
424+
manager = DummyFileManager(root, lock=NETCDF4_PYTHON_LOCK)
425425

426426
self._manager = manager
427427
self._group = group
@@ -510,17 +510,21 @@ def open(
510510
"<xarray-in-memory-write>", mode=mode, memory=memory, **kwargs
511511
)
512512
close = _CloseWithCopy(filename, nc4_dataset)
513-
manager = DummyFileManager(nc4_dataset, close=close)
513+
manager = DummyFileManager(nc4_dataset, close=close, lock=lock)
514514

515515
elif isinstance(filename, bytes | memoryview):
516516
assert mode == "r"
517517
kwargs["memory"] = filename
518518
manager = PickleableFileManager(
519-
netCDF4.Dataset, "<xarray-in-memory-read>", mode=mode, kwargs=kwargs
519+
netCDF4.Dataset,
520+
"<xarray-in-memory-read>",
521+
mode=mode,
522+
kwargs=kwargs,
523+
lock=lock,
520524
)
521525
else:
522526
manager = CachingFileManager(
523-
netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
527+
netCDF4.Dataset, filename, mode=mode, kwargs=kwargs, lock=lock
524528
)
525529
return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
526530

0 commit comments

Comments
 (0)