Skip to content

Commit dd02d75

Browse files
[backport 3.0.x] BUG: keep fsspec OpenFile alive for chained URL reads (#65470) (#65485)
1 parent aef3d0f commit dd02d75

2 files changed

Lines changed: 10 additions & 4 deletions

File tree

‎doc/source/whatsnew/v3.0.3.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ Bug fixes
4848
raising an error instead of propagating the missing value (:issue:`64968`)
4949
- Fixed a bug in :meth:`Series.rank` with period dtype and missing values, always sorting missing values at the top, regardless of the ``na_option`` value (:issue:`64976`)
5050

51+
- Fixed a bug in reading files with a chained URL scheme (e.g. ``tar://``) with the latest version of ``fsspec`` (:issue:`65462`).
5152
- Fixed a hang in :meth:`Series.str.replace` for PyArrow-backed string data when replacing an empty pattern; behavior now matches Python's ``str.replace`` (:issue:`64941`).
5253

5354
.. ---------------------------------------------------------------------------

‎pandas/io/common.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ class IOArgs:
9999
mode: str
100100
compression: CompressionDict
101101
should_close: bool = False
102+
close_handles: list[Any] = dataclasses.field(default_factory=list)
102103

103104

104105
@dataclasses.dataclass
@@ -447,9 +448,10 @@ def _get_filepath_or_buffer(
447448
pass
448449

449450
try:
450-
file_obj = fsspec.open(
451+
open_file = fsspec.open(
451452
filepath_or_buffer, mode=fsspec_mode, **(storage_options or {})
452-
).open()
453+
)
454+
file_obj = open_file.open()
453455
# GH 34626 Reads from Public Buckets without Credentials needs anon=True
454456
except tuple(err_types_to_retry_with_anon):
455457
if storage_options is None:
@@ -458,14 +460,16 @@ def _get_filepath_or_buffer(
458460
# don't mutate user input.
459461
storage_options = dict(storage_options)
460462
storage_options["anon"] = True
461-
file_obj = fsspec.open(
463+
open_file = fsspec.open(
462464
filepath_or_buffer, mode=fsspec_mode, **(storage_options or {})
463-
).open()
465+
)
466+
file_obj = open_file.open()
464467

465468
return IOArgs(
466469
filepath_or_buffer=file_obj,
467470
encoding=encoding,
468471
compression=compression,
472+
close_handles=[open_file],
469473
should_close=True,
470474
mode=fsspec_mode,
471475
)
@@ -976,6 +980,7 @@ def get_handle(
976980
if ioargs.should_close:
977981
assert not isinstance(ioargs.filepath_or_buffer, str)
978982
handles.append(ioargs.filepath_or_buffer)
983+
handles.extend(ioargs.close_handles)
979984

980985
return IOHandles(
981986
# error: Argument "handle" to "IOHandles" has incompatible type

0 commit comments

Comments
 (0)