Skip to content

Commit c75410b

Browse files
authored
spec: get_file: don't pollute with files and fds (#1191)
* spec: get_file: don't pollute with files and fds Currently if anything happens in fs.open we will leave an empty file in user's workspace and also won't close the file descriptor. Both things are pretty bad. * http: _get_file: don't leak file descriptor
1 parent d648c3b commit c75410b

2 files changed

Lines changed: 30 additions & 22 deletions

File tree

‎fsspec/implementations/http.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -246,13 +246,20 @@ async def _get_file(
246246

247247
callback.set_size(size)
248248
self._raise_not_found_for_status(r, rpath)
249-
if not isfilelike(lpath):
250-
lpath = open(lpath, "wb")
251-
chunk = True
252-
while chunk:
253-
chunk = await r.content.read(chunk_size)
254-
lpath.write(chunk)
255-
callback.relative_update(len(chunk))
249+
if isfilelike(lpath):
250+
outfile = lpath
251+
else:
252+
outfile = open(lpath, "wb")
253+
254+
try:
255+
chunk = True
256+
while chunk:
257+
chunk = await r.content.read(chunk_size)
258+
outfile.write(chunk)
259+
callback.relative_update(len(chunk))
260+
finally:
261+
if not isfilelike(lpath):
262+
outfile.close()
256263

257264
async def _put_file(
258265
self,

‎fsspec/spec.py‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -848,25 +848,26 @@ def get_file(
848848
"""Copy single remote file to local"""
849849
if isfilelike(lpath):
850850
outfile = lpath
851-
else:
852-
if self.isdir(rpath):
853-
os.makedirs(lpath, exist_ok=True)
854-
return None
851+
elif self.isdir(rpath):
852+
os.makedirs(lpath, exist_ok=True)
853+
return None
855854

855+
with self.open(rpath, "rb", **kwargs) as f1:
856856
if outfile is None:
857857
outfile = open(lpath, "wb")
858858

859-
with self.open(rpath, "rb", **kwargs) as f1:
860-
callback.set_size(getattr(f1, "size", None))
861-
data = True
862-
while data:
863-
data = f1.read(self.blocksize)
864-
segment_len = outfile.write(data)
865-
if segment_len is None:
866-
segment_len = len(data)
867-
callback.relative_update(segment_len)
868-
if not isfilelike(lpath):
869-
outfile.close()
859+
try:
860+
callback.set_size(getattr(f1, "size", None))
861+
data = True
862+
while data:
863+
data = f1.read(self.blocksize)
864+
segment_len = outfile.write(data)
865+
if segment_len is None:
866+
segment_len = len(data)
867+
callback.relative_update(segment_len)
868+
finally:
869+
if not isfilelike(lpath):
870+
outfile.close()
870871

871872
def get(self, rpath, lpath, recursive=False, callback=_DEFAULT_CALLBACK, **kwargs):
872873
"""Copy file(s) to local.

0 commit comments

Comments
 (0)