Skip to content

Commit 8643878

Browse files
authored
Small safety improvements (#2039)
1 parent a1c16ab commit 8643878

10 files changed

Lines changed: 70 additions & 132 deletions

File tree

‎.github/workflows/main.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,12 @@ jobs:
5959

6060
steps:
6161
- name: Checkout
62-
uses: actions/checkout@v5
62+
uses: actions/checkout@v6
6363
with:
6464
fetch-depth: 0
6565

6666
- name: Setup conda
67-
uses: conda-incubator/setup-miniconda@v3
67+
uses: conda-incubator/setup-miniconda@v4
6868
with:
6969
environment-file: ci/environment-win.yml
7070

‎ci/environment-win.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ name: test_env
22
channels:
33
- conda-forge
44
dependencies:
5-
- python=3.11
5+
- python=3.13
66
- s3fs
77
- pytest
8+
- pytest-asyncio
9+
- pytest-mock
10+
- requests
11+
- numpy

‎fsspec/asyn.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ async def _run_coros_in_chunks(
236236

237237
if batch_size == -1:
238238
batch_size = len(coros)
239-
240-
assert batch_size > 0
239+
elif batch_size <= 0:
240+
raise ValueError
241241

242242
async def _run_coro(coro, i):
243243
try:

‎fsspec/implementations/cache_metadata.py‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import os
4-
import pickle
54
import time
65
from typing import TYPE_CHECKING
76

@@ -29,8 +28,7 @@ class CacheMetadata:
2928
accessing the cached files and blocks is not.
3029
3130
Metadata is stored in a single file per storage directory in JSON format.
32-
For backward compatibility, also reads metadata stored in pickle format
33-
which is converted to JSON when next saved.
31+
For backward compatibility. No longer supports pickle.
3432
"""
3533

3634
def __init__(self, storage: list[str]):
@@ -48,31 +46,19 @@ def __init__(self, storage: list[str]):
4846
self._storage = storage
4947
self.cached_files: list[Detail] = [{}]
5048

51-
# Private attribute to force saving of metadata in pickle format rather than
52-
# JSON for use in tests to confirm can read both pickle and JSON formats.
53-
self._force_save_pickle = False
54-
5549
def _load(self, fn: str) -> Detail:
5650
"""Low-level function to load metadata from specific file"""
57-
try:
58-
with open(fn, "r") as f:
59-
loaded = json.load(f)
60-
except ValueError:
61-
with open(fn, "rb") as f:
62-
loaded = pickle.load(f)
51+
with open(fn, "r") as f:
52+
loaded = json.load(f)
6353
for c in loaded.values():
6454
if isinstance(c.get("blocks"), list):
6555
c["blocks"] = set(c["blocks"])
6656
return loaded
6757

6858
def _save(self, metadata_to_save: Detail, fn: str) -> None:
6959
"""Low-level function to save metadata to specific file"""
70-
if self._force_save_pickle:
71-
with atomic_write(fn) as f:
72-
pickle.dump(metadata_to_save, f)
73-
else:
74-
with atomic_write(fn, mode="w") as f:
75-
json.dump(metadata_to_save, f)
60+
with atomic_write(fn, mode="w") as f:
61+
json.dump(metadata_to_save, f)
7662

7763
def _scan_locations(
7864
self, writable_only: bool = False

‎fsspec/implementations/dbfs.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -441,9 +441,10 @@ def __init__(
441441
if block_size is None or block_size == "default":
442442
block_size = self.DEFAULT_BLOCK_SIZE
443443

444-
assert block_size == self.DEFAULT_BLOCK_SIZE, (
445-
f"Only the default block size is allowed, not {block_size}"
446-
)
444+
if block_size != self.DEFAULT_BLOCK_SIZE:
445+
raise ValueError(
446+
f"Only the default block size is allowed, not {block_size}"
447+
)
447448

448449
super().__init__(
449450
fs,

‎fsspec/implementations/reference.py‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,13 +1012,17 @@ def _process_references0(self, references):
10121012

10131013
def _process_references1(self, references, template_overrides=None):
10141014
if not self.simple_templates or self.templates:
1015-
import jinja2
1015+
import jinja2.sandbox
10161016
self.references = {}
10171017
self._process_templates(references.get("templates", {}))
10181018

10191019
@lru_cache(1000)
10201020
def _render_jinja(u):
1021-
return jinja2.Template(u).render(**self.templates)
1021+
return (
1022+
jinja2.sandbox.SandboxedEnvironment()
1023+
.from_string(u)
1024+
.render(**self.templates)
1025+
)
10221026

10231027
for k, v in references.get("refs", {}).items():
10241028
if isinstance(v, str):
@@ -1049,11 +1053,13 @@ def _process_templates(self, tmp):
10491053
tmp.update(self.template_overrides)
10501054
for k, v in tmp.items():
10511055
if "{{" in v:
1052-
import jinja2
1056+
import jinja2.sandbox
10531057

1054-
self.templates[k] = lambda temp=v, **kwargs: jinja2.Template(
1055-
temp
1056-
).render(**kwargs)
1058+
self.templates[k] = (
1059+
lambda temp=v, **kwargs: jinja2.sandbox.SandboxedEnvironment()
1060+
.from_string(temp)
1061+
.render(**kwargs)
1062+
)
10571063
else:
10581064
self.templates[k] = v
10591065

@@ -1075,16 +1081,28 @@ def _process_gen(self, gens):
10751081
for values in itertools.product(*dimension.values())
10761082
)
10771083
for pr in products:
1078-
import jinja2
1084+
import jinja2.sandbox
10791085

1080-
key = jinja2.Template(gen["key"]).render(**pr, **self.templates)
1081-
url = jinja2.Template(gen["url"]).render(**pr, **self.templates)
1086+
key = (
1087+
jinja2.sandbox.SandboxedEnvironment()
1088+
.from_string(gen["key"])
1089+
.render(**pr, **self.templates)
1090+
)
1091+
url = (
1092+
jinja2.sandbox.SandboxedEnvironment()
1093+
.from_string(gen["url"])
1094+
.render(**pr, **self.templates)
1095+
)
10821096
if ("offset" in gen) and ("length" in gen):
10831097
offset = int(
1084-
jinja2.Template(gen["offset"]).render(**pr, **self.templates)
1098+
jinja2.sandbox.SandboxedEnvironment()
1099+
.from_string(gen["offset"])
1100+
.render(**pr, **self.templates)
10851101
)
10861102
length = int(
1087-
jinja2.Template(gen["length"]).render(**pr, **self.templates)
1103+
jinja2.sandbox.SandboxedEnvironment()
1104+
.from_string(gen["length"])
1105+
.render(**pr, **self.templates)
10881106
)
10891107
out[key] = [url, offset, length]
10901108
elif ("offset" in gen) ^ ("length" in gen):

‎fsspec/implementations/tests/test_cached.py‎

Lines changed: 11 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import json
22
import os
3-
import pickle
43
import shutil
54
import tempfile
65

@@ -118,8 +117,7 @@ def test_mapper():
118117
@pytest.mark.parametrize(
119118
"cache_mapper", [BasenameCacheMapper(), BasenameCacheMapper(1), HashCacheMapper()]
120119
)
121-
@pytest.mark.parametrize("force_save_pickle", [True, False])
122-
def test_metadata(tmpdir, cache_mapper, force_save_pickle):
120+
def test_metadata(tmpdir, cache_mapper):
123121
source = os.path.join(tmpdir, "source")
124122
afile = os.path.join(source, "afile")
125123
os.mkdir(source)
@@ -131,7 +129,6 @@ def test_metadata(tmpdir, cache_mapper, force_save_pickle):
131129
cache_storage=os.path.join(tmpdir, "cache"),
132130
cache_mapper=cache_mapper,
133131
)
134-
fs._metadata._force_save_pickle = force_save_pickle
135132

136133
with fs.open(afile, "rb") as f:
137134
assert f.read(5) == b"test"
@@ -154,42 +151,6 @@ def test_metadata(tmpdir, cache_mapper, force_save_pickle):
154151
assert detail["fn"] == "source_@_afile"
155152

156153

157-
def test_metadata_replace_pickle_with_json(tmpdir):
158-
# For backward compatibility will allow reading of old pickled metadata.
159-
# When the metadata is next saved, it is in json format.
160-
source = os.path.join(tmpdir, "source")
161-
afile = os.path.join(source, "afile")
162-
os.mkdir(source)
163-
open(afile, "w").write("test")
164-
165-
# Save metadata in pickle format, to simulate old metadata
166-
fs = fsspec.filesystem(
167-
"filecache",
168-
target_protocol="file",
169-
cache_storage=os.path.join(tmpdir, "cache"),
170-
)
171-
fs._metadata._force_save_pickle = True
172-
with fs.open(afile, "rb") as f:
173-
assert f.read(5) == b"test"
174-
175-
# Confirm metadata is in pickle format
176-
cache_fn = os.path.join(fs.storage[-1], "cache")
177-
with open(cache_fn, "rb") as f:
178-
metadata = pickle.load(f)
179-
assert list(metadata.keys()) == [make_path_posix(afile)]
180-
181-
# Force rewrite of metadata, now in json format
182-
fs._metadata._force_save_pickle = False
183-
fs.pop_from_cache(afile)
184-
with fs.open(afile, "rb") as f:
185-
assert f.read(5) == b"test"
186-
187-
# Confirm metadata is in json format
188-
with open(cache_fn, "r") as f:
189-
metadata = json.load(f)
190-
assert list(metadata.keys()) == [make_path_posix(afile)]
191-
192-
193154
def test_constructor_kwargs(tmpdir):
194155
fs = fsspec.filesystem("filecache", target_protocol="file", same_names=True)
195156
assert isinstance(fs._mapper, BasenameCacheMapper)
@@ -212,15 +173,16 @@ def test_constructor_kwargs(tmpdir):
212173

213174

214175
def test_idempotent():
176+
import pickle
177+
215178
fs = CachingFileSystem("file")
216179
fs2 = CachingFileSystem("file")
217180
assert fs2 is fs
218181
fs3 = pickle.loads(pickle.dumps(fs))
219182
assert fs3.storage == fs.storage
220183

221184

222-
@pytest.mark.parametrize("force_save_pickle", [True, False])
223-
def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
185+
def test_blockcache_workflow(ftp_writable, tmp_path):
224186
host, port, user, pw = ftp_writable
225187
fs = FTPFileSystem(host, port, user, pw)
226188
with fs.open("/out", "wb") as f:
@@ -240,7 +202,6 @@ def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
240202

241203
# Open the blockcache and read a little bit of the data
242204
fs = fsspec.filesystem("blockcache", **fs_kwargs)
243-
fs._metadata._force_save_pickle = force_save_pickle
244205
with fs.open("/out", "rb", block_size=5) as f:
245206
assert f.read(5) == b"test\n"
246207

@@ -249,18 +210,13 @@ def test_blockcache_workflow(ftp_writable, tmp_path, force_save_pickle):
249210
del fs
250211

251212
# Check that cache file only has the first two blocks
252-
if force_save_pickle:
253-
with open(tmp_path / "cache", "rb") as f:
254-
cache = pickle.load(f)
255-
else:
256-
with open(tmp_path / "cache", "r") as f:
257-
cache = json.load(f)
213+
with open(tmp_path / "cache", "r") as f:
214+
cache = json.load(f)
258215
assert "/out" in cache
259216
assert cache["/out"]["blocks"] == [0, 1]
260217

261218
# Reopen the same cache and read some more...
262219
fs = fsspec.filesystem("blockcache", **fs_kwargs)
263-
fs._metadata._force_save_pickle = force_save_pickle
264220
with fs.open("/out", block_size=5) as f:
265221
assert f.read(5) == b"test\n"
266222
f.seek(30)
@@ -347,8 +303,7 @@ def test_clear():
347303
assert len(os.listdir(cache1)) < 2
348304

349305

350-
@pytest.mark.parametrize("force_save_pickle", [True, False])
351-
def test_clear_expired(tmp_path, force_save_pickle):
306+
def test_clear_expired(tmp_path):
352307
def __ager(cache_fn, fn, del_fn=False):
353308
"""
354309
Modify the cache file to virtually add time lag to selected files.
@@ -366,23 +321,15 @@ def __ager(cache_fn, fn, del_fn=False):
366321
import time
367322

368323
if os.path.exists(cache_fn):
369-
if force_save_pickle:
370-
with open(cache_fn, "rb") as f:
371-
cached_files = pickle.load(f)
372-
else:
373-
with open(cache_fn, "r") as f:
374-
cached_files = json.load(f)
324+
with open(cache_fn, "r") as f:
325+
cached_files = json.load(f)
375326
fn_posix = pathlib.Path(fn).as_posix()
376327
cached_files[fn_posix]["time"] = cached_files[fn_posix]["time"] - 691200
377328
assert os.access(cache_fn, os.W_OK), "Cache is not writable"
378329
if del_fn:
379330
del cached_files[fn_posix]["fn"]
380-
if force_save_pickle:
381-
with open(cache_fn, "wb") as f:
382-
pickle.dump(cached_files, f)
383-
else:
384-
with open(cache_fn, "w") as f:
385-
json.dump(cached_files, f)
331+
with open(cache_fn, "w") as f:
332+
json.dump(cached_files, f)
386333
time.sleep(1)
387334

388335
origin = tmp_path.joinpath("origin")
@@ -414,7 +361,6 @@ def __ager(cache_fn, fn, del_fn=False):
414361
fs = fsspec.filesystem(
415362
"filecache", target_protocol="file", cache_storage=str(cache1), cache_check=1
416363
)
417-
fs._metadata._force_save_pickle = force_save_pickle
418364
assert fs.cat(str(f1)) == data
419365

420366
# populates "last" cache if file not found in first one
@@ -424,7 +370,6 @@ def __ager(cache_fn, fn, del_fn=False):
424370
cache_storage=[str(cache1), str(cache2)],
425371
cache_check=1,
426372
)
427-
fs._metadata._force_save_pickle = force_save_pickle
428373
assert fs.cat(str(f2)) == data
429374
assert fs.cat(str(f3)) == data
430375
assert len(os.listdir(cache2)) == 3
@@ -456,7 +401,6 @@ def __ager(cache_fn, fn, del_fn=False):
456401
same_names=True,
457402
cache_check=1,
458403
)
459-
fs._metadata._force_save_pickle = force_save_pickle
460404
assert fs.cat(str(f4)) == data
461405

462406
cache_fn = os.path.join(fs.storage[-1], "cache")
@@ -473,7 +417,6 @@ def __ager(cache_fn, fn, del_fn=False):
473417
same_names=True,
474418
cache_check=1,
475419
)
476-
fs._metadata._force_save_pickle = force_save_pickle
477420
assert fs.cat(str(f1)) == data
478421

479422
cache_fn = os.path.join(fs.storage[-1], "cache")
@@ -517,21 +460,6 @@ def test_pop():
517460
assert fs._check_file(f1)
518461

519462

520-
def test_write_pickle_context():
521-
tmp = str(tempfile.mkdtemp())
522-
fn = tmp + "afile"
523-
url = f"simplecache::file://{fn}"
524-
with fsspec.open(url, "wb") as f:
525-
pickle.loads(pickle.dumps(f))
526-
f.write(b"hello ")
527-
pickle.dumps(f)
528-
529-
with pytest.raises(ValueError):
530-
pickle.dumps(f)
531-
532-
assert open(fn, "rb").read() == b"hello "
533-
534-
535463
def test_blocksize(ftp_writable):
536464
host, port, user, pw = ftp_writable
537465
fs = FTPFileSystem(host, port, user, pw)

0 commit comments

Comments
 (0)