Skip to content

Commit a1c16ab

Browse files
authored
Honour simple_templates everywhere in referenceFS (#2029)
1 parent 85facaa commit a1c16ab

4 files changed

Lines changed: 47 additions & 9 deletions

File tree

‎fsspec/implementations/reference.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,9 @@ class ReferenceFileSystem(AsyncFileSystem):
615615
Reference dict format:
616616
{path0: bytes_data, path1: (target_url, offset, size)}
617617
https://github.com/fsspec/kerchunk/blob/main/README.md
618+
619+
simple_references: if True (default), no jinja interpreting is done,
620+
which is the safe option.
618621
"""
619622

620623
protocol = "reference"
@@ -1056,6 +1059,8 @@ def _process_templates(self, tmp):
10561059

10571060
def _process_gen(self, gens):
10581061
out = {}
1062+
if self.simple_templates:
1063+
return out
10591064
for gen in gens:
10601065
dimension = {
10611066
k: (

‎fsspec/implementations/tests/test_dbfs.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@
2525
import os
2626
from urllib.parse import urlparse
2727

28-
import numpy
2928
import pytest
3029

3130
import fsspec
3231

3332
pytest.skip("These tests need to be re-recorded.", allow_module_level=True)
3433

34+
numpy = pytest.importorskip("numpy")
3535
DUMMY_INSTANCE = "my_instance.com"
3636
INSTANCE = os.getenv("DBFS_INSTANCE", DUMMY_INSTANCE)
3737
TOKEN = os.getenv("DBFS_TOKEN", "")

‎fsspec/implementations/tests/test_gist.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import pytest
22

33
import fsspec
4+
5+
pytest.importorskip("requests")
6+
47
from fsspec.implementations.gist import GistFileSystem
58

69
# if sys.version_info[:2] != (3, 12):

‎fsspec/implementations/tests/test_reference.py‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,12 @@ def test_spec1_gen_variants():
365365
},
366366
],
367367
}
368-
fsspec.filesystem("reference", fo=missing_length_spec, target_protocol="http")
368+
fsspec.filesystem(
369+
"reference",
370+
fo=missing_length_spec,
371+
simple_templates=False,
372+
target_protocol="http",
373+
)
369374

370375
with pytest.raises(ValueError):
371376
missing_offset_spec = {
@@ -380,7 +385,12 @@ def test_spec1_gen_variants():
380385
},
381386
],
382387
}
383-
fsspec.filesystem("reference", fo=missing_offset_spec, target_protocol="http")
388+
fsspec.filesystem(
389+
"reference",
390+
simple_templates=False,
391+
fo=missing_offset_spec,
392+
target_protocol="http",
393+
)
384394

385395
url_only_gen_spec = {
386396
"version": 1,
@@ -394,7 +404,12 @@ def test_spec1_gen_variants():
394404
],
395405
}
396406

397-
fs = fsspec.filesystem("reference", fo=url_only_gen_spec, target_protocol="http")
407+
fs = fsspec.filesystem(
408+
"reference",
409+
simple_templates=False,
410+
fo=url_only_gen_spec,
411+
target_protocol="http",
412+
)
398413
assert fs.references == {
399414
"gen_key0": ["http://server.domain/path_0"],
400415
"gen_key1": ["http://server.domain/path_1"],
@@ -857,10 +872,7 @@ def test_deep_parq(m, engine):
857872
pytest.importorskip("kerchunk")
858873
zarr = pytest.importorskip("zarr")
859874
skip_zarr_2()
860-
if zarr.__version__.split(".") >= ["3", "2"]:
861-
kw = {}
862-
else:
863-
kw = {"zarr_format": 2}
875+
kw = {"zarr_format": 2}
864876

865877
lz = fsspec.implementations.reference.LazyReferenceMapper.create(
866878
"memory://out.parq",
@@ -886,7 +898,7 @@ def test_deep_parq(m, engine):
886898
g = zarr.open_group(
887899
"reference://",
888900
storage_options={"fo": "memory://out.parq", "remote_protocol": "memory"},
889-
zarr_version=2,
901+
**kw,
890902
)
891903
assert g["instant"]["one"][:].tolist() == [1, 2, 3]
892904
assert sorted(_["name"] for _ in lz.ls("")) == [
@@ -965,3 +977,21 @@ def test_parquet_no_references(m):
965977
lz.flush()
966978

967979
assert arr[...].tolist() == 1 # scalar, equal to fill value
980+
981+
982+
def test_no_default_jinja_execution():
983+
pytest.importorskip("jinja2")
984+
manifest = {
985+
"version": 1,
986+
"templates": {},
987+
"refs": {},
988+
"gen": [
989+
{"key": "{{ 1 / 0 }}", "url": "file:///dev/null", "dimensions": {"i": [0]}}
990+
],
991+
}
992+
993+
# no error - not evaluated
994+
fsspec.filesystem("reference", fo=manifest)
995+
996+
with pytest.raises(ZeroDivisionError):
997+
fsspec.filesystem("reference", fo=manifest, simple_templates=False)

0 commit comments

Comments
 (0)