Skip to content

Commit aa9ca52

Browse files
feat: require opt-in to read data encoded with the pickle codec
Decoding a chunk with the numcodecs `pickle` codec calls `pickle.loads` on bytes from the store, so opening an untrusted Zarr v2 array can run arbitrary code. Reading such data now raises `PickleCodecDisabledError` unless the new `array.allow_pickle` config option is `True`, set with `zarr.config.set({"array.allow_pickle": True})` or the environment variable `ZARR_ARRAY__ALLOW_PICKLE=True`. The check sits in `V2Codec._decode_sync`, before any codec runs, and covers the pickle codec as compressor or filter. Opening the array and writing with the pickle codec are unchanged. Closes zarr-developers#4404
1 parent 89f058c commit aa9ca52

7 files changed

Lines changed: 125 additions & 2 deletions

File tree

‎changes/4404.feature.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Reading data encoded with the `pickle` codec now requires an explicit opt-in, because unpickling data from an untrusted store can execute arbitrary code. Enable it with `zarr.config.set({"array.allow_pickle": True})` or the environment variable `ZARR_ARRAY__ALLOW_PICKLE=True`. Without the opt-in, reading such data raises `zarr.errors.PickleCodecDisabledError`. Writing with the `pickle` codec is unchanged.

‎docs/user-guide/config.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ Configuration options include the following:
4040
- Whether empty chunks are written to storage `array.write_empty_chunks`
4141
- Enable experimental rectilinear chunks `array.rectilinear_chunks`
4242
- Whether missing chunks are filled with the array's fill value on read `array.read_missing_chunks` (default `True`). Set to `False` to raise a [`ChunkNotFoundError`][zarr.errors.ChunkNotFoundError] instead.
43+
- Whether data encoded with the `pickle` codec can be read `array.allow_pickle` (default `False`). Unpickling can execute arbitrary code, so only enable this for data from a source you trust, e.g. with `zarr.config.set({'array.allow_pickle': True})` or the environment variable `ZARR_ARRAY__ALLOW_PICKLE=True`. While it is disabled, reading such data raises a [`PickleCodecDisabledError`][zarr.errors.PickleCodecDisabledError]. Writing with the `pickle` codec is not affected.
4344
- Async and threading options, e.g. `async.concurrency` and `threading.max_workers`
4445
- Selections of implementations of codecs, codec pipelines and buffers
4546
- Enabling GPU support with `zarr.config.enable_gpu()`. See [GPU support](gpu.md) for more.

‎src/zarr/codecs/_v2.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
from numcodecs.compat import ensure_bytes, ensure_ndarray_like
99

1010
from zarr.abc.codec import ArrayBytesCodec
11+
from zarr.core.config import config
12+
from zarr.errors import PickleCodecDisabledError
1113
from zarr.registry import get_ndbuffer_class
1214

1315
if TYPE_CHECKING:
@@ -16,6 +18,26 @@
1618
from zarr.core.buffer import Buffer, NDBuffer
1719

1820

21+
def _check_pickle_allowed(codecs: tuple[Numcodec | None, ...]) -> None:
22+
"""
23+
Raise if any of ``codecs`` is the pickle codec and reading pickled data has not been enabled.
24+
25+
Unpickling can run arbitrary code, so data from an untrusted store must not be decoded
26+
with the pickle codec unless the user has explicitly opted in.
27+
"""
28+
if not any(getattr(codec, "codec_id", None) == "pickle" for codec in codecs):
29+
return
30+
# Compare with ``True`` rather than testing truthiness: donfig keeps an environment
31+
# variable it cannot parse as a Python literal (e.g. ``false``) as a non-empty string.
32+
if config.get("array.allow_pickle") is not True:
33+
raise PickleCodecDisabledError(
34+
"Refusing to decode data with the 'pickle' codec, because unpickling data "
35+
"from an untrusted source can execute arbitrary code. If you trust this data, "
36+
"enable the pickle codec with zarr.config.set({'array.allow_pickle': True}) "
37+
"or set the environment variable ZARR_ARRAY__ALLOW_PICKLE=True."
38+
)
39+
40+
1941
@dataclass(frozen=True)
2042
class V2Codec(ArrayBytesCodec):
2143
filters: tuple[Numcodec, ...] | None
@@ -28,6 +50,7 @@ def _decode_sync(
2850
chunk_bytes: Buffer,
2951
chunk_spec: ArraySpec,
3052
) -> NDBuffer:
53+
_check_pickle_allowed((self.compressor, *(self.filters or ())))
3154
cdata = chunk_bytes.as_array_like()
3255
# decompress
3356
if self.compressor:

‎src/zarr/core/config.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ def enable_gpu(self) -> ConfigSet:
103103
"rectilinear_chunks": False,
104104
"sharding_coalesce_max_gap_bytes": 1 << 20, # 1 MiB
105105
"sharding_coalesce_max_bytes": 16 << 20, # 16 MiB
106+
# Decoding data with the pickle codec can execute arbitrary code,
107+
# so it must be explicitly enabled.
108+
"allow_pickle": False,
106109
},
107110
"async": {"concurrency": 10, "timeout": None},
108111
"threading": {"max_workers": None},

‎src/zarr/errors.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"MetadataValidationError",
1313
"NegativeStepError",
1414
"NodeTypeValidationError",
15+
"PickleCodecDisabledError",
1516
"UnknownCodecError",
1617
"UnstableSpecificationWarning",
1718
"VindexInvalidSelectionError",
@@ -101,6 +102,15 @@ class UnknownCodecError(BaseZarrError):
101102
"""
102103

103104

105+
class PickleCodecDisabledError(BaseZarrError):
106+
"""
107+
Raised when reading data encoded with the pickle codec without opting in.
108+
109+
Unpickling can execute arbitrary code, so decoding with the pickle codec is
110+
disabled unless the ``array.allow_pickle`` configuration option is set to ``True``.
111+
"""
112+
113+
104114
class NodeTypeValidationError(MetadataValidationError):
105115
"""
106116
Specialized exception when the node_type of the metadata document is incorrect.

‎tests/test_config.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ def test_config_defaults_set() -> None:
5959
"rectilinear_chunks": False,
6060
"sharding_coalesce_max_gap_bytes": 1 << 20,
6161
"sharding_coalesce_max_bytes": 16 << 20,
62+
"allow_pickle": False,
6263
},
6364
"async": {"concurrency": 10, "timeout": None},
6465
"threading": {"max_workers": None},

‎tests/test_v2.py‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
import json
2+
import pickle
23
from pathlib import Path
34
from typing import Any, Literal
45

56
import numpy as np
67
import pytest
7-
from numcodecs import Delta, Zlib
8+
from numcodecs import Delta, Pickle, VLenUTF8, Zlib
89
from numcodecs.blosc import Blosc
910
from numcodecs.zstd import Zstd
1011

@@ -20,7 +21,7 @@
2021
from zarr.core.dtype.wrapper import ZDType
2122
from zarr.core.group import Group
2223
from zarr.core.sync import sync
23-
from zarr.errors import ZarrDeprecationWarning
24+
from zarr.errors import PickleCodecDisabledError, ZarrDeprecationWarning
2425
from zarr.storage import MemoryStore, StorePath
2526

2627

@@ -310,3 +311,86 @@ def test_other_dtype_roundtrip(fill_value: bytes | None, tmp_path: Path) -> None
310311
za[...] = a
311312
za = zarr.open_array(store=array_path)
312313
assert (a == za[:]).all()
314+
315+
316+
def _pickle_compressed_array() -> zarr.Array[Any]:
317+
# Writing with the pickle codec is allowed without opting in, only decoding is gated.
318+
arr = zarr.create_array(
319+
store={},
320+
shape=(3,),
321+
dtype=VariableLengthUTF8(), # type: ignore[arg-type]
322+
zarr_format=2,
323+
filters=[VLenUTF8()],
324+
compressors=Pickle(),
325+
fill_value="",
326+
)
327+
arr[:] = np.array(["a", "bb", "ccc"])
328+
return arr
329+
330+
331+
def test_pickle_codec_read_disabled_by_default() -> None:
332+
"""Reading data encoded with the pickle codec fails unless the user opts in."""
333+
arr = _pickle_compressed_array()
334+
with pytest.raises(PickleCodecDisabledError, match=r"array\.allow_pickle"):
335+
arr[:]
336+
337+
338+
def test_pickle_codec_read_does_not_unpickle() -> None:
339+
"""A malicious chunk is not unpickled when the pickle codec is disabled."""
340+
341+
class Payload:
342+
def __reduce__(self) -> tuple[Any, ...]:
343+
return (exec, ("raise AssertionError('payload was unpickled')",))
344+
345+
meta = {
346+
"zarr_format": 2,
347+
"shape": [1],
348+
"chunks": [1],
349+
"dtype": "|O",
350+
"compressor": {"id": "pickle"},
351+
"fill_value": None,
352+
"filters": [{"id": "vlen-utf8"}],
353+
"order": "C",
354+
}
355+
buffer_cls = default_buffer_prototype().buffer
356+
store = {
357+
".zarray": buffer_cls.from_bytes(json.dumps(meta).encode()),
358+
"0": buffer_cls.from_bytes(pickle.dumps(Payload())),
359+
}
360+
arr = zarr.open_array(store=MemoryStore(store), mode="r", zarr_format=2)
361+
with pytest.raises(PickleCodecDisabledError):
362+
arr[:]
363+
364+
365+
def test_pickle_codec_read_enabled_via_config() -> None:
366+
arr = _pickle_compressed_array()
367+
with config.set({"array.allow_pickle": True}):
368+
np.testing.assert_array_equal(arr[:], np.array(["a", "bb", "ccc"]))
369+
370+
371+
@pytest.mark.parametrize(("env_value", "allowed"), [("True", True), ("false", False)])
372+
def test_pickle_codec_read_enabled_via_env_var(
373+
monkeypatch: pytest.MonkeyPatch, env_value: str, allowed: bool
374+
) -> None:
375+
"""
376+
The opt-in can be set with an environment variable. Only a value that parses to ``True``
377+
enables it: donfig keeps ``false`` as a (truthy) string, which must not enable it.
378+
"""
379+
arr = _pickle_compressed_array()
380+
monkeypatch.setenv("ZARR_ARRAY__ALLOW_PICKLE", env_value)
381+
config.refresh()
382+
if allowed:
383+
np.testing.assert_array_equal(arr[:], np.array(["a", "bb", "ccc"]))
384+
else:
385+
with pytest.raises(PickleCodecDisabledError):
386+
arr[:]
387+
388+
389+
def test_pickle_opt_in_does_not_affect_other_codecs() -> None:
390+
"""Arrays that do not use the pickle codec are read without opting in."""
391+
assert config.get("array.allow_pickle") is False
392+
arr = zarr.create_array(
393+
store={}, shape=(3,), dtype="<i4", zarr_format=2, filters=[Delta(dtype="<i4")]
394+
)
395+
arr[:] = [1, 2, 3]
396+
np.testing.assert_array_equal(arr[:], [1, 2, 3])

0 commit comments

Comments
 (0)