|
1 | 1 | import json |
| 2 | +import pickle |
2 | 3 | from pathlib import Path |
3 | 4 | from typing import Any, Literal |
4 | 5 |
|
| 6 | +import numcodecs.errors |
| 7 | +import numcodecs.pickles |
5 | 8 | import numpy as np |
6 | 9 | import pytest |
7 | | -from numcodecs import Delta, Zlib |
| 10 | +from numcodecs import Delta, Pickle, VLenUTF8, Zlib |
8 | 11 | from numcodecs.blosc import Blosc |
9 | 12 | from numcodecs.zstd import Zstd |
10 | 13 |
|
|
20 | 23 | from zarr.core.dtype.wrapper import ZDType |
21 | 24 | from zarr.core.group import Group |
22 | 25 | from zarr.core.sync import sync |
23 | | -from zarr.errors import ZarrDeprecationWarning |
| 26 | +from zarr.errors import PickleCodecDisabledError, ZarrDeprecationWarning |
24 | 27 | from zarr.storage import MemoryStore, StorePath |
25 | 28 |
|
26 | 29 |
|
@@ -310,3 +313,116 @@ def test_other_dtype_roundtrip(fill_value: bytes | None, tmp_path: Path) -> None |
310 | 313 | za[...] = a |
311 | 314 | za = zarr.open_array(store=array_path) |
312 | 315 | assert (a == za[:]).all() |
| 316 | + |
| 317 | + |
| 318 | +def _pickle_compressed_array() -> zarr.Array[Any]: |
| 319 | + # Writing with the pickle codec is allowed without opting in, only decoding is gated. |
| 320 | + arr = zarr.create_array( |
| 321 | + store={}, |
| 322 | + shape=(3,), |
| 323 | + dtype=VariableLengthUTF8(), # type: ignore[arg-type] |
| 324 | + zarr_format=2, |
| 325 | + filters=[VLenUTF8()], |
| 326 | + compressors=Pickle(), |
| 327 | + fill_value="", |
| 328 | + ) |
| 329 | + arr[:] = np.array(["a", "bb", "ccc"]) |
| 330 | + return arr |
| 331 | + |
| 332 | + |
| 333 | +@pytest.fixture |
| 334 | +def allow_numcodecs_unpickle(monkeypatch: pytest.MonkeyPatch) -> None: |
| 335 | + """ |
| 336 | + Opt in to numcodecs' own gate on ``Pickle.decode``. |
| 337 | +
|
| 338 | + numcodecs refuses to unpickle unless the process opts in (GHSA-jqc8-qjfv-5m6r). zarr's |
| 339 | + opt-in is separate, so tests that read pickled data after opting in to zarr also need this. |
| 340 | + """ |
| 341 | + monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", True) |
| 342 | + |
| 343 | + |
| 344 | +def test_pickle_codec_read_disabled_by_default() -> None: |
| 345 | + """Reading data encoded with the pickle codec fails unless the user opts in.""" |
| 346 | + arr = _pickle_compressed_array() |
| 347 | + with pytest.raises( |
| 348 | + PickleCodecDisabledError, match=r"array\.allow_unsafe_pickle_decode" |
| 349 | + ) as excinfo: |
| 350 | + arr[:] |
| 351 | + # numcodecs has its own opt-in; the message must name it so users are not left guessing |
| 352 | + assert "ALLOW_UNSAFE_PICKLE_DECODE" in str(excinfo.value) |
| 353 | + |
| 354 | + |
| 355 | +def test_pickle_codec_read_does_not_unpickle() -> None: |
| 356 | + """A malicious chunk is not unpickled when the pickle codec is disabled.""" |
| 357 | + |
| 358 | + class Payload: |
| 359 | + def __reduce__(self) -> tuple[Any, ...]: |
| 360 | + return (exec, ("raise AssertionError('payload was unpickled')",)) |
| 361 | + |
| 362 | + meta = { |
| 363 | + "zarr_format": 2, |
| 364 | + "shape": [1], |
| 365 | + "chunks": [1], |
| 366 | + "dtype": "|O", |
| 367 | + "compressor": {"id": "pickle"}, |
| 368 | + "fill_value": None, |
| 369 | + "filters": [{"id": "vlen-utf8"}], |
| 370 | + "order": "C", |
| 371 | + } |
| 372 | + buffer_cls = default_buffer_prototype().buffer |
| 373 | + store = { |
| 374 | + ".zarray": buffer_cls.from_bytes(json.dumps(meta).encode()), |
| 375 | + "0": buffer_cls.from_bytes(pickle.dumps(Payload())), |
| 376 | + } |
| 377 | + arr = zarr.open_array(store=MemoryStore(store), mode="r", zarr_format=2) |
| 378 | + with pytest.raises(PickleCodecDisabledError): |
| 379 | + arr[:] |
| 380 | + |
| 381 | + |
| 382 | +def test_pickle_codec_zarr_opt_in_does_not_bypass_numcodecs( |
| 383 | + monkeypatch: pytest.MonkeyPatch, |
| 384 | +) -> None: |
| 385 | + """Opting in to zarr does not disable numcodecs' own gate on ``Pickle.decode``.""" |
| 386 | + monkeypatch.setattr(numcodecs.pickles, "ALLOW_UNSAFE_PICKLE_DECODE", False) |
| 387 | + arr = _pickle_compressed_array() |
| 388 | + with ( |
| 389 | + config.set({"array.allow_unsafe_pickle_decode": True}), |
| 390 | + pytest.raises(numcodecs.errors.UnsafeDecodeError), |
| 391 | + ): |
| 392 | + arr[:] |
| 393 | + |
| 394 | + |
| 395 | +@pytest.mark.usefixtures("allow_numcodecs_unpickle") |
| 396 | +def test_pickle_codec_read_enabled_via_config() -> None: |
| 397 | + arr = _pickle_compressed_array() |
| 398 | + with config.set({"array.allow_unsafe_pickle_decode": True}): |
| 399 | + np.testing.assert_array_equal(arr[:], np.array(["a", "bb", "ccc"])) |
| 400 | + |
| 401 | + |
| 402 | +@pytest.mark.usefixtures("allow_numcodecs_unpickle") |
| 403 | +@pytest.mark.parametrize(("env_value", "allowed"), [("True", True), ("false", False)]) |
| 404 | +def test_pickle_codec_read_enabled_via_env_var( |
| 405 | + monkeypatch: pytest.MonkeyPatch, env_value: str, allowed: bool |
| 406 | +) -> None: |
| 407 | + """ |
| 408 | + The opt-in can be set with an environment variable. Only a value that parses to ``True`` |
| 409 | + enables it: donfig keeps ``false`` as a (truthy) string, which must not enable it. |
| 410 | + """ |
| 411 | + arr = _pickle_compressed_array() |
| 412 | + monkeypatch.setenv("ZARR_ARRAY__ALLOW_UNSAFE_PICKLE_DECODE", env_value) |
| 413 | + config.refresh() |
| 414 | + if allowed: |
| 415 | + np.testing.assert_array_equal(arr[:], np.array(["a", "bb", "ccc"])) |
| 416 | + else: |
| 417 | + with pytest.raises(PickleCodecDisabledError): |
| 418 | + arr[:] |
| 419 | + |
| 420 | + |
| 421 | +def test_pickle_opt_in_does_not_affect_other_codecs() -> None: |
| 422 | + """Arrays that do not use the pickle codec are read without opting in.""" |
| 423 | + assert config.get("array.allow_unsafe_pickle_decode") is False |
| 424 | + arr = zarr.create_array( |
| 425 | + store={}, shape=(3,), dtype="<i4", zarr_format=2, filters=[Delta(dtype="<i4")] |
| 426 | + ) |
| 427 | + arr[:] = [1, 2, 3] |
| 428 | + np.testing.assert_array_equal(arr[:], [1, 2, 3]) |
0 commit comments