|
| 1 | +"""Strategies for each codec this package defines that holds no other field. |
| 2 | +
|
| 3 | +A codec handed an array draws what fits the chunk: a `bytes` codec an |
| 4 | +`endian` for a type whose values take several bytes, a `transpose` an |
| 5 | +`order` with an entry for each axis. Handed `Chunk()`, nothing is known |
| 6 | +of the chunk, and what is drawn is what the codec's rules accept. |
| 7 | +""" |
| 8 | + |
| 9 | +from __future__ import annotations |
| 10 | + |
| 11 | +from typing import TYPE_CHECKING, Any, Final |
| 12 | + |
| 13 | +from hypothesis import strategies as st |
| 14 | + |
| 15 | +from zarr_metadata.definition._definition import AcceptedField |
| 16 | +from zarr_metadata.testing._data_type import ( |
| 17 | + COMPLEX_WIDTHS, |
| 18 | + FLOAT_WIDTHS, |
| 19 | + INTEGER_RANGES, |
| 20 | + FloatWidth, |
| 21 | + finite_floats, |
| 22 | + float_spellings, |
| 23 | +) |
| 24 | +from zarr_metadata.testing._strategy import EMPTY_CONFIGURATIONS, MAX_RANK, CodecStrategy |
| 25 | +from zarr_metadata.v3._definition import Chunk, storage_of |
| 26 | +from zarr_metadata.v3.codec._arithmetic import NOT_NUMBERS |
| 27 | +from zarr_metadata.v3.codec.blosc import BLOSC_CNAME, BLOSC_CODEC, BLOSC_NO_SHUFFLE, BLOSC_SHUFFLE |
| 28 | +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, ENDIANNESS |
| 29 | +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC |
| 30 | +from zarr_metadata.v3.codec.gzip import GZIP_CODEC |
| 31 | +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC |
| 32 | +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC |
| 33 | +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC, ZSTD_MAX_LEVEL, ZSTD_MIN_LEVEL |
| 34 | +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE |
| 35 | +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE |
| 36 | + |
| 37 | +if TYPE_CHECKING: |
| 38 | + from collections.abc import Mapping |
| 39 | + |
| 40 | + from zarr_metadata._common import JSONValue |
| 41 | + from zarr_metadata.v3._definition import CodecDefinition |
| 42 | + |
| 43 | +_BLOSC_MAX_TYPESIZE: Final = 255 |
| 44 | +"""The widest element c-blosc takes: `BLOSC_MAX_TYPESIZE`.""" |
| 45 | + |
| 46 | +_BLOSC_MAX_BLOCKSIZE: Final = 2**31 - 1 |
| 47 | +"""The largest block c-blosc takes, a signed 32-bit size; 0 lets it choose.""" |
| 48 | + |
| 49 | +_ENDIAN: Final = st.sampled_from(ENDIANNESS) |
| 50 | + |
| 51 | +_NUMPY_TIMES: Final = frozenset({NUMPY_DATETIME64_DATA_TYPE, NUMPY_TIMEDELTA64_DATA_TYPE}) |
| 52 | +"""The numpy time types, each stored as a signed 64-bit integer.""" |
| 53 | + |
| 54 | +_NUMPY_TIME_RANGE: Final = (-(2**63) + 1, 2**63 - 1) |
| 55 | +"""The signed 64-bit integers but the least, which numpy reads as `NaT`: the finite numpy times.""" |
| 56 | + |
| 57 | + |
| 58 | +def _bytes(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: |
| 59 | + """An `endian`, required for a data type whose values take several bytes; None for one whose values vary in size.""" |
| 60 | + storage = None if chunk.data_type is None else storage_of(chunk.data_type) |
| 61 | + if storage == "variable_length": |
| 62 | + return None |
| 63 | + if storage == "multi_byte": |
| 64 | + return st.fixed_dictionaries({"endian": _ENDIAN}) |
| 65 | + return st.fixed_dictionaries({}, optional={"endian": _ENDIAN}) |
| 66 | + |
| 67 | + |
| 68 | +@st.composite |
| 69 | +def _blosc(draw: st.DrawFn) -> Mapping[str, JSONValue]: |
| 70 | + """A compressor, a level, a shuffle and a block size; and a `typesize`, which only `noshuffle` may leave out. |
| 71 | +
|
| 72 | + Under `noshuffle` the spec says `typesize` "is ignored", so it is |
| 73 | + drawn there too, a spelling that is not the simplest. |
| 74 | + """ |
| 75 | + shuffle = draw(st.sampled_from(BLOSC_SHUFFLE)) |
| 76 | + typesize = st.integers(1, _BLOSC_MAX_TYPESIZE) |
| 77 | + required: dict[str, st.SearchStrategy[JSONValue]] = { |
| 78 | + "cname": st.sampled_from(BLOSC_CNAME), |
| 79 | + "clevel": st.integers(0, 9), |
| 80 | + "shuffle": st.just(shuffle), |
| 81 | + "blocksize": st.integers(0, _BLOSC_MAX_BLOCKSIZE), |
| 82 | + } |
| 83 | + if shuffle == BLOSC_NO_SHUFFLE: |
| 84 | + return draw(st.fixed_dictionaries(required, optional={"typesize": typesize})) |
| 85 | + return draw(st.fixed_dictionaries({**required, "typesize": typesize})) |
| 86 | + |
| 87 | + |
| 88 | +def _order(rank: int) -> st.SearchStrategy[Mapping[str, JSONValue]]: |
| 89 | + return st.permutations(range(rank)).map(lambda order: {"order": tuple(order)}) |
| 90 | + |
| 91 | + |
| 92 | +def _transpose(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]]: |
| 93 | + """A permutation of the chunk's axes; of some number of axes, when the chunk does not say.""" |
| 94 | + if chunk.rank is not None: |
| 95 | + return _order(chunk.rank) |
| 96 | + return st.integers(0, MAX_RANK).flatmap(_order) |
| 97 | + |
| 98 | + |
| 99 | +def _finite_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: |
| 100 | + """A finite float of `width` bits, as a number or the hex string of its bits.""" |
| 101 | + return finite_floats(width).flatmap(lambda value: float_spellings(value, width)) |
| 102 | + |
| 103 | + |
| 104 | +def _non_zero_float_values(width: FloatWidth) -> st.SearchStrategy[JSONValue]: |
| 105 | + """A finite, non-zero float of `width` bits, as a number or the hex string of its bits.""" |
| 106 | + return ( |
| 107 | + finite_floats(width) |
| 108 | + .filter(lambda value: value != 0) |
| 109 | + .flatmap(lambda value: float_spellings(value, width)) |
| 110 | + ) |
| 111 | + |
| 112 | + |
| 113 | +def _scale_offset(chunk: Chunk) -> st.SearchStrategy[Mapping[str, JSONValue]] | None: |
| 114 | + """An `offset` and a `scale`, each a finite value of the data type the codec is handed, the scale not zero; None for a data type whose values are not numbers. |
| 115 | +
|
| 116 | + Each is a fill value of the chunk's data type |
| 117 | + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/codecs/scale_offset/README.md?plain=1#L39-L43), |
| 118 | + a float's as a number or the hex string of its bits, a complex |
| 119 | + number's as a pair of them. Each is drawn finite, and the scale |
| 120 | + non-zero, so that decoding never divides by zero; a scale small |
| 121 | + enough that decoding overflows is still drawn. A numpy time is drawn |
| 122 | + as the signed 64-bit integer it is stored as, never `NaT`, since it |
| 123 | + "is compatible with any codec that supports arrays of signed 64-bit |
| 124 | + integers" |
| 125 | + (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/data-types/numpy.datetime64/README.md?plain=1#L120). |
| 126 | + A data type this package does not define, whose values it cannot |
| 127 | + draw, takes only the configuration with neither, the identity. |
| 128 | + Handed a chunk whose data type is unknown, the values are small |
| 129 | + non-negative integers, which every integer and float type holds. |
| 130 | + """ |
| 131 | + data_type = chunk.data_type |
| 132 | + definition = data_type.definition if isinstance(data_type, AcceptedField) else None |
| 133 | + if definition is None: |
| 134 | + offset: st.SearchStrategy[JSONValue] = st.integers(0, 127) |
| 135 | + scale: st.SearchStrategy[JSONValue] = st.integers(1, 127) |
| 136 | + elif definition.name in NOT_NUMBERS: |
| 137 | + return None |
| 138 | + elif definition in INTEGER_RANGES or definition in _NUMPY_TIMES: |
| 139 | + low, high = INTEGER_RANGES.get(definition, _NUMPY_TIME_RANGE) |
| 140 | + offset = st.integers(low, high) |
| 141 | + scale = st.integers(low, high).filter(lambda value: value != 0) |
| 142 | + elif definition in FLOAT_WIDTHS: |
| 143 | + offset = _finite_float_values(FLOAT_WIDTHS[definition]) |
| 144 | + scale = _non_zero_float_values(FLOAT_WIDTHS[definition]) |
| 145 | + elif definition in COMPLEX_WIDTHS: |
| 146 | + width = COMPLEX_WIDTHS[definition] |
| 147 | + offset = st.tuples(_finite_float_values(width), _finite_float_values(width)) |
| 148 | + # Non-zero in either part is a non-zero complex number. |
| 149 | + scale = st.one_of( |
| 150 | + st.tuples(_non_zero_float_values(width), _finite_float_values(width)), |
| 151 | + st.tuples(_finite_float_values(width), _non_zero_float_values(width)), |
| 152 | + ) |
| 153 | + else: |
| 154 | + return EMPTY_CONFIGURATIONS |
| 155 | + return st.fixed_dictionaries({}, optional={"offset": offset, "scale": scale}) |
| 156 | + |
| 157 | + |
| 158 | +def _unchunked(configurations: st.SearchStrategy[Mapping[str, JSONValue]]) -> CodecStrategy: |
| 159 | + """A codec whose configurations do not depend on the chunk it is handed.""" |
| 160 | + return CodecStrategy(configurations=lambda _chunk: configurations) |
| 161 | + |
| 162 | + |
| 163 | +CODEC_STRATEGIES: Final[Mapping[CodecDefinition[Any], CodecStrategy]] = { |
| 164 | + BLOSC_CODEC: _unchunked(_blosc()), |
| 165 | + BYTES_CODEC: CodecStrategy(configurations=_bytes), |
| 166 | + CRC32C_CODEC: _unchunked(EMPTY_CONFIGURATIONS), |
| 167 | + GZIP_CODEC: _unchunked(st.fixed_dictionaries({"level": st.integers(0, 9)})), |
| 168 | + SCALE_OFFSET_CODEC: CodecStrategy(configurations=_scale_offset), |
| 169 | + TRANSPOSE_CODEC: CodecStrategy(configurations=_transpose), |
| 170 | + ZSTD_CODEC: _unchunked( |
| 171 | + st.fixed_dictionaries( |
| 172 | + {"level": st.integers(ZSTD_MIN_LEVEL, ZSTD_MAX_LEVEL)}, |
| 173 | + optional={"checksum": st.booleans()}, |
| 174 | + ) |
| 175 | + ), |
| 176 | +} |
| 177 | +"""A strategy for each codec that holds no other field.""" |
| 178 | + |
| 179 | +__all__ = ["CODEC_STRATEGIES"] |
0 commit comments