|
| 1 | +"""JSON values, and the problems found with them: where reading starts, below every layer. |
| 2 | +
|
| 3 | +`refine_json` is the one walk that normalizes and judges a value, and |
| 4 | +`refine_user_data` the same walk over user data, where a non-finite number |
| 5 | +is the float it is. |
| 6 | +`ValidationProblem` and `MetadataValidationError` are what every |
| 7 | +layer reports with: the checker, the definitions and the model alike. |
| 8 | +""" |
| 9 | + |
| 10 | +from __future__ import annotations |
| 11 | + |
| 12 | +import math |
| 13 | +from collections.abc import Mapping, Sequence |
| 14 | +from dataclasses import dataclass |
| 15 | +from typing import Literal, TypeGuard, cast, get_args |
| 16 | + |
| 17 | +from zarr_metadata._common import JSONValue |
| 18 | + |
| 19 | +ProblemKind = Literal["missing_key", "invalid_type", "invalid_value", "invalid_json", "unknown_key"] |
| 20 | +"""Machine-readable classification of a `ValidationProblem`. |
| 21 | +
|
| 22 | +- `missing_key`: a required key (document key or store key) is absent. |
| 23 | +- `invalid_type`: a value has the wrong structural type (e.g. a string where |
| 24 | + a mapping is required, a non-JSON-serializable object). |
| 25 | +- `invalid_value`: a value has an acceptable type but an invalid content |
| 26 | + (e.g. `zarr_format: 2` in a v3 document, `order: "Q"`). |
| 27 | +- `invalid_json`: bytes that do not decode as JSON. |
| 28 | +- `unknown_key`: a key an object's type does not declare, where the type |
| 29 | + says it is closed, as a closed TypedDict does. Whether a Zarr |
| 30 | + configuration is closed is rarely said (zarr-developers/zarr-specs#270 |
| 31 | + has been open since 2023), and many readers refuse such a key. It gets |
| 32 | + a kind of its own so that a caller who tolerates it can tell it from a |
| 33 | + wrong value, and so that it never masks the other findings about the |
| 34 | + same object. |
| 35 | +""" |
| 36 | + |
| 37 | + |
| 38 | +@dataclass(frozen=True, slots=True) |
| 39 | +class ValidationProblem: |
| 40 | + """A single problem found in a value: where it is, what is wrong, and what kind of wrong. |
| 41 | +
|
| 42 | + `loc` is the path from the root of what was judged to the offending |
| 43 | + value, e.g. `("codecs", 0, "name")` in a document, and an empty `loc` |
| 44 | + refers to that root. |
| 45 | + `kind` classifies the failure mode for programmatic dispatch; `message` |
| 46 | + is the human-readable description. |
| 47 | + """ |
| 48 | + |
| 49 | + loc: tuple[str | int, ...] |
| 50 | + message: str |
| 51 | + kind: ProblemKind |
| 52 | + |
| 53 | + def __post_init__(self) -> None: |
| 54 | + # The runtime half of the annotations: a rule written without a type |
| 55 | + # checker, as an extension's may be, fails where it builds a problem |
| 56 | + # rather than reporting one at a location that is not one. |
| 57 | + loc = cast("object", self.loc) |
| 58 | + if not isinstance(loc, tuple) or not all( |
| 59 | + isinstance(part, str) or (isinstance(part, int) and not isinstance(part, bool)) |
| 60 | + for part in cast("tuple[object, ...]", loc) |
| 61 | + ): |
| 62 | + msg = f"a ValidationProblem's loc is a tuple of keys and indices, got {loc!r}" |
| 63 | + raise TypeError(msg) |
| 64 | + message = cast("object", self.message) |
| 65 | + if not isinstance(message, str): |
| 66 | + msg = f"a ValidationProblem's message is a string, got {message!r}" |
| 67 | + raise TypeError(msg) |
| 68 | + kind = cast("object", self.kind) |
| 69 | + if not isinstance(kind, str) or kind not in get_args(ProblemKind): |
| 70 | + msg = f"a ValidationProblem's kind is one of {get_args(ProblemKind)!r}, got {kind!r}" |
| 71 | + raise TypeError(msg) |
| 72 | + |
| 73 | + def __str__(self) -> str: |
| 74 | + location = ".".join(str(part) for part in self.loc) if self.loc else "<root>" |
| 75 | + return f"{location}: {self.message}" |
| 76 | + |
| 77 | + |
| 78 | +class MetadataValidationError(ValueError): |
| 79 | + """Raised when a value fails validation, by the entry points that raise rather than report. |
| 80 | +
|
| 81 | + Carries every problem found (not just the first) in `.problems`, as an |
| 82 | + immutable tuple: a raised error is a finished report, and a caller |
| 83 | + inspecting it must not be able to edit the record. |
| 84 | + """ |
| 85 | + |
| 86 | + problems: tuple[ValidationProblem, ...] |
| 87 | + |
| 88 | + def __init__(self, problems: Sequence[ValidationProblem]) -> None: |
| 89 | + self.problems = tuple(problems) |
| 90 | + for entry in cast("tuple[object, ...]", self.problems): |
| 91 | + # The runtime half of the annotation: a caller that is not |
| 92 | + # type-checked, and hands over anything else, fails here |
| 93 | + # rather than far away, where a `loc` is read off it. |
| 94 | + if not isinstance(entry, ValidationProblem): |
| 95 | + msg = ( |
| 96 | + "MetadataValidationError takes ValidationProblem values, " |
| 97 | + f"got {type(entry).__name__}" |
| 98 | + ) |
| 99 | + raise TypeError(msg) |
| 100 | + super().__init__("\n".join(str(problem) for problem in self.problems)) |
| 101 | + |
| 102 | + def __reduce__( |
| 103 | + self, |
| 104 | + ) -> tuple[ |
| 105 | + type[MetadataValidationError], tuple[tuple[ValidationProblem, ...]], dict[str, object] |
| 106 | + ]: |
| 107 | + # An exception pickles and copies as its class called with its |
| 108 | + # `args`, which here are the message; it is built from its problems, |
| 109 | + # and the rest of its state -- its notes among it -- follows. |
| 110 | + return (type(self), (self.problems,), self.__dict__) |
| 111 | + |
| 112 | + |
| 113 | +def prefixed( |
| 114 | + loc_head: str | int, problems: Sequence[ValidationProblem] |
| 115 | +) -> tuple[ValidationProblem, ...]: |
| 116 | + """Prepend `loc_head` to the `loc` of every problem (for nested validators).""" |
| 117 | + return tuple(ValidationProblem((loc_head, *p.loc), p.message, p.kind) for p in problems) |
| 118 | + |
| 119 | + |
| 120 | +def validate_json(value: object) -> tuple[ValidationProblem, ...]: |
| 121 | + """Return every reason `value` is not JSON-serializable (recursively): `refine_json`'s problems.""" |
| 122 | + return refine_json(value)[1] |
| 123 | + |
| 124 | + |
| 125 | +def refine_json( |
| 126 | + value: object, loc: tuple[str | int, ...] = () |
| 127 | +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: |
| 128 | + """`value` as JSON with arrays as tuples, or None with every reason it is not JSON. |
| 129 | +
|
| 130 | + One walk normalizes and judges: a mapping becomes a `dict` with |
| 131 | + string keys, a sequence a tuple, and a float must be finite. A value |
| 132 | + that is not JSON is None, with the problems located at the leaves |
| 133 | + that are not. JSON's `null` is None too, with no problem, so whether |
| 134 | + `value` is JSON is whether there are problems. |
| 135 | + """ |
| 136 | + return _refine(value, loc, finite=True) |
| 137 | + |
| 138 | + |
| 139 | +def refine_user_data( |
| 140 | + value: object, loc: tuple[str | int, ...] = () |
| 141 | +) -> tuple[JSONValue | None, tuple[ValidationProblem, ...]]: |
| 142 | + """User data refined as `refine_json` refines JSON, a non-finite number being the float it is. |
| 143 | +
|
| 144 | + A node's attributes are user data. The spec asks only that each be a |
| 145 | + JSON value and interprets none, and zarr-python writes them with the |
| 146 | + defaults of Python's `json` module, so an attribute can hold `NaN`, |
| 147 | + `Infinity` or `-Infinity` -- xarray's `_FillValue`, a CF |
| 148 | + `missing_value` -- which RFC 8259 lacks. Wherever the spec interprets |
| 149 | + a value it spells those numbers as strings, so a validator walks what |
| 150 | + it interprets with `refine_json`, and only user data with this. |
| 151 | + """ |
| 152 | + return _refine(value, loc, finite=False) |
| 153 | + |
| 154 | + |
| 155 | +_Refined = tuple[JSONValue | None, tuple[ValidationProblem, ...]] |
| 156 | + |
| 157 | + |
| 158 | +def _refine(value: object, loc: tuple[str | int, ...], *, finite: bool) -> _Refined: |
| 159 | + """`refine_json`, a non-finite number being JSON unless `finite`.""" |
| 160 | + if isinstance(value, float): |
| 161 | + if not finite or math.isfinite(value): |
| 162 | + return value, () |
| 163 | + return None, ( |
| 164 | + ValidationProblem(loc, f"non-finite float {value!r} is not JSON", "invalid_value"), |
| 165 | + ) |
| 166 | + if isinstance(value, (str, int, bool)) or value is None: |
| 167 | + return value, () |
| 168 | + if isinstance(value, Mapping): |
| 169 | + # Walked here rather than through `_refine_members`, so that each |
| 170 | + # level of nesting costs one frame, as deep as the interpreter goes. |
| 171 | + members: dict[str, JSONValue] = {} |
| 172 | + found_in_members: list[ValidationProblem] = [] |
| 173 | + for key, item in cast("Mapping[object, object]", value).items(): |
| 174 | + if not isinstance(key, str): |
| 175 | + found_in_members.append( |
| 176 | + ValidationProblem(loc, f"non-string key {key!r} in JSON object", "invalid_type") |
| 177 | + ) |
| 178 | + continue |
| 179 | + member, found = _refine(item, (*loc, key), finite=finite) |
| 180 | + found_in_members.extend(found) |
| 181 | + if len(found) == 0: |
| 182 | + members[key] = member |
| 183 | + return (members if len(found_in_members) == 0 else None), tuple(found_in_members) |
| 184 | + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): |
| 185 | + entries: list[JSONValue] = [] |
| 186 | + found_in_entries: list[ValidationProblem] = [] |
| 187 | + for index, item in enumerate(cast("Sequence[object]", value)): |
| 188 | + entry, found = _refine(item, (*loc, index), finite=finite) |
| 189 | + found_in_entries.extend(found) |
| 190 | + if len(found) == 0: |
| 191 | + entries.append(entry) |
| 192 | + return (tuple(entries) if len(found_in_entries) == 0 else None), tuple(found_in_entries) |
| 193 | + return None, ( |
| 194 | + ValidationProblem(loc, f"not a JSON-serializable value: {value!r}", "invalid_type"), |
| 195 | + ) |
| 196 | + |
| 197 | + |
| 198 | +def is_canonical_json(value: object, *, finite: bool = True) -> TypeGuard[JSONValue]: |
| 199 | + """Whether `value` already uses the concrete containers in `JSONValue`. |
| 200 | +
|
| 201 | + A non-finite number counts only when `finite` is false, as a document's |
| 202 | + guard passes it: where one may be is the document's validator's to say. |
| 203 | + """ |
| 204 | + if isinstance(value, float): |
| 205 | + return not finite or math.isfinite(value) |
| 206 | + if isinstance(value, (str, int, bool)) or value is None: |
| 207 | + return True |
| 208 | + if isinstance(value, (list, tuple)): |
| 209 | + sequence = cast("list[object] | tuple[object, ...]", value) |
| 210 | + return all(is_canonical_json(item, finite=finite) for item in sequence) |
| 211 | + if isinstance(value, dict): |
| 212 | + mapping = cast("dict[object, object]", value) |
| 213 | + return all( |
| 214 | + isinstance(key, str) and is_canonical_json(item, finite=finite) |
| 215 | + for key, item in mapping.items() |
| 216 | + ) |
| 217 | + return False |
| 218 | + |
| 219 | + |
| 220 | +def is_json(value: object) -> TypeGuard[JSONValue]: |
| 221 | + """Whether `value` is a canonical JSON structure (recursively).""" |
| 222 | + return is_canonical_json(value) |
| 223 | + |
| 224 | + |
| 225 | +def parse_json(value: object) -> JSONValue: |
| 226 | + """Return a canonical `JSONValue`, or raise `MetadataValidationError`.""" |
| 227 | + refined, problems = refine_json(value) |
| 228 | + if len(problems) != 0: |
| 229 | + raise MetadataValidationError(problems) |
| 230 | + return refined |
| 231 | + |
| 232 | + |
| 233 | +def arrays_to_tuples(obj: object) -> object: |
| 234 | + """Recursively materialize mappings and convert array-like values to tuples.""" |
| 235 | + if isinstance(obj, Sequence) and not isinstance(obj, (str, bytes, bytearray)): |
| 236 | + sequence = cast("Sequence[object]", obj) |
| 237 | + converted_sequence = tuple(arrays_to_tuples(item) for item in sequence) |
| 238 | + if isinstance(obj, tuple) and all( |
| 239 | + converted is original |
| 240 | + for converted, original in zip(converted_sequence, sequence, strict=True) |
| 241 | + ): |
| 242 | + return sequence |
| 243 | + return converted_sequence |
| 244 | + if isinstance(obj, Mapping): |
| 245 | + mapping = cast("Mapping[object, object]", obj) |
| 246 | + converted: dict[object, object] = { |
| 247 | + key: arrays_to_tuples(value) for key, value in mapping.items() |
| 248 | + } |
| 249 | + if isinstance(obj, dict) and all(converted[key] is value for key, value in mapping.items()): |
| 250 | + return mapping |
| 251 | + return converted |
| 252 | + return obj |
| 253 | + |
| 254 | + |
| 255 | +__all__ = [ |
| 256 | + "MetadataValidationError", |
| 257 | + "ProblemKind", |
| 258 | + "ValidationProblem", |
| 259 | + "arrays_to_tuples", |
| 260 | + "is_canonical_json", |
| 261 | + "is_json", |
| 262 | + "parse_json", |
| 263 | + "prefixed", |
| 264 | + "refine_json", |
| 265 | + "refine_user_data", |
| 266 | + "validate_json", |
| 267 | +] |
0 commit comments