Skip to content

Commit d21a5b4

Browse files
authored
fix(zarr-metadata): attributes may hold NaN and the infinities (#4420)
zarr-python writes attributes with the defaults of Python's `json` module, so an xarray `_FillValue` of NaN is stored as a bare `NaN`, and the model refused to read such a store. Attributes are user data, which the spec asks only to be JSON values and never interprets, so a non-finite number in them is now read, validated and written back as Python's `json` writes it. Anywhere the spec interprets a value one is still refused, and the store reader now says where instead of failing to decode the document. `refine_json` walks a value once, normalizing and judging it, and `refine_user_data` is the same walk where a non-finite number is the float it is. Only the validators say where user data is: they walk a node's attributes with `refine_user_data`, the store reader decodes and leaves the judging to them, and `to_key_value` validates what it writes with the `parse_*` its reader uses, so it no longer writes a document the reader refuses. Assisted-by: ClaudeCode:claude-opus-5-5
1 parent 4aeac75 commit d21a5b4

9 files changed

Lines changed: 636 additions & 69 deletions

File tree

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
Attributes are user data, and a non-finite number in them is read,
2+
validated and written back. zarr-python writes attributes with the
3+
defaults of Python's `json` module, so an xarray `_FillValue` or a CF
4+
`missing_value` of NaN is stored as a bare `NaN`, which RFC 8259 lacks;
5+
the model refused such a document and could not read the store at all.
6+
A node's `attributes`, a v2 `.zattrs`, the `.zattrs` entries of a v2
7+
`.zmetadata`, and the attributes of the nodes a v3 group's inline
8+
`consolidated_metadata` holds may now hold `NaN`, `Infinity` and
9+
`-Infinity`, in every document `validate_*`, `parse_*` and `is_*`, and in
10+
`from_key_value` and `to_key_value`. Wherever the spec interprets a value
11+
it spells those numbers as strings, so anywhere else a non-finite number
12+
is still refused, and the store reader now says where one is instead of
13+
failing to decode the document. `is_json`, `validate_json` and
14+
`parse_json` judge by RFC 8259 alone, so a document the node guards
15+
accept can hold a value `is_json` refuses.
16+
17+
`to_key_value` validates the document it writes as `from_key_value`
18+
validates what it reads, and raises `MetadataValidationError` with every
19+
problem instead of writing a document the reader refuses. A model built
20+
by hand is not validated, so the writer is where these are caught:
21+
22+
- a non-finite number outside attributes, which was refused before too;
23+
- a non-string attribute key, which was written as a string;
24+
- a value that is not JSON, which raised `TypeError`;
25+
- anything else the reader refuses, such as `dimension_names` longer than
26+
`shape`, which was written and then refused on read.
27+
28+
Any sequence is written as an array. A number too large for a float, such
29+
as `1e400`, is read as infinity, so in attributes it is now accepted and
30+
written back as `Infinity`.
31+
32+
Through `zarr_metadata.pydantic`, pydantic writes a non-finite attribute
33+
as `null` unless the enclosing model sets `ser_json_inf_nan="constants"`.

‎packages/zarr-metadata/src/zarr_metadata/model/_array.py‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,10 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3ArrayMetadata:
328328
def to_key_value(
329329
self, *, indent: int | str | None = None
330330
) -> Mapping[ZarrV3ArrayMetadataStoreKey, bytes]:
331-
return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)}
331+
# A model built by hand is not validated: its document is written only
332+
# if it reads as `from_json` reads one, and every problem is raised.
333+
document = parse_array_metadata_v3(self.to_json())
334+
return {ZARR_V3_ARRAY_METADATA_STORE_KEY: dump_store_json(document, indent=indent)}
332335

333336

334337
class ZarrV2ArrayMetadataPartial(TypedDict, total=False):
@@ -493,11 +496,16 @@ def to_key_value(
493496
) -> Mapping[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]:
494497
# Attributes live only in the sibling `.zattrs` file; the `.zarray`
495498
# document must exclude them. The `.zattrs` key is present exactly
496-
# when attributes are set (even empty) — UNSET emits no file.
497-
zarray = {k: v for k, v in self.to_json().items() if k != "attributes"}
499+
# when attributes are set (even empty) — UNSET emits no file. A model
500+
# built by hand is not validated: its document is written only if it
501+
# reads as `from_json` reads one, and every problem is raised.
502+
document = parse_array_metadata_v2(self.to_json())
503+
zarray = {k: v for k, v in document.items() if k != "attributes"}
498504
out: dict[ZarrV2ArrayMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = {
499505
ZARR_V2_ARRAY_METADATA_STORE_KEY: dump_store_json(zarray, indent=indent)
500506
}
501-
if self.attributes is not UNSET:
502-
out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(self.attributes, indent=indent)
507+
if "attributes" in document:
508+
out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(
509+
document["attributes"], indent=indent
510+
)
503511
return out

‎packages/zarr-metadata/src/zarr_metadata/model/_group.py‎

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,9 @@
2424
load_store_json,
2525
parse_group_metadata_v2,
2626
parse_group_metadata_v3,
27+
refine_json,
28+
refine_user_data,
2729
validate_consolidated_metadata_v3,
28-
validate_json,
2930
)
3031
from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY
3132
from zarr_metadata.v2.consolidated import ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY
@@ -182,7 +183,10 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV3GroupMetadata:
182183
def to_key_value(
183184
self, *, indent: int | str | None = None
184185
) -> Mapping[ZarrV3GroupMetadataStoreKey, bytes]:
185-
return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)}
186+
# A model built by hand is not validated: its document is written only
187+
# if it reads as `from_json` reads one, and every problem is raised.
188+
document = parse_group_metadata_v3(self.to_json())
189+
return {ZARR_V3_GROUP_METADATA_STORE_KEY: dump_store_json(document, indent=indent)}
186190

187191

188192
@dataclass(frozen=True, slots=True, kw_only=True)
@@ -339,13 +343,18 @@ def to_key_value(
339343
) -> Mapping[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes]:
340344
# Attributes live only in the sibling `.zattrs` file; the `.zgroup`
341345
# document must exclude them. The `.zattrs` key is present exactly
342-
# when attributes are set (even empty) — UNSET emits no file.
343-
zgroup = {k: v for k, v in self.to_json().items() if k != "attributes"}
346+
# when attributes are set (even empty) — UNSET emits no file. A model
347+
# built by hand is not validated: its document is written only if it
348+
# reads as `from_json` reads one, and every problem is raised.
349+
document = parse_group_metadata_v2(self.to_json())
350+
zgroup = {k: v for k, v in document.items() if k != "attributes"}
344351
out: dict[ZarrV2GroupMetadataStoreKey | ZarrV2AttributesStoreKey, bytes] = {
345352
ZARR_V2_GROUP_METADATA_STORE_KEY: dump_store_json(zgroup, indent=indent)
346353
}
347-
if self.attributes is not UNSET:
348-
out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(self.attributes, indent=indent)
354+
if "attributes" in document:
355+
out[ZARR_V2_ATTRIBUTES_STORE_KEY] = dump_store_json(
356+
document["attributes"], indent=indent
357+
)
349358
return out
350359

351360

@@ -399,6 +408,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata:
399408
"invalid_value",
400409
)
401410
)
411+
refined: dict[str, JSONValue] = {}
402412
if "metadata" in doc:
403413
entries = doc["metadata"]
404414
if not isinstance(entries, Mapping) or not all(
@@ -410,20 +420,20 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata:
410420
)
411421
)
412422
else:
423+
# Each entry is the document its key names: a `.zattrs` is
424+
# user data, and any other is JSON by RFC 8259.
413425
for key, value in cast("Mapping[str, object]", entries).items():
414-
problems.extend(
415-
ValidationProblem(
416-
("metadata", key, *problem.loc), problem.message, problem.kind
417-
)
418-
for problem in validate_json(value)
426+
refine = (
427+
refine_user_data
428+
if key.rsplit("/", 1)[-1] == ZARR_V2_ATTRIBUTES_STORE_KEY
429+
else refine_json
419430
)
431+
entry, found = refine(value, ("metadata", key))
432+
problems.extend(found)
433+
refined[key] = entry
420434
if len(problems) != 0:
421435
raise MetadataValidationError(problems)
422-
entries_tupled = cast(
423-
"dict[str, JSONValue]",
424-
arrays_to_tuples(dict(cast("Mapping[str, object]", doc["metadata"]))),
425-
)
426-
return cls(metadata=entries_tupled)
436+
return cls(metadata=refined)
427437

428438
@classmethod
429439
def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetadata:
@@ -432,6 +442,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ConsolidatedMetad
432442
def to_key_value(
433443
self, *, indent: int | str | None = None
434444
) -> Mapping[ZarrV2ConsolidatedMetadataStoreKey, bytes]:
435-
return {
436-
ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(self.to_json(), indent=indent)
437-
}
445+
# A model built by hand is not validated: it is written only as
446+
# `from_json` reads it, and every problem is raised.
447+
document = ZarrV2ConsolidatedMetadata.from_json(self.to_json()).to_json()
448+
return {ZARR_V2_CONSOLIDATED_METADATA_STORE_KEY: dump_store_json(document, indent=indent)}

0 commit comments

Comments
 (0)