Skip to content

Commit 34ba848

Browse files
committed
fix(zarr-metadata): bytes are not a JSON array
Four checks asked only whether a value was a `Sequence` other than a `str`, and `bytes` is one, so an empty `bytes` or `bytearray` passed as an array with no items: `parse_array_metadata_v3` returned `storage_transformers=b""` typed as a tuple, and the v2 readers took `dtype=b""` and `filters=b""`. `_is_array` says what a JSON array is, and those four checks and `_is_int_sequence` ask it. Both are `TypeGuard`s, which removes the casts that restated what they checked. Assisted-by: ClaudeCode:claude-opus-5-5
1 parent d73427e commit 34ba848

3 files changed

Lines changed: 97 additions & 46 deletions

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
A `bytes` or `bytearray` is not a JSON array. Where a document expects an
2+
array, an empty one passed as an array with no items:
3+
`parse_array_metadata_v3` returned `storage_transformers=b""`, and the v2
4+
readers took `dtype=b""` and `filters=b""`, then failed when the model was
5+
written. The validators and `parse_*` now refuse one at `codecs`,
6+
`storage_transformers`, `dimension_names`, a v2 `dtype` and its field
7+
records, and v2 `filters`.

‎packages/zarr-metadata/src/zarr_metadata/model/_validation.py‎

Lines changed: 44 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -426,19 +426,25 @@ def parse_metadata_field_v3(value: object) -> ZarrV3MetadataFieldJSON:
426426
return cast(ZarrV3MetadataFieldJSON, normalized)
427427

428428

429-
def _is_int_sequence(value: object) -> bool:
430-
"""Whether `value` is a non-string sequence of integers.
429+
def _is_array(value: object) -> TypeGuard[Sequence[object]]:
430+
"""Whether `value` reads as a JSON array: a sequence that is not a string or bytes.
431+
432+
`str`, `bytes` and `bytearray` are sequences to Python, and none of them
433+
is an array to JSON. A `TypeGuard`, not a `TypeIs`: a `str` is a
434+
`Sequence[object]` this says no to.
435+
"""
436+
return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray))
437+
438+
439+
def _is_int_sequence(value: object) -> TypeGuard[Sequence[int]]:
440+
"""Whether `value` is a JSON array of integers.
431441
432442
JSON booleans decode to `bool`, which is an `int` subclass in Python but
433-
is not an integer in a metadata document, so booleans are excluded.
443+
is not an integer in a metadata document, so booleans are excluded. A
444+
`TypeGuard`, not a `TypeIs`: `bytes` is a `Sequence[int]` this says no to.
434445
"""
435-
return (
436-
not isinstance(value, (str, bytes, bytearray))
437-
and isinstance(value, Sequence)
438-
and all(
439-
isinstance(item, int) and not isinstance(item, bool)
440-
for item in cast("Sequence[object]", value)
441-
)
446+
return _is_array(value) and all(
447+
isinstance(item, int) and not isinstance(item, bool) for item in value
442448
)
443449

444450

@@ -452,7 +458,7 @@ def _validate_dim_sequence(doc: Mapping[object, object], key: str) -> tuple[Vali
452458
value = doc[key]
453459
if not _is_int_sequence(value):
454460
return (ValidationProblem((key,), "expected a sequence of int", "invalid_type"),)
455-
if any(item < 0 for item in cast("Sequence[int]", value)):
461+
if any(item < 0 for item in value):
456462
return (ValidationProblem((key,), "expected non-negative integers", "invalid_value"),)
457463
return ()
458464

@@ -467,19 +473,16 @@ def _is_dtype_v2(value: object) -> bool:
467473
"""
468474
if isinstance(value, str):
469475
return True
470-
if not isinstance(value, Sequence):
476+
if not _is_array(value):
471477
return False
472-
for record in cast("Sequence[object]", value):
473-
if isinstance(record, str) or not isinstance(record, Sequence):
474-
return False
475-
fields = cast("Sequence[object]", record)
476-
if len(fields) not in (2, 3):
478+
for record in value:
479+
if not _is_array(record) or len(record) not in (2, 3):
477480
return False
478-
if not isinstance(fields[0], str):
481+
if not isinstance(record[0], str):
479482
return False
480-
if not _is_dtype_v2(fields[1]):
483+
if not _is_dtype_v2(record[1]):
481484
return False
482-
if len(fields) == 3 and not _is_int_sequence(fields[2]):
485+
if len(record) == 3 and not _is_int_sequence(record[2]):
483486
return False
484487
return True
485488

@@ -619,16 +622,16 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]:
619622
for key in ("codecs", "storage_transformers"):
620623
if key in doc:
621624
entries = doc[key]
622-
if isinstance(entries, str) or not isinstance(entries, Sequence):
625+
if not _is_array(entries):
623626
problems.append(ValidationProblem((key,), "expected a sequence", "invalid_type"))
624627
else:
625-
if key == "codecs" and len(cast("Sequence[object]", entries)) == 0:
628+
if key == "codecs" and len(entries) == 0:
626629
problems.append(
627630
ValidationProblem(
628631
("codecs",), "expected at least one codec", "invalid_value"
629632
)
630633
)
631-
for index, entry in enumerate(cast("Sequence[object]", entries)):
634+
for index, entry in enumerate(entries):
632635
problems.extend(_prefix(key, _prefix(index, validate_metadata_field_v3(entry))))
633636
if "attributes" in doc:
634637
problems.extend(_validate_attributes(doc["attributes"]))
@@ -637,21 +640,18 @@ def validate_array_metadata_v3(value: object) -> tuple[ValidationProblem, ...]:
637640
# field-level loc, not per-bad-item locs; per-index locs are reserved for
638641
# the metadata-field lists (codecs, storage_transformers).
639642
names = doc["dimension_names"]
640-
if isinstance(names, str) or not isinstance(names, Sequence):
643+
shape = doc.get("shape")
644+
if not _is_array(names):
641645
problems.append(
642646
ValidationProblem(("dimension_names",), "expected a sequence", "invalid_type")
643647
)
644-
elif not all(
645-
item is None or isinstance(item, str) for item in cast("Sequence[object]", names)
646-
):
648+
elif not all(item is None or isinstance(item, str) for item in names):
647649
problems.append(
648650
ValidationProblem(
649651
("dimension_names",), "expected items of str or None", "invalid_type"
650652
)
651653
)
652-
elif _is_int_sequence(doc.get("shape")) and len(cast("Sequence[object]", names)) != len(
653-
cast("Sequence[int]", doc["shape"])
654-
):
654+
elif _is_int_sequence(shape) and len(names) != len(shape):
655655
problems.append(
656656
ValidationProblem(
657657
("dimension_names",),
@@ -704,22 +704,22 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]:
704704
chunks_problems = _validate_dim_sequence(doc, "chunks")
705705
problems.extend(shape_problems)
706706
problems.extend(chunks_problems)
707+
shape = doc.get("shape")
708+
chunks = doc.get("chunks")
707709
if (
708710
len(shape_problems) == 0
709711
and len(chunks_problems) == 0
710-
and _is_int_sequence(doc.get("shape"))
711-
and _is_int_sequence(doc.get("chunks"))
712+
and _is_int_sequence(shape)
713+
and _is_int_sequence(chunks)
714+
and len(shape) != len(chunks)
712715
):
713-
shape = cast("Sequence[int]", doc["shape"])
714-
chunks = cast("Sequence[int]", doc["chunks"])
715-
if len(shape) != len(chunks):
716-
problems.append(
717-
ValidationProblem(
718-
("chunks",),
719-
"expected the same number of dimensions as shape",
720-
"invalid_value",
721-
)
716+
problems.append(
717+
ValidationProblem(
718+
("chunks",),
719+
"expected the same number of dimensions as shape",
720+
"invalid_value",
722721
)
722+
)
723723
if "dtype" in doc and not _is_dtype_v2(doc["dtype"]):
724724
problems.append(
725725
ValidationProblem(
@@ -741,9 +741,7 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]:
741741
if "filters" in doc:
742742
filters = doc["filters"]
743743
if filters is not None and (
744-
isinstance(filters, str)
745-
or not isinstance(filters, Sequence)
746-
or not all(_is_codec_v2(item) for item in cast("Sequence[object]", filters))
744+
not _is_array(filters) or not all(_is_codec_v2(item) for item in filters)
747745
):
748746
problems.append(
749747
ValidationProblem(
@@ -752,10 +750,10 @@ def validate_array_metadata_v2(value: object) -> tuple[ValidationProblem, ...]:
752750
"invalid_type",
753751
)
754752
)
755-
elif filters is not None:
753+
elif _is_array(filters):
756754
# "A list of JSON objects providing codec configurations, or
757755
# null" (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L76-L79): an empty list is a list.
758-
for index, item in enumerate(cast("Sequence[object]", filters)):
756+
for index, item in enumerate(filters):
759757
problems.extend(_prefix("filters", _prefix(index, validate_json(item))))
760758
if "dimension_separator" in doc and doc["dimension_separator"] not in (".", "/"):
761759
problems.append(

‎packages/zarr-metadata/tests/model/test_array.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,6 +1367,52 @@ def test_v2_dtype_must_be_string_or_records() -> None:
13671367
assert [(p.loc, p.kind) for p in problems] == [(("dtype",), "invalid_type")]
13681368

13691369

1370+
@pytest.mark.parametrize(
1371+
("validate", "document", "loc"),
1372+
[
1373+
(
1374+
validate_array_metadata_v3,
1375+
{**ZarrV3ArrayMetadata.create_default().to_json(), "codecs": b""},
1376+
("codecs",),
1377+
),
1378+
(
1379+
validate_array_metadata_v3,
1380+
{**ZarrV3ArrayMetadata.create_default().to_json(), "storage_transformers": bytearray()},
1381+
("storage_transformers",),
1382+
),
1383+
(
1384+
validate_array_metadata_v3,
1385+
{**ZarrV3ArrayMetadata.create_default(shape=()).to_json(), "dimension_names": b""},
1386+
("dimension_names",),
1387+
),
1388+
(
1389+
validate_array_metadata_v2,
1390+
{**ZarrV2ArrayMetadata.create_default().to_json(), "dtype": b""},
1391+
("dtype",),
1392+
),
1393+
(
1394+
validate_array_metadata_v2,
1395+
{**ZarrV2ArrayMetadata.create_default().to_json(), "dtype": (("f0", b""),)},
1396+
("dtype",),
1397+
),
1398+
(
1399+
validate_array_metadata_v2,
1400+
{**ZarrV2ArrayMetadata.create_default().to_json(), "filters": b""},
1401+
("filters",),
1402+
),
1403+
],
1404+
ids=["codecs", "storage-transformers", "dimension-names", "dtype", "dtype-record", "filters"],
1405+
)
1406+
def test_error_bytes_are_not_an_array(
1407+
validate: Callable[[object], tuple[ValidationProblem, ...]],
1408+
document: dict[str, object],
1409+
loc: tuple[str | int, ...],
1410+
) -> None:
1411+
"""`bytes` is a sequence to Python and not an array to JSON: where a document
1412+
expects an array, an empty one no longer passes as one with no items."""
1413+
assert [(p.loc, p.kind) for p in validate(document)] == [(loc, "invalid_type")]
1414+
1415+
13701416
def test_v2_structured_dtype_records_accepted() -> None:
13711417
"""A structured v2 dtype (field records, optionally nested/shaped) validates."""
13721418
dtype = (("a", "<i4"), ("b", (("c", "|u1"),)), ("d", "<f8", (2, 2)))

0 commit comments

Comments
 (0)