Skip to content

Commit e4abca3

Browse files
fix: accept lists of byte values as fill values for byte dtypes
The Zarr V3 spec allows fill values for byte data types to be given either as a base64-encoded string or as an array of byte values, but from_json_scalar only accepted the string form. NullTerminatedBytes, RawBytes, and VariableLengthBytes now accept a JSON list of integers in [0, 255] and reject non-byte elements with a clear TypeError. Refs zarr-developers#4453 Generated with Devin Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent f3d406f commit e4abca3

3 files changed

Lines changed: 68 additions & 4 deletions

File tree

‎src/zarr/core/dtype/npy/bytes.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
check_dtype_spec_v2,
1919
v3_unstable_dtype_warning,
2020
)
21-
from zarr.core.dtype.npy.common import check_json_str
21+
from zarr.core.dtype.npy.common import check_json_byte_list, check_json_str
2222
from zarr.core.dtype.wrapper import TBaseDType, ZDType
2323
from zarr.errors import DataTypeValidationError
2424

@@ -524,8 +524,10 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.bytes_:
524524

525525
if check_json_str(data):
526526
return self.to_native_dtype().type(base64.standard_b64decode(data.encode("ascii")))
527+
if check_json_byte_list(data):
528+
return self.to_native_dtype().type(bytes(data))
527529
raise TypeError(
528-
f"Invalid type: {data}. Expected a base64-encoded string."
530+
f"Invalid type: {data}. Expected a base64-encoded string or a list of byte values."
529531
) # pragma: no cover
530532

531533
@property
@@ -919,7 +921,11 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> np.void:
919921
"""
920922
if check_json_str(data):
921923
return self.to_native_dtype().type(base64.standard_b64decode(data))
922-
raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover
924+
if check_json_byte_list(data):
925+
return self.to_native_dtype().type(bytes(data))
926+
raise TypeError(
927+
f"Invalid type: {data}. Expected a base64-encoded string or a list of byte values."
928+
) # pragma: no cover
923929

924930
@property
925931
def item_size(self) -> int:
@@ -1200,7 +1206,11 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> bytes:
12001206

12011207
if check_json_str(data):
12021208
return base64.standard_b64decode(data.encode("ascii"))
1203-
raise TypeError(f"Invalid type: {data}. Expected a string.") # pragma: no cover
1209+
if check_json_byte_list(data):
1210+
return bytes(data)
1211+
raise TypeError(
1212+
f"Invalid type: {data}. Expected a base64-encoded string or a list of byte values."
1213+
) # pragma: no cover
12041214

12051215
def _check_scalar(self, data: object) -> TypeGuard[BytesLike]:
12061216
"""

‎src/zarr/core/dtype/npy/common.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -564,3 +564,22 @@ def check_json_bool(data: JSON) -> TypeGuard[bool]:
564564
True if the data is a boolean, False otherwise.
565565
"""
566566
return isinstance(data, bool)
567+
568+
569+
def check_json_byte_list(data: JSON) -> TypeGuard[list[int]]:
570+
"""
571+
Check if a JSON value is a list of byte values.
572+
573+
Parameters
574+
----------
575+
data : JSON
576+
The JSON value to check.
577+
578+
Returns
579+
-------
580+
Bool
581+
True if the data is a list of integers in the range [0, 255], False otherwise.
582+
"""
583+
return isinstance(data, list) and all(
584+
isinstance(e, int) and not isinstance(e, bool) and 0 <= e <= 255 for e in data
585+
)

‎tests/test_dtype/test_npy/test_bytes.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,38 @@ def test_invalid_size(zdtype_cls: type[NullTerminatedBytes] | type[RawBytes]) ->
171171
msg = f"length must be >= 1, got {length}."
172172
with pytest.raises(ValueError, match=msg):
173173
zdtype_cls(length=length)
174+
175+
176+
@pytest.mark.parametrize(
177+
("zdtype", "expected"),
178+
[
179+
(NullTerminatedBytes(length=2), np.bytes_(b"ab")),
180+
(RawBytes(length=2), np.void(b"ab")),
181+
(VariableLengthBytes(), b"ab"),
182+
],
183+
)
184+
def test_byte_list_fill_value(
185+
zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, expected: object
186+
) -> None:
187+
"""
188+
Test that byte data types accept a JSON list of byte values as a fill value,
189+
per the Zarr V3 spec, and that it matches the base64 form.
190+
"""
191+
from_byte_list = zdtype.from_json_scalar([97, 98], zarr_format=3)
192+
from_base64 = zdtype.from_json_scalar("YWI=", zarr_format=3)
193+
assert from_byte_list == expected
194+
assert from_byte_list == from_base64
195+
196+
197+
@pytest.mark.parametrize(
198+
"zdtype", [NullTerminatedBytes(length=2), RawBytes(length=2), VariableLengthBytes()]
199+
)
200+
@pytest.mark.parametrize("data", [[256], [-1], ["a"], [97.0], [True]])
201+
def test_byte_list_fill_value_invalid(
202+
zdtype: NullTerminatedBytes | RawBytes | VariableLengthBytes, data: list[object]
203+
) -> None:
204+
"""
205+
Test that a JSON list containing non-byte values is rejected as a fill value.
206+
"""
207+
with pytest.raises(TypeError, match="byte values"):
208+
zdtype.from_json_scalar(data, zarr_format=3)

0 commit comments

Comments
 (0)