Skip to content

Commit e6658d2

Browse files
d-v-bclaude
andauthored
fix(metadata): read Zarr V3.1 short-hand extension names (zarr-developers#4506)
* fix(metadata): read Zarr V3.1 short-hand extension names Zarr V3.1 lets an extension definition that needs no configuration be written as a bare name: `"crc32c"` is equivalent to `{"name": "crc32c"}`. Reading such a document raised `TypeError: Expected dict, got <class 'str'>`. `expand_short_hand_name` expands a bare name at the parser of each array extension point that took only the object form: codecs (including codecs nested in a sharding codec), chunk_grid, chunk_key_encoding and storage_transformers. data_type already accepted a bare name. A bare name for an extension that requires a configuration (`"regular"`, `"gzip"`) is still rejected, with the missing-configuration error. Metadata is still written in the object form, which Zarr V3.0 readers require. Refs zarr-developers#3188 Assisted-by: ClaudeCode:claude-opus-5-5 * docs(changes): add changelog fragment for short-hand extension names Assisted-by: ClaudeCode:claude-opus-5-5 * docs(changes): renumber short-hand names fragment to the upstream PR Assisted-by: ClaudeCode:claude-opus-5-5 * fix(chunk-key-encoding): accept a bare encoding name in ChunkKeyEncodingLike parse_chunk_key_encoding now expands short-hand names, so array creation functions accept `chunk_key_encoding="v2"`. Declare `str` in the type, drop the type ignore it made unnecessary, and test the chunk keys each short-hand name produces. Correct the fragment: a short-hand storage transformer was kept as a bare string on main, not rejected. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 <[email protected]> --------- Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 6595318 commit e6658d2

6 files changed

Lines changed: 157 additions & 3 deletions

File tree

‎changes/4506.bugfix.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Array metadata that uses Zarr V3.1 short-hand extension names, such as
2+
`"codecs": ["bytes", "crc32c"]` or `"chunk_key_encoding": "default"`, can now be
3+
read. A bare name is read as an object with only a `name` key. Previously, a
4+
short-hand codec or chunk key encoding raised
5+
`TypeError: Expected dict, got <class 'str'>`, and a short-hand storage
6+
transformer was kept as a bare string. Metadata is still written in the object
7+
form, which Zarr V3.0 readers require. The `chunk_key_encoding` parameter of
8+
`create_array` and the other array creation functions also accepts a bare name,
9+
such as `chunk_key_encoding="v2"`.

‎src/zarr/core/chunk_key_encodings.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from zarr.core.common import (
1212
JSON,
1313
NamedConfig,
14+
expand_short_hand_name,
1415
parse_named_configuration,
1516
)
1617
from zarr.core.json_parse import parse_field
@@ -62,7 +63,7 @@ def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str:
6263

6364

6465
type ChunkKeyEncodingLike = (
65-
dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any]
66+
str | dict[str, JSON] | ChunkKeyEncodingParams | ChunkKeyEncoding | NamedConfig[str, Any]
6667
)
6768

6869

@@ -112,6 +113,8 @@ def parse_chunk_key_encoding(data: ChunkKeyEncodingLike) -> ChunkKeyEncoding:
112113
if isinstance(data, ChunkKeyEncoding):
113114
return data
114115

116+
data = expand_short_hand_name(data)
117+
115118
# handle ChunkKeyEncodingParams
116119
if "name" in data and "separator" in data:
117120
data = {"name": data["name"], "configuration": {"separator": data["separator"]}} # type: ignore[typeddict-item]

‎src/zarr/core/common.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,27 @@ def parse_name(data: JSON, expected: str | None = None) -> str:
220220
raise ValueError(f"Expected '{expected}'. Got {data} instead.")
221221

222222

223+
@overload
224+
def expand_short_hand_name(data: str) -> dict[str, JSON]: ...
225+
226+
227+
@overload
228+
def expand_short_hand_name[T](data: T) -> T: ...
229+
230+
231+
def expand_short_hand_name(data: object) -> object:
232+
"""
233+
Expand the short-hand name form of a Zarr V3 extension definition: a bare string `name`
234+
is equivalent to the object `{"name": name}`. Any other value is returned unchanged.
235+
236+
Zarr V3.1 permits the short-hand form at every extension point of array metadata. See
237+
https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#short-hand-names
238+
"""
239+
if isinstance(data, str):
240+
return {"name": data}
241+
return data
242+
243+
223244
def parse_configuration(data: JSON) -> JSON:
224245
if not isinstance(data, dict):
225246
raise TypeError(f"Expected dict, got {type(data)}")

‎src/zarr/core/metadata/v3.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
NodeType,
2929
compress_rle,
3030
expand_rle,
31+
expand_short_hand_name,
3132
parse_chunk_edge,
3233
parse_chunk_shape,
3334
parse_named_configuration,
@@ -82,6 +83,7 @@ def parse_codecs(data: object) -> tuple[Codec, ...]:
8283
): # Can't use Codec here because of mypy limitation
8384
out += (c,)
8485
else:
86+
c = expand_short_hand_name(c)
8587
name_parsed, _ = parse_named_configuration(c, require_configuration=False)
8688

8789
codec_cls = get_codec_class(name_parsed)
@@ -156,7 +158,7 @@ def parse_storage_transformers(data: object) -> tuple[dict[str, JSON], ...]:
156158
# and then returned ``data`` itself, which exhausted (and discarded) a
157159
# one-shot iterable and could return a value typed as a tuple that was not
158160
# actually a tuple.
159-
return tuple(data)
161+
return tuple(expand_short_hand_name(t) for t in data)
160162
raise TypeError(
161163
f"Invalid storage_transformers. Expected an iterable of dicts. Got {type(data)} instead."
162164
)
@@ -458,6 +460,7 @@ def parse_chunk_grid(
458460
if isinstance(data, (RegularChunkGridMetadata, RectilinearChunkGridMetadata)):
459461
return data
460462

463+
data = expand_short_hand_name(data)
461464
name, _ = parse_named_configuration(data)
462465
if name == "regular":
463466
return RegularChunkGridMetadata.from_dict(data) # type: ignore[arg-type]
@@ -712,7 +715,7 @@ def from_dict(cls, data: dict[str, JSON], *, path: str | None = None) -> Self:
712715
metadata = cls(
713716
shape=_data_typed["shape"],
714717
chunk_grid=_data_typed["chunk_grid"], # type: ignore[arg-type]
715-
chunk_key_encoding=_data_typed["chunk_key_encoding"], # type: ignore[arg-type]
718+
chunk_key_encoding=_data_typed["chunk_key_encoding"],
716719
codecs=_data_typed["codecs"],
717720
# Attribute values are arbitrary JSON, so they have no field-specific
718721
# schema to validate. `__init__` checks the outer dict via `parse_attributes`.

‎tests/test_chunk_key_encodings.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,15 @@
11
from __future__ import annotations
22

3+
from typing import TYPE_CHECKING
4+
35
import pytest
46

7+
import zarr
58
from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding
9+
from zarr.storage import MemoryStore
10+
11+
if TYPE_CHECKING:
12+
from zarr.core.common import ZarrFormat
613

714

815
@pytest.mark.parametrize("separator", ["/", "."])
@@ -63,3 +70,32 @@ def test_default_encoding_rejects_key_using_the_other_separator() -> None:
6370

6471
with pytest.raises(ValueError, match="Invalid chunk key"):
6572
encoding.decode_chunk_key("c.0.1")
73+
74+
75+
@pytest.mark.parametrize(
76+
("name", "zarr_format", "chunk_keys"),
77+
[
78+
("default", 3, ["c/0/0", "c/0/1"]),
79+
("v2", 3, ["0.0", "0.1"]),
80+
("v2", 2, ["0.0", "0.1"]),
81+
],
82+
)
83+
async def test_create_array_short_hand_chunk_key_encoding(
84+
name: str, zarr_format: ZarrFormat, chunk_keys: list[str]
85+
) -> None:
86+
"""A bare chunk key encoding name given at array creation selects that encoding with
87+
its default separator, as it does when read from Zarr V3.1 short-hand metadata."""
88+
store = MemoryStore()
89+
array = zarr.create_array(
90+
store,
91+
shape=(1, 2),
92+
chunks=(1, 1),
93+
dtype="uint8",
94+
fill_value=0,
95+
chunk_key_encoding=name,
96+
zarr_format=zarr_format,
97+
)
98+
array[:] = 1
99+
metadata_keys = {"zarr.json", ".zarray", ".zattrs"}
100+
keys = sorted([key async for key in store.list() if key not in metadata_keys])
101+
assert keys == chunk_keys

‎tests/test_metadata/test_v3.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,61 @@ def test_array_metadata_keys_matches_typeddict() -> None:
260260
),
261261
id="extra_fields",
262262
),
263+
# Zarr V3.1 short-hand names: a bare string is read as `{"name": <string>}`, and
264+
# written back in the object form that Zarr V3.0 readers require.
265+
Expect(
266+
input={"codecs": ["bytes"]},
267+
output=minimal_metadata_dict_v3(codecs=_UINT8_CODECS),
268+
id="short_hand_codec",
269+
),
270+
Expect(
271+
input={"codecs": [{"name": "bytes"}, "crc32c"]},
272+
output=minimal_metadata_dict_v3(codecs=(*_UINT8_CODECS, {"name": "crc32c"})),
273+
id="short_hand_codec_mixed_with_object",
274+
),
275+
Expect(
276+
input={
277+
"codecs": [
278+
{
279+
"name": "sharding_indexed",
280+
"configuration": {
281+
"chunk_shape": [2, 2],
282+
"codecs": ["bytes"],
283+
"index_codecs": [*_FLOAT64_CODECS, "crc32c"],
284+
},
285+
}
286+
]
287+
},
288+
output=minimal_metadata_dict_v3(
289+
codecs=(
290+
{
291+
"name": "sharding_indexed",
292+
"configuration": {
293+
"chunk_shape": (2, 2),
294+
"codecs": _UINT8_CODECS,
295+
"index_codecs": (*_FLOAT64_CODECS, {"name": "crc32c"}),
296+
"index_location": "end",
297+
},
298+
},
299+
)
300+
),
301+
id="short_hand_codecs_nested_in_sharding",
302+
),
303+
Expect(
304+
input={"chunk_key_encoding": "v2"},
305+
output=minimal_metadata_dict_v3(
306+
chunk_key_encoding={"name": "v2", "configuration": {"separator": "."}},
307+
codecs=_UINT8_CODECS,
308+
),
309+
id="short_hand_chunk_key_encoding",
310+
),
311+
Expect(
312+
input={"storage_transformers": ["foo"]},
313+
output=minimal_metadata_dict_v3(
314+
storage_transformers=({"name": "foo"},), codecs=_UINT8_CODECS
315+
),
316+
id="short_hand_storage_transformer",
317+
),
263318
],
264319
ids=lambda case: case.id,
265320
)
@@ -299,6 +354,33 @@ def test_array_metadata_from_dict_fails(case: ExpectFail[dict[str, Any]]) -> Non
299354
ArrayV3Metadata.from_dict(d) # type: ignore[arg-type]
300355

301356

357+
@pytest.mark.parametrize(
358+
"overrides",
359+
[
360+
{"chunk_grid": "regular"},
361+
{"codecs": ["bytes", "gzip"]},
362+
{"codecs": ["sharding_indexed"]},
363+
],
364+
ids=["chunk_grid", "codec", "sharding_codec"],
365+
)
366+
def test_short_hand_name_requiring_configuration_rejected(overrides: dict[str, Any]) -> None:
367+
"""A short-hand name is only valid for an extension that needs no configuration."""
368+
with pytest.raises(ValueError, match="does not have a 'configuration' key"):
369+
ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(**overrides)) # type: ignore[arg-type]
370+
371+
372+
def test_short_hand_codec_name_unknown_rejected() -> None:
373+
"""A short-hand codec name with no registered implementation is rejected by name."""
374+
with pytest.raises(UnknownCodecError, match="'not-a-codec'"):
375+
ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(codecs=["bytes", "not-a-codec"])) # type: ignore[arg-type]
376+
377+
378+
def test_short_hand_chunk_key_encoding_name_unknown_rejected() -> None:
379+
"""A short-hand chunk key encoding name with no registered implementation is rejected."""
380+
with pytest.raises(ValueError, match="'not-an-encoding' not found"):
381+
ArrayV3Metadata.from_dict(minimal_metadata_dict_v3(chunk_key_encoding="not-an-encoding")) # type: ignore[arg-type]
382+
383+
302384
@pytest.mark.parametrize(
303385
"case",
304386
[

0 commit comments

Comments
 (0)