Skip to content

Commit df1a0cc

Browse files
vup903claude
andcommitted
Fix lint and make TypedDict NotRequired robust under future annotations (zarr-developers#3285)
Apply ruff check/format to satisfy the Lint CI hook. Keep typing.Union/Optional spellings in tests (with noqa) to cover that origin path alongside X | Y. Rework _parse_typeddict to derive required/optional from get_type_hints(include_extras=True) + __total__ instead of __required_keys__, so class-syntax NotRequired is detected even when 'from __future__ import annotations' stringizes the hints (as in zarr's metadata modules). test_json_parse + test_common + test_metadata = 393 passed. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 0eba162 commit df1a0cc

2 files changed

Lines changed: 55 additions & 34 deletions

File tree

‎src/zarr/core/json_parse.py‎

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
import types
2626
from collections.abc import Mapping, Sequence
27-
from typing import Any, Literal, Union, get_args, get_origin
27+
from typing import Any, Literal, NotRequired, Required, Union, get_args, get_origin
2828

2929
from typing_extensions import get_type_hints, is_typeddict
3030

@@ -101,8 +101,7 @@ def parse_json(value: object, type_annotation: object) -> Any:
101101

102102
# 9. Fallback
103103
raise TypeError(
104-
f"Cannot parse value {value!r} against unsupported type annotation "
105-
f"{type_annotation!r}."
104+
f"Cannot parse value {value!r} against unsupported type annotation {type_annotation!r}."
106105
)
107106

108107

@@ -154,7 +153,7 @@ def _parse_union(value: object, type_annotation: object) -> Any:
154153
for member in members:
155154
try:
156155
return parse_json(value, member)
157-
except (ValueError, TypeError) as exc: # noqa: PERF203
156+
except (ValueError, TypeError) as exc:
158157
errors.append(f" - against {member!r}: {exc}")
159158
joined = "\n".join(errors)
160159
raise ValueError(
@@ -193,7 +192,9 @@ def _parse_tuple(value: object, type_annotation: object) -> tuple[Any, ...]:
193192
f"Expected a sequence of length {len(args)}, got {value!r} of "
194193
f"length {len(value)} instead."
195194
)
196-
return tuple(parse_json(item, element_type) for item, element_type in zip(value, args, strict=True))
195+
return tuple(
196+
parse_json(item, element_type) for item, element_type in zip(value, args, strict=True)
197+
)
197198

198199

199200
def _parse_sequence(value: object, type_annotation: object) -> tuple[Any, ...]:
@@ -239,19 +240,36 @@ def _parse_mapping(value: object, type_annotation: object) -> dict[str, Any]:
239240
def _parse_typeddict(value: object, type_annotation: Any) -> dict[str, Any]:
240241
"""Validate a :class:`~typing.TypedDict` annotation.
241242
242-
Each required key must be present and is parsed against its annotation.
243-
Optional keys are parsed when present. Totality is respected via the
244-
TypedDict's ``__required_keys__`` / ``__optional_keys__`` attributes.
243+
Each required key must be present and is parsed against its annotation;
244+
optional keys are parsed when present. Whether a key is required is derived
245+
from the *resolved* annotations -- their ``Required`` / ``NotRequired``
246+
wrappers combined with the TypedDict's totality (``__total__``).
247+
248+
This deliberately does not use ``__required_keys__`` / ``__optional_keys__``:
249+
under ``from __future__ import annotations`` those are computed from
250+
stringized annotations at class-creation time, so a ``NotRequired[...]``
251+
wrapper is invisible to them. Resolving the hints with
252+
``include_extras=True`` evaluates the strings and makes the wrappers visible.
245253
"""
246254
if not isinstance(value, Mapping):
247255
raise TypeError(f"Expected a mapping, got {value!r} instead.")
248256

249-
# ``include_extras=True`` keeps ``Required`` / ``NotRequired`` / ``ReadOnly``
250-
# wrappers visible if needed; the required/optional split is taken from the
251-
# TypedDict's own bookkeeping which already accounts for totality.
252-
hints = get_type_hints(type_annotation, include_extras=False)
253-
required_keys: frozenset[str] = getattr(type_annotation, "__required_keys__", frozenset())
254-
optional_keys: frozenset[str] = getattr(type_annotation, "__optional_keys__", frozenset())
257+
hints = get_type_hints(type_annotation, include_extras=True)
258+
total = getattr(type_annotation, "__total__", True)
259+
260+
required_keys: set[str] = set()
261+
field_types: dict[str, Any] = {}
262+
for key, hint in hints.items():
263+
origin = get_origin(hint)
264+
if origin is Required:
265+
required_keys.add(key)
266+
field_types[key] = get_args(hint)[0]
267+
elif origin is NotRequired:
268+
field_types[key] = get_args(hint)[0]
269+
else:
270+
if total:
271+
required_keys.add(key)
272+
field_types[key] = hint
255273

256274
missing = [key for key in required_keys if key not in value]
257275
if missing:
@@ -261,9 +279,9 @@ def _parse_typeddict(value: object, type_annotation: Any) -> dict[str, Any]:
261279
)
262280

263281
result: dict[str, Any] = {}
264-
for key in required_keys | optional_keys:
282+
for key, field_type in field_types.items():
265283
if key in value:
266-
result[key] = parse_json(value[key], hints[key])
284+
result[key] = parse_json(value[key], field_type)
267285
# Preserve any extra keys not declared on the TypedDict.
268286
for key, val in value.items():
269287
if key not in result:

‎tests/test_json_parse.py‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@
1010
from __future__ import annotations
1111

1212
from collections.abc import Mapping, Sequence
13-
from typing import TYPE_CHECKING, Literal, Optional, Union
13+
from typing import TYPE_CHECKING, Literal, NotRequired, Optional, Union
1414

1515
import pytest
1616

1717
# ``parse_json`` (the module under test) uses ``typing_extensions`` for its
1818
# TypedDict bookkeeping, so we build TypedDicts the same way for consistency.
19-
from typing_extensions import NotRequired, TypedDict
19+
from typing_extensions import TypedDict
2020

2121
from zarr.core.json_parse import parse_json
2222

@@ -159,7 +159,7 @@ def test_literal_one_does_not_match_bool_literal() -> None:
159159

160160

161161
def test_literal_mixed_members() -> None:
162-
annotation = Literal["a", 3, None]
162+
annotation = Literal["a", 3] | None
163163
assert parse_json("a", annotation) == "a"
164164
assert parse_json(3, annotation) == 3
165165
# ``None`` is a valid literal member.
@@ -169,11 +169,15 @@ def test_literal_mixed_members() -> None:
169169
# ---------------------------------------------------------------------------
170170
# Union / Optional
171171
# ---------------------------------------------------------------------------
172+
# These tests deliberately use the ``typing.Union`` / ``typing.Optional``
173+
# spelling (which yields a ``typing.Union`` origin) to cover that code path;
174+
# the ``X | Y`` spelling (``types.UnionType``) is covered separately below.
175+
# The legacy-spelling lines therefore suppress ruff's UP007/UP045 rules.
172176

173177

174178
@pytest.mark.parametrize("value", ["hello", 5])
175179
def test_union_each_member_accepted(value: Any) -> None:
176-
assert parse_json(value, Union[str, int]) == value
180+
assert parse_json(value, Union[str, int]) == value # noqa: UP007
177181

178182

179183
def test_union_pep604_syntax() -> None:
@@ -184,27 +188,27 @@ def test_union_pep604_syntax() -> None:
184188

185189
def test_union_no_member_matches_raises() -> None:
186190
with pytest.raises(ValueError, match="Expected a value matching one of"):
187-
parse_json(1.5, Union[str, int])
191+
parse_json(1.5, Union[str, int]) # noqa: UP007
188192

189193

190194
def test_union_error_lists_each_member() -> None:
191195
with pytest.raises(ValueError, match="Tried each union member"):
192-
parse_json([], Union[str, int])
196+
parse_json([], Union[str, int]) # noqa: UP007
193197

194198

195199
@pytest.mark.parametrize("value", [None, "x"])
196200
def test_optional_accepts_none_and_value(value: Any) -> None:
197-
assert parse_json(value, Optional[str]) == value
201+
assert parse_json(value, Optional[str]) == value # noqa: UP045
198202

199203

200204
def test_optional_rejects_other_types() -> None:
201205
with pytest.raises(ValueError, match="Expected a value matching one of"):
202-
parse_json(5, Optional[str])
206+
parse_json(5, Optional[str]) # noqa: UP045
203207

204208

205209
def test_union_with_coercion_returns_coerced_value() -> None:
206210
"""A union member that coerces (Sequence -> tuple) returns the coerced form."""
207-
result = parse_json([1, 2, 3], Union[str, Sequence[int]])
211+
result = parse_json([1, 2, 3], Union[str, Sequence[int]]) # noqa: UP007
208212
assert result == (1, 2, 3)
209213
assert isinstance(result, tuple)
210214

@@ -358,15 +362,14 @@ class Point(TypedDict):
358362

359363

360364
# NOTE: This module uses ``from __future__ import annotations``, which stringizes
361-
# annotations. With the *class* TypedDict syntax, ``typing_extensions`` cannot see
362-
# a ``NotRequired[...]`` wrapper at class-creation time (the annotation is a string),
363-
# so the key is wrongly recorded as required. This is a known limitation of
364-
# stringized annotations + ``NotRequired`` -- NOT a bug in ``json_parse``. The
365-
# functional ``TypedDict(...)`` form keeps ``NotRequired`` as a real runtime object,
366-
# so ``__optional_keys__`` is populated correctly; we use it here to genuinely
367-
# exercise NotRequired. (``total=False``, used by ``PartialConfig`` below, is
368-
# class-level and works correctly even with stringized annotations.)
369-
Config = TypedDict("Config", {"name": str, "count": NotRequired[int]})
365+
# annotations -- so a class-syntax ``NotRequired[...]`` is invisible to the
366+
# TypedDict's ``__optional_keys__``. ``_parse_typeddict`` handles this by
367+
# resolving the hints with ``get_type_hints(..., include_extras=True)`` and
368+
# reading the ``Required`` / ``NotRequired`` wrappers directly, so the class
369+
# form below genuinely exercises NotRequired under stringized annotations.
370+
class Config(TypedDict):
371+
name: str
372+
count: NotRequired[int]
370373

371374

372375
class PartialConfig(TypedDict, total=False):

0 commit comments

Comments
 (0)