Skip to content

Commit 8e56629

Browse files
committed
Support type refinement in TypedDict subclasses
Allow readonly keys to be refined in subclasses, in line with PEP 705. For cases where refinement is not permitted by spec, provide a more detailed error message. As placeholders may now affect validation without ending up in the final TypedDictType, an `analysis_incomplete` field has been added, triggering reanalysis in subsequent rounds. This closes python#7435
1 parent 8ec6f22 commit 8e56629

4 files changed

Lines changed: 251 additions & 100 deletions

File tree

‎mypy/semanal.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2101,7 +2101,7 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool:
21012101
if (
21022102
defn.info
21032103
and defn.info.typeddict_type
2104-
and not has_placeholder(defn.info.typeddict_type)
2104+
and not defn.info.typeddict_type.analysis_incomplete
21052105
):
21062106
# Don't reprocess everything
21072107
is_typeddict = True

‎mypy/semanal_typeddict.py‎

Lines changed: 203 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,12 @@
22

33
from __future__ import annotations
44

5-
from collections.abc import Collection
6-
from typing import Final
5+
from typing import Final, NamedTuple
76

87
from mypy import errorcodes as codes, message_registry
98
from mypy.errorcodes import ErrorCode
109
from mypy.expandtype import expand_type
1110
from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type
12-
from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE
1311
from mypy.messages import MessageBuilder
1412
from mypy.nodes import (
1513
ARG_NAMED,
@@ -41,6 +39,7 @@
4139
require_bool_literal_argument,
4240
)
4341
from mypy.state import state
42+
from mypy.subtypes import is_equivalent, is_subtype
4443
from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type
4544
from mypy.types import (
4645
TPDICT_NAMES,
@@ -59,6 +58,14 @@
5958
)
6059

6160

61+
class FieldSource(NamedTuple):
62+
field_type: Type
63+
is_readonly: bool
64+
is_required: bool
65+
base_name: str | None = None
66+
child_field_ctx: AssignmentStmt | None = None
67+
68+
6269
class TypedDictAnalyzer:
6370
def __init__(
6471
self, options: Options, api: SemanticAnalyzerInterface, msg: MessageBuilder
@@ -101,20 +108,20 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N
101108
if isinstance(defn.analyzed, TypedDictExpr):
102109
existing_info = defn.analyzed.info
103110

104-
field_types: dict[str, Type] | None
105111
if (
106112
len(defn.base_type_exprs) == 1
107113
and isinstance(defn.base_type_exprs[0], RefExpr)
108114
and defn.base_type_exprs[0].fullname in TPDICT_NAMES
109115
):
110116
# Building a new TypedDict
111-
field_types, statements, required_keys, readonly_keys = (
112-
self.analyze_typeddict_classdef_fields(defn)
113-
)
114-
if field_types is None:
117+
field_sources, statements = self.analyze_typeddict_classdef_fields(defn)
118+
if field_sources is None:
115119
return True, None # Defer
116120
if self.api.is_func_scope() and "@" not in defn.name:
117121
defn.name += "@" + str(defn.line)
122+
field_types = {key: source.field_type for (key, source) in field_sources.items()}
123+
required_keys = {key for (key, source) in field_sources.items() if source.is_required}
124+
readonly_keys = {key for (key, source) in field_sources.items() if source.is_readonly}
118125
info = self.build_typeddict_typeinfo(
119126
defn.name, field_types, required_keys, readonly_keys, defn.line, existing_info
120127
)
@@ -154,45 +161,41 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N
154161
else:
155162
self.fail("All bases of a new TypedDict must be TypedDict types", defn)
156163

157-
field_types = {}
158-
required_keys = set()
159-
readonly_keys = set()
160-
# Iterate over bases in reverse order so that leftmost base class' keys take precedence
161-
for base in reversed(typeddict_bases):
162-
self.add_keys_and_types_from_base(
163-
base, field_types, required_keys, readonly_keys, defn
164-
)
165-
new_field_types, new_statements, new_required_keys, new_readonly_keys = (
166-
self.analyze_typeddict_classdef_fields(defn, oldfields=field_types)
167-
)
168-
if new_field_types is None:
164+
bases_info: list[tuple[TypeInfo, dict[str, Type]]] = []
165+
for base in typeddict_bases:
166+
base_info = self.fetch_keys_and_types_from_base(base, defn)
167+
if base_info is not None:
168+
bases_info.append(base_info)
169+
new_field_sources, new_statements = self.analyze_typeddict_classdef_fields(defn)
170+
if new_field_sources is None:
169171
return True, None # Defer
170-
field_types.update(new_field_types)
171-
required_keys.update(new_required_keys)
172-
readonly_keys.update(new_readonly_keys)
172+
field_types, required_keys, readonly_keys, any_placeholders = (
173+
self.resolve_field_inheritance(bases_info, new_field_sources, defn)
174+
)
173175
info = self.build_typeddict_typeinfo(
174-
defn.name, field_types, required_keys, readonly_keys, defn.line, existing_info
176+
defn.name,
177+
field_types,
178+
required_keys,
179+
readonly_keys,
180+
defn.line,
181+
existing_info,
182+
analysis_incomplete=any_placeholders,
175183
)
176184
defn.analyzed = TypedDictExpr(info)
177185
defn.analyzed.line = defn.line
178186
defn.analyzed.column = defn.column
179187
defn.defs.body = new_statements
180188
return True, info
181189

182-
def add_keys_and_types_from_base(
183-
self,
184-
base: Expression,
185-
field_types: dict[str, Type],
186-
required_keys: set[str],
187-
readonly_keys: set[str],
188-
ctx: Context,
189-
) -> None:
190+
def fetch_keys_and_types_from_base(
191+
self, base: Expression, ctx: Context
192+
) -> tuple[TypeInfo, dict[str, Type]] | None:
190193
info = self._parse_typeddict_base(base, ctx)
191194
base_args: list[Type] = []
192195
if isinstance(base, IndexExpr):
193196
args = self.analyze_base_args(base, ctx)
194197
if args is None:
195-
return
198+
return None
196199
base_args = args
197200

198201
assert info.typeddict_type is not None
@@ -211,13 +214,159 @@ def add_keys_and_types_from_base(
211214

212215
with state.strict_optional_set(self.options.strict_optional):
213216
valid_items = self.map_items_to_base(valid_items, tvars, base_args)
214-
for key in base_items:
215-
if key in field_types:
216-
self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx)
217217

218-
field_types.update(valid_items)
219-
required_keys.update(base_typed_dict.required_keys)
220-
readonly_keys.update(base_typed_dict.readonly_keys)
218+
return info, valid_items
219+
220+
def field_sources_in_reverse_mro(
221+
self,
222+
bases: list[tuple[TypeInfo, dict[str, Type]]],
223+
child_field_sources: dict[str, FieldSource],
224+
) -> dict[str, list[FieldSource]]:
225+
"""Find all keys in bases and child, mapping them to a list of sources.
226+
227+
Iterate bases in reverse order to preserve key ordering for display.
228+
"""
229+
result: dict[str, list[FieldSource]] = {}
230+
for base_info, base_fields in reversed(bases):
231+
assert base_info.typeddict_type is not None
232+
for field_name, field_type in base_fields.items():
233+
source = FieldSource(
234+
base_name=base_info.name,
235+
field_type=field_type,
236+
is_readonly=field_name in base_info.typeddict_type.readonly_keys,
237+
is_required=field_name in base_info.typeddict_type.required_keys,
238+
)
239+
result.setdefault(field_name, []).append(source)
240+
for field_name, source in child_field_sources.items():
241+
result.setdefault(field_name, []).append(source)
242+
return result
243+
244+
def primary_source(self, sources: list[FieldSource]) -> FieldSource:
245+
"""Select a primary source from a reverse-MRO-ordered list of sources.
246+
247+
The primary source will be the last in the MRO list, skipping readonly
248+
base class sources unless they are the only available option.
249+
"""
250+
if sources[-1].child_field_ctx:
251+
return sources[-1]
252+
mutable_sources = (s for s in reversed(sources) if not s.is_readonly)
253+
return next(mutable_sources, sources[-1])
254+
255+
def verify_field_compatibility(
256+
self,
257+
field_name: str,
258+
source: FieldSource,
259+
field_type: Type,
260+
is_readonly: bool,
261+
is_required: bool,
262+
primary_source_base_name: str | None,
263+
ctx: Context,
264+
) -> None:
265+
"""Verify compatibility of the final child type field with a base class source.
266+
267+
Verifies type compatibility, requiredness, and mutability.
268+
"""
269+
if has_placeholder(field_type) or has_placeholder(source.field_type):
270+
# Cannot verify type compatibility yet, but analysis will be deferred
271+
# and rerun later
272+
is_type_compatible = True
273+
elif source.is_readonly:
274+
is_type_compatible = is_subtype(field_type, source.field_type)
275+
else:
276+
is_type_compatible = is_equivalent(field_type, source.field_type)
277+
if not is_type_compatible:
278+
if primary_source_base_name is None:
279+
self.fail(
280+
f'Definition of field "{field_name}" incompatible with base class "{source.base_name}"',
281+
ctx,
282+
)
283+
else:
284+
self.fail(
285+
f'Incompatible definitions of field "{field_name}" in base classes "{source.base_name}" and "{primary_source_base_name}"',
286+
ctx,
287+
)
288+
if is_readonly:
289+
self.note(
290+
f'This can be resolved by redeclaring the field "{field_name}" with a mutually compatible type',
291+
ctx,
292+
)
293+
elif source.is_required and not is_required:
294+
if primary_source_base_name is None:
295+
self.fail(
296+
f'Field "{field_name}" is required in base class "{source.base_name}"', ctx
297+
)
298+
else:
299+
self.fail(
300+
f'Field "{field_name}" is required in base class "{source.base_name}" but not in base class "{primary_source_base_name}"',
301+
ctx,
302+
)
303+
elif not source.is_required and not source.is_readonly and is_required:
304+
if primary_source_base_name is None:
305+
self.fail(
306+
f'Field "{field_name}" is not required and not readonly in base class "{source.base_name}"',
307+
ctx,
308+
)
309+
else:
310+
self.fail(
311+
f'Field "{field_name}" is required in base class "{primary_source_base_name}" but not in base class "{source.base_name}"',
312+
ctx,
313+
)
314+
315+
def resolve_field_inheritance(
316+
self,
317+
bases: list[tuple[TypeInfo, dict[str, Type]]],
318+
child_field_sources: dict[str, FieldSource],
319+
ctx: Context,
320+
) -> tuple[dict[str, Type], set[str], set[str], bool]:
321+
"""Determine field types, requiredness, and readonlyness.
322+
323+
Additionally returns if any placeholders were seen, as they will prevent full
324+
analysis, but may not result in placeholders in the final type.
325+
"""
326+
field_sources = self.field_sources_in_reverse_mro(bases, child_field_sources)
327+
field_types: dict[str, Type] = {}
328+
required_keys: set[str] = set()
329+
readonly_keys: set[str] = set()
330+
any_placeholders = False
331+
332+
for field_name, sources in field_sources.items():
333+
primary_source = self.primary_source(sources)
334+
# If a read-only field is only defined in base classes, joining the types
335+
# is unlikely to produce a tight enough result. We could check all the
336+
# candidates from the base classes, but it would be O(n^2) complexity
337+
# to find out which is a supertype of all the others. Instead, use the
338+
# first definition in MRO order, and let the user provide the correct
339+
# definition in the subclass if this fails.
340+
field_types[field_name] = primary_source.field_type
341+
342+
if primary_source.is_readonly:
343+
# If the primary source is readonly, all sources are readonly
344+
is_readonly = True
345+
is_required = any(source.is_required for source in sources)
346+
else:
347+
is_readonly = False
348+
is_required = primary_source.is_required
349+
350+
if is_required:
351+
required_keys.add(field_name)
352+
if is_readonly:
353+
readonly_keys.add(field_name)
354+
355+
for source in sources:
356+
if has_placeholder(source.field_type):
357+
any_placeholders = True
358+
if source is not primary_source:
359+
self.verify_field_compatibility(
360+
field_name,
361+
source,
362+
field_types[field_name],
363+
is_readonly,
364+
is_required,
365+
primary_source.base_name,
366+
primary_source.child_field_ctx or ctx,
367+
)
368+
369+
return field_types, required_keys, readonly_keys, any_placeholders
221370

222371
def _parse_typeddict_base(self, base: Expression, ctx: Context) -> TypeInfo:
223372
if isinstance(base, RefExpr):
@@ -288,22 +437,19 @@ def map_items_to_base(
288437
return mapped_items
289438

290439
def analyze_typeddict_classdef_fields(
291-
self, defn: ClassDef, oldfields: Collection[str] | None = None
292-
) -> tuple[dict[str, Type] | None, list[Statement], set[str], set[str]]:
440+
self, defn: ClassDef
441+
) -> tuple[dict[str, FieldSource] | None, list[Statement]]:
293442
"""Analyze fields defined in a TypedDict class definition.
294443

295444
This doesn't consider inherited fields (if any). Also consider totality,
296445
if given.
297446

298447
Return tuple with these items:
299-
* Dict of key -> type (or None if found an incomplete reference -> deferral)
448+
* Dict of key -> field source (or None if found an incomplete reference -> deferral)
300449
* List of statements from defn.defs.body that are legally allowed to be a
301450
part of a TypedDict definition
302-
* Set of required keys
303451
"""
304-
fields: dict[str, Type] = {}
305-
readonly_keys = set[str]()
306-
required_keys = set[str]()
452+
fields: dict[str, FieldSource] = {}
307453
statements: list[Statement] = []
308454

309455
total: bool | None = True
@@ -333,8 +479,6 @@ def analyze_typeddict_classdef_fields(
333479
self.fail(TPDICT_CLASS_ERROR, stmt)
334480
else:
335481
name = stmt.lvalues[0].name
336-
if name in (oldfields or []):
337-
self.fail(f'Overwriting TypedDict field "{name}" while extending', stmt)
338482
if name in fields:
339483
self.fail(f'Duplicate TypedDict key "{name}"', stmt)
340484
continue
@@ -353,18 +497,18 @@ def analyze_typeddict_classdef_fields(
353497
prohibit_special_class_field_types="TypedDict",
354498
)
355499
if analyzed is None:
356-
return None, [], set(), set() # Need to defer
500+
return None, [] # Need to defer
357501
field_type = analyzed
358502
if not has_placeholder(analyzed):
359503
stmt.type = self.extract_meta_info(analyzed, stmt)[0]
360504

361505
field_type, required, readonly = self.extract_meta_info(field_type)
362-
fields[name] = field_type
363-
364-
if (total or required is True) and required is not False:
365-
required_keys.add(name)
366-
if readonly:
367-
readonly_keys.add(name)
506+
fields[name] = FieldSource(
507+
field_type=field_type,
508+
is_required=(total or required is True) and required is not False,
509+
is_readonly=readonly,
510+
child_field_ctx=stmt,
511+
)
368512

369513
# ...despite possible minor failures that allow further analysis.
370514
if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax:
@@ -373,7 +517,7 @@ def analyze_typeddict_classdef_fields(
373517
# x: int assigns rvalue to TempNode(AnyType())
374518
self.fail("Right hand side values are not supported in TypedDict", stmt)
375519

376-
return fields, statements, required_keys, readonly_keys
520+
return fields, statements
377521

378522
def extract_meta_info(
379523
self, typ: Type, context: Context | None = None
@@ -597,6 +741,7 @@ def build_typeddict_typeinfo(
597741
readonly_keys: set[str],
598742
line: int,
599743
existing_info: TypeInfo | None,
744+
analysis_incomplete: bool = False,
600745
) -> TypeInfo:
601746
# Prefer typing then typing_extensions if available.
602747
fallback = (
@@ -607,7 +752,8 @@ def build_typeddict_typeinfo(
607752
assert fallback is not None
608753
info = existing_info or self.api.basic_new_typeinfo(name, fallback, line)
609754
typeddict_type = TypedDictType(item_types, required_keys, readonly_keys, fallback)
610-
if has_placeholder(typeddict_type):
755+
if has_placeholder(typeddict_type) or analysis_incomplete:
756+
typeddict_type.analysis_incomplete = True
611757
self.api.process_placeholder(
612758
None, "TypedDict item", info, force_progress=typeddict_type != info.typeddict_type
613759
)

0 commit comments

Comments
 (0)