Skip to content

Commit e6945e9

Browse files
author
Alice Purcell
committed
Support type refinement in TypedDict subclasses
Allow readonly keys to be refined in subclasses, in line with PEP 705. For cases where refinement is not permitted by spec, provide a more detailed error message. This closes python#7435
1 parent c140787 commit e6945e9

2 files changed

Lines changed: 422 additions & 57 deletions

File tree

‎mypy/semanal_typeddict.py‎

Lines changed: 181 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,12 @@
22

33
from __future__ import annotations
44

5-
from collections.abc import Collection
6-
from typing import Final
5+
from typing import Final, NamedTuple
76

87
from mypy import errorcodes as codes, message_registry
98
from mypy.errorcodes import ErrorCode
109
from mypy.expandtype import expand_type
1110
from mypy.exprtotype import TypeTranslationError, expr_to_unanalyzed_type
12-
from mypy.message_registry import TYPEDDICT_OVERRIDE_MERGE
1311
from mypy.messages import MessageBuilder
1412
from mypy.nodes import (
1513
ARG_NAMED,
@@ -41,6 +39,7 @@
4139
require_bool_literal_argument,
4240
)
4341
from mypy.state import state
42+
from mypy.subtypes import is_equivalent, is_subtype
4443
from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type
4544
from mypy.types import (
4645
TPDICT_NAMES,
@@ -59,6 +58,14 @@
5958
)
6059

6160

61+
class FieldSource(NamedTuple):
62+
field_type: Type
63+
is_readonly: bool
64+
is_required: bool
65+
base_name: str | None = None
66+
child_field_ctx: AssignmentStmt | None = None
67+
68+
6269
class TypedDictAnalyzer:
6370
def __init__(
6471
self, options: Options, api: SemanticAnalyzerInterface, msg: MessageBuilder
@@ -101,20 +108,20 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N
101108
if isinstance(defn.analyzed, TypedDictExpr):
102109
existing_info = defn.analyzed.info
103110

104-
field_types: dict[str, Type] | None
105111
if (
106112
len(defn.base_type_exprs) == 1
107113
and isinstance(defn.base_type_exprs[0], RefExpr)
108114
and defn.base_type_exprs[0].fullname in TPDICT_NAMES
109115
):
110116
# Building a new TypedDict
111-
field_types, statements, required_keys, readonly_keys = (
112-
self.analyze_typeddict_classdef_fields(defn)
113-
)
114-
if field_types is None:
117+
field_sources, statements = self.analyze_typeddict_classdef_fields(defn)
118+
if field_sources is None:
115119
return True, None # Defer
116120
if self.api.is_func_scope() and "@" not in defn.name:
117121
defn.name += "@" + str(defn.line)
122+
field_types = {key: source.field_type for (key, source) in field_sources.items()}
123+
required_keys = {key for (key, source) in field_sources.items() if source.is_required}
124+
readonly_keys = {key for (key, source) in field_sources.items() if source.is_readonly}
118125
info = self.build_typeddict_typeinfo(
119126
defn.name, field_types, required_keys, readonly_keys, defn.line, existing_info
120127
)
@@ -154,22 +161,17 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N
154161
else:
155162
self.fail("All bases of a new TypedDict must be TypedDict types", defn)
156163

157-
field_types = {}
158-
required_keys = set()
159-
readonly_keys = set()
160-
# Iterate over bases in reverse order so that leftmost base class' keys take precedence
161-
for base in reversed(typeddict_bases):
162-
self.add_keys_and_types_from_base(
163-
base, field_types, required_keys, readonly_keys, defn
164-
)
165-
(new_field_types, new_statements, new_required_keys, new_readonly_keys) = (
166-
self.analyze_typeddict_classdef_fields(defn, oldfields=field_types)
167-
)
168-
if new_field_types is None:
164+
bases_info: list[tuple[TypeInfo, dict[str, Type]]] = []
165+
for base in typeddict_bases:
166+
base_info = self.fetch_keys_and_types_from_base(base, defn)
167+
if base_info is not None:
168+
bases_info.append(base_info)
169+
(new_field_sources, new_statements) = self.analyze_typeddict_classdef_fields(defn)
170+
if new_field_sources is None:
169171
return True, None # Defer
170-
field_types.update(new_field_types)
171-
required_keys.update(new_required_keys)
172-
readonly_keys.update(new_readonly_keys)
172+
field_types, required_keys, readonly_keys = self.resolve_field_inheritance(
173+
bases_info, new_field_sources, defn
174+
)
173175
info = self.build_typeddict_typeinfo(
174176
defn.name, field_types, required_keys, readonly_keys, defn.line, existing_info
175177
)
@@ -179,20 +181,15 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> tuple[bool, TypeInfo | N
179181
defn.defs.body = new_statements
180182
return True, info
181183

182-
def add_keys_and_types_from_base(
183-
self,
184-
base: Expression,
185-
field_types: dict[str, Type],
186-
required_keys: set[str],
187-
readonly_keys: set[str],
188-
ctx: Context,
189-
) -> None:
184+
def fetch_keys_and_types_from_base(
185+
self, base: Expression, ctx: Context
186+
) -> tuple[TypeInfo, dict[str, Type]] | None:
190187
info = self._parse_typeddict_base(base, ctx)
191188
base_args: list[Type] = []
192189
if isinstance(base, IndexExpr):
193190
args = self.analyze_base_args(base, ctx)
194191
if args is None:
195-
return
192+
return None
196193
base_args = args
197194

198195
assert info.typeddict_type is not None
@@ -211,13 +208,147 @@ def add_keys_and_types_from_base(
211208

212209
with state.strict_optional_set(self.options.strict_optional):
213210
valid_items = self.map_items_to_base(valid_items, tvars, base_args)
214-
for key in base_items:
215-
if key in field_types:
216-
self.fail(TYPEDDICT_OVERRIDE_MERGE.format(key), ctx)
217211

218-
field_types.update(valid_items)
219-
required_keys.update(base_typed_dict.required_keys)
220-
readonly_keys.update(base_typed_dict.readonly_keys)
212+
return info, valid_items
213+
214+
def field_sources_in_reverse_mro(
215+
self,
216+
bases: list[tuple[TypeInfo, dict[str, Type]]],
217+
child_field_sources: dict[str, FieldSource],
218+
) -> dict[str, list[FieldSource]]:
219+
"""Find all keys in bases and child, mapping them to a list of sources.
220+
221+
Iterate bases in reverse order to preserve key ordering for display.
222+
"""
223+
result: dict[str, list[FieldSource]] = {}
224+
for base_info, base_fields in reversed(bases):
225+
assert base_info.typeddict_type is not None
226+
for field_name, field_type in base_fields.items():
227+
source = FieldSource(
228+
base_name=base_info.name,
229+
field_type=field_type,
230+
is_readonly=field_name in base_info.typeddict_type.readonly_keys,
231+
is_required=field_name in base_info.typeddict_type.required_keys,
232+
)
233+
result.setdefault(field_name, []).append(source)
234+
for field_name, source in child_field_sources.items():
235+
result.setdefault(field_name, []).append(source)
236+
return result
237+
238+
def primary_source(self, sources: list[FieldSource]) -> FieldSource:
239+
"""Select a primary source from a reverse-MRO-ordered list of sources.
240+
241+
The primary source will be the last in the MRO list, skipping readonly
242+
base class sources unless they are the only available option.
243+
"""
244+
if sources[-1].child_field_ctx:
245+
return sources[-1]
246+
mutable_sources = (s for s in reversed(sources) if not s.is_readonly)
247+
return next(mutable_sources, sources[-1])
248+
249+
def verify_compatibility(
250+
self,
251+
field_name: str,
252+
source: FieldSource,
253+
field_type: Type,
254+
is_readonly: bool,
255+
is_required: bool,
256+
primary_source_base_name: str | None,
257+
ctx: Context,
258+
) -> None:
259+
"""Verify compatibility of the final child type field with a base class source.
260+
261+
Verifies type compatibility, requiredness, and mutability.
262+
"""
263+
if source.is_readonly:
264+
is_type_compatible = is_subtype(field_type, source.field_type)
265+
else:
266+
is_type_compatible = is_equivalent(field_type, source.field_type)
267+
if not is_type_compatible:
268+
if primary_source_base_name is None:
269+
self.fail(
270+
f'Definition of field "{field_name}" incompatible with base class "{source.base_name}"',
271+
ctx,
272+
)
273+
else:
274+
self.fail(
275+
f'Incompatible definitions of field "{field_name}" in base classes "{source.base_name}" and "{primary_source_base_name}"',
276+
ctx,
277+
)
278+
if is_readonly:
279+
self.note(
280+
f'This can be resolved by redeclaring the field "{field_name}" with a mutually compatible type',
281+
ctx,
282+
)
283+
elif source.is_required and not is_required:
284+
if primary_source_base_name is None:
285+
self.fail(
286+
f'Field "{field_name}" is required in base class "{source.base_name}"', ctx
287+
)
288+
else:
289+
self.fail(
290+
f'Field "{field_name}" is required in base class "{source.base_name}" but not in base class "{primary_source_base_name}"',
291+
ctx,
292+
)
293+
elif not source.is_required and not source.is_readonly and is_required:
294+
if primary_source_base_name is None:
295+
self.fail(
296+
f'Field "{field_name}" is not required and not readonly in base class "{source.base_name}"',
297+
ctx,
298+
)
299+
else:
300+
self.fail(
301+
f'Field "{field_name}" is required in base class "{primary_source_base_name}" but not in base class "{source.base_name}"',
302+
ctx,
303+
)
304+
305+
def resolve_field_inheritance(
306+
self,
307+
bases: list[tuple[TypeInfo, dict[str, Type]]],
308+
child_field_sources: dict[str, FieldSource],
309+
ctx: Context,
310+
) -> tuple[dict[str, Type], set[str], set[str]]:
311+
field_sources = self.field_sources_in_reverse_mro(bases, child_field_sources)
312+
field_types: dict[str, Type] = {}
313+
required_keys: set[str] = set()
314+
readonly_keys: set[str] = set()
315+
316+
for field_name, sources in field_sources.items():
317+
primary_source = self.primary_source(sources)
318+
# If a read-only field is only defined in base classes, joining the types
319+
# is unlikely to produce a tight enough result. We could check all the
320+
# candidates from the base classes, but it would be O(n^2) complexity
321+
# to find out which is a supertype of all the others. Instead, use the
322+
# first definition in MRO order, and let the user provide the correct
323+
# definition in the subclass if this fails.
324+
field_types[field_name] = primary_source.field_type
325+
326+
if primary_source.is_readonly:
327+
# If the primary source is readonly, all sources are readonly
328+
is_readonly = True
329+
is_required = any(source.is_required for source in sources)
330+
else:
331+
is_readonly = False
332+
is_required = primary_source.is_required
333+
334+
if is_required:
335+
required_keys.add(field_name)
336+
if is_readonly:
337+
readonly_keys.add(field_name)
338+
339+
for source in sources:
340+
if source is not primary_source:
341+
self.verify_compatibility(
342+
field_name,
343+
source,
344+
field_types[field_name],
345+
is_readonly,
346+
is_required,
347+
primary_source.base_name,
348+
primary_source.child_field_ctx or ctx,
349+
)
350+
351+
return field_types, required_keys, readonly_keys
221352

222353
def _parse_typeddict_base(self, base: Expression, ctx: Context) -> TypeInfo:
223354
if isinstance(base, RefExpr):
@@ -288,22 +419,19 @@ def map_items_to_base(
288419
return mapped_items
289420

290421
def analyze_typeddict_classdef_fields(
291-
self, defn: ClassDef, oldfields: Collection[str] | None = None
292-
) -> tuple[dict[str, Type] | None, list[Statement], set[str], set[str]]:
422+
self, defn: ClassDef
423+
) -> tuple[dict[str, FieldSource] | None, list[Statement]]:
293424
"""Analyze fields defined in a TypedDict class definition.
294425
295426
This doesn't consider inherited fields (if any). Also consider totality,
296427
if given.
297428
298429
Return tuple with these items:
299-
* Dict of key -> type (or None if found an incomplete reference -> deferral)
430+
* Dict of key -> field source (or None if found an incomplete reference -> deferral)
300431
* List of statements from defn.defs.body that are legally allowed to be a
301432
part of a TypedDict definition
302-
* Set of required keys
303433
"""
304-
fields: dict[str, Type] = {}
305-
readonly_keys = set[str]()
306-
required_keys = set[str]()
434+
fields: dict[str, FieldSource] = {}
307435
statements: list[Statement] = []
308436

309437
total: bool | None = True
@@ -333,8 +461,6 @@ def analyze_typeddict_classdef_fields(
333461
self.fail(TPDICT_CLASS_ERROR, stmt)
334462
else:
335463
name = stmt.lvalues[0].name
336-
if name in (oldfields or []):
337-
self.fail(f'Overwriting TypedDict field "{name}" while extending', stmt)
338464
if name in fields:
339465
self.fail(f'Duplicate TypedDict key "{name}"', stmt)
340466
continue
@@ -353,18 +479,18 @@ def analyze_typeddict_classdef_fields(
353479
prohibit_special_class_field_types="TypedDict",
354480
)
355481
if analyzed is None:
356-
return None, [], set(), set() # Need to defer
482+
return None, [] # Need to defer
357483
field_type = analyzed
358484
if not has_placeholder(analyzed):
359485
stmt.type = self.extract_meta_info(analyzed, stmt)[0]
360486

361487
field_type, required, readonly = self.extract_meta_info(field_type)
362-
fields[name] = field_type
363-
364-
if (total or required is True) and required is not False:
365-
required_keys.add(name)
366-
if readonly:
367-
readonly_keys.add(name)
488+
fields[name] = FieldSource(
489+
field_type=field_type,
490+
is_required=(total or required is True) and required is not False,
491+
is_readonly=readonly,
492+
child_field_ctx=stmt,
493+
)
368494

369495
# ...despite possible minor failures that allow further analysis.
370496
if stmt.type is None or hasattr(stmt, "new_syntax") and not stmt.new_syntax:
@@ -373,7 +499,7 @@ def analyze_typeddict_classdef_fields(
373499
# x: int assigns rvalue to TempNode(AnyType())
374500
self.fail("Right hand side values are not supported in TypedDict", stmt)
375501

376-
return fields, statements, required_keys, readonly_keys
502+
return fields, statements
377503

378504
def extract_meta_info(
379505
self, typ: Type, context: Context | None = None

0 commit comments

Comments
 (0)