Skip to content

Commit dd4b8a2

Browse files
committed
Fix crash with generic class definition in function
Fixes python#12112. The reason why mypy was crashing with a "Must not defer during final iteration" error in the following snippet: from typing import TypeVar def test() -> None: T = TypeVar('T', bound='Foo') class Foo: def bar(self, foo: T) -> None: pass ...was because mypy did not seem to be updating the types of the `bar` callable on each pass: the `bind_function_type_variables` method in `typeanal.py` always returned the _old_ type variables instead of using the new updated ones we found by calling `self.lookup_qualified(...)`. This in turn prevented us from making any forward progress when mypy generated a CallableType containing a placedholder type variable. So, we repeated the semanal passes until we hit the limit and crashed. I opted to fix this by having the function always return the newly-bound TypeVarLikeType instead. (Hopefully this is safe -- the way mypy likes mutating types always makes it hard to reason about this sort of stuff). Interestingly, my fix for this bug introduced a regression in one of our existing tests: from typing import NamedTuple, TypeVar T = TypeVar("T") NT = NamedTuple("NT", [("key", int), ("value", T)]) # Test thinks the revealed type should be: # def [T] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1, fallback=__main__.NT[T`-1]] # # ...but we started seeing: # def [T, _NT <: Tuple[builtins.int, T`-1]] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1, fallback=test.WTF[T`-1]] reveal_type(NT) What seems to be happening here is that during the first pass, we add two type vars to the `tvar_scope` inside `bind_function_type_variables`: `T` with id -1 and `_NT` with id -2. But in the second pass, we lose track of the `T` typevar definition and/or introduce a fresh scope somewhere and infer `_NT` with id -1 instead? So now mypy thinks there are two type variables associated with this NamedTuple, which results in the screwed-up type definition. I wasn't really sure how to fix this, but I thought it was weird that: 1. We were using negative IDs instead of positive ones. The former is meant to be for class definitions, which is what we're using here. 2. We weren't wrapping this whole thing in a new tvar scope frame, given we're nominally synthesizing a new class. So I did that, and the tests started passing? I wasn't able to repro this issue for TypedDicts, but opted to introduce a new tvar scope frame there as well for consistency.
1 parent d619c78 commit dd4b8a2

6 files changed

Lines changed: 62 additions & 47 deletions

File tree

‎mypy/semanal.py‎

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2738,30 +2738,32 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool:
27382738
return False
27392739
lvalue = s.lvalues[0]
27402740
name = lvalue.name
2741-
internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple(
2742-
s.rvalue, name, self.is_func_scope()
2743-
)
2744-
if internal_name is None:
2745-
return False
2746-
if isinstance(lvalue, MemberExpr):
2747-
self.fail("NamedTuple type as an attribute is not supported", lvalue)
2748-
return False
2749-
if internal_name != name:
2750-
self.fail(
2751-
'First argument to namedtuple() should be "{}", not "{}"'.format(
2752-
name, internal_name
2753-
),
2754-
s.rvalue,
2755-
code=codes.NAME_MATCH,
2741+
namespace = self.qualified_name(name)
2742+
with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
2743+
internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple(
2744+
s.rvalue, name, self.is_func_scope()
27562745
)
2746+
if internal_name is None:
2747+
return False
2748+
if isinstance(lvalue, MemberExpr):
2749+
self.fail("NamedTuple type as an attribute is not supported", lvalue)
2750+
return False
2751+
if internal_name != name:
2752+
self.fail(
2753+
'First argument to namedtuple() should be "{}", not "{}"'.format(
2754+
name, internal_name
2755+
),
2756+
s.rvalue,
2757+
code=codes.NAME_MATCH,
2758+
)
2759+
return True
2760+
# Yes, it's a valid namedtuple, but defer if it is not ready.
2761+
if not info:
2762+
self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
2763+
else:
2764+
self.setup_type_vars(info.defn, tvar_defs)
2765+
self.setup_alias_type_vars(info.defn)
27572766
return True
2758-
# Yes, it's a valid namedtuple, but defer if it is not ready.
2759-
if not info:
2760-
self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
2761-
else:
2762-
self.setup_type_vars(info.defn, tvar_defs)
2763-
self.setup_alias_type_vars(info.defn)
2764-
return True
27652767

27662768
def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool:
27672769
"""Check if s defines a typed dict."""
@@ -2775,22 +2777,24 @@ def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool:
27752777
return False
27762778
lvalue = s.lvalues[0]
27772779
name = lvalue.name
2778-
is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict(
2779-
s.rvalue, name, self.is_func_scope()
2780-
)
2781-
if not is_typed_dict:
2782-
return False
2783-
if isinstance(lvalue, MemberExpr):
2784-
self.fail("TypedDict type as attribute is not supported", lvalue)
2785-
return False
2786-
# Yes, it's a valid typed dict, but defer if it is not ready.
2787-
if not info:
2788-
self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
2789-
else:
2790-
defn = info.defn
2791-
self.setup_type_vars(defn, tvar_defs)
2792-
self.setup_alias_type_vars(defn)
2793-
return True
2780+
namespace = self.qualified_name(name)
2781+
with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
2782+
is_typed_dict, info, tvar_defs = self.typed_dict_analyzer.check_typeddict(
2783+
s.rvalue, name, self.is_func_scope()
2784+
)
2785+
if not is_typed_dict:
2786+
return False
2787+
if isinstance(lvalue, MemberExpr):
2788+
self.fail("TypedDict type as attribute is not supported", lvalue)
2789+
return False
2790+
# Yes, it's a valid typed dict, but defer if it is not ready.
2791+
if not info:
2792+
self.mark_incomplete(name, lvalue, becomes_typeinfo=True)
2793+
else:
2794+
defn = info.defn
2795+
self.setup_type_vars(defn, tvar_defs)
2796+
self.setup_alias_type_vars(defn)
2797+
return True
27942798

27952799
def analyze_lvalues(self, s: AssignmentStmt) -> None:
27962800
# We cannot use s.type, because analyze_simple_literal_type() will set it.

‎mypy/semanal_namedtuple.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,11 +321,12 @@ def parse_namedtuple_args(
321321
) -> None | (tuple[list[str], list[Type], list[Expression], str, list[TypeVarLikeType], bool]):
322322
"""Parse a namedtuple() call into data needed to construct a type.
323323
324-
Returns a 5-tuple:
324+
Returns a 6-tuple:
325325
- List of argument names
326326
- List of argument types
327327
- List of default values
328328
- First argument of namedtuple
329+
- All typevars found in the field definition
329330
- Whether all types are ready.
330331
331332
Return None if the definition didn't typecheck.

‎mypy/typeanal.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1331,29 +1331,29 @@ def bind_function_type_variables(
13311331
) -> Sequence[TypeVarLikeType]:
13321332
"""Find the type variables of the function type and bind them in our tvar_scope"""
13331333
if fun_type.variables:
1334+
defs = []
13341335
for var in fun_type.variables:
13351336
var_node = self.lookup_qualified(var.name, defn)
13361337
assert var_node, "Binding for function type variable not found within function"
13371338
var_expr = var_node.node
13381339
assert isinstance(var_expr, TypeVarLikeExpr)
1339-
self.tvar_scope.bind_new(var.name, var_expr)
1340-
return fun_type.variables
1340+
binding = self.tvar_scope.bind_new(var.name, var_expr)
1341+
defs.append(binding)
1342+
return defs
13411343
typevars = self.infer_type_variables(fun_type)
13421344
# Do not define a new type variable if already defined in scope.
13431345
typevars = [
13441346
(name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn)
13451347
]
1346-
defs: list[TypeVarLikeType] = []
1348+
defs = []
13471349
for name, tvar in typevars:
13481350
if not self.tvar_scope.allow_binding(tvar.fullname):
13491351
self.fail(
13501352
f'Type variable "{name}" is bound by an outer class',
13511353
defn,
13521354
code=codes.VALID_TYPE,
13531355
)
1354-
self.tvar_scope.bind_new(name, tvar)
1355-
binding = self.tvar_scope.get_binding(tvar.fullname)
1356-
assert binding is not None
1356+
binding = self.tvar_scope.bind_new(name, tvar)
13571357
defs.append(binding)
13581358

13591359
return defs

‎test-data/unit/check-classes.test‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1060,6 +1060,16 @@ def f() -> None:
10601060
a.g(a) # E: Too many arguments for "g" of "A"
10611061
[targets __main__, __main__.f]
10621062

1063+
[case testGenericClassWithinFunction]
1064+
from typing import TypeVar
1065+
1066+
def test() -> None:
1067+
T = TypeVar('T', bound='Foo')
1068+
class Foo:
1069+
def bar(self, foo: T) -> T:
1070+
pass
1071+
reveal_type(Foo.bar) # N: Revealed type is "def [T <: __main__.Foo@5] (self: __main__.Foo@5, foo: T`-1) -> T`-1"
1072+
10631073
[case testConstructNestedClass]
10641074
import typing
10651075
class A:

‎test-data/unit/check-namedtuple.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1284,7 +1284,7 @@ from typing import NamedTuple, TypeVar
12841284

12851285
T = TypeVar("T")
12861286
NT = NamedTuple("NT", [("key", int), ("value", T)])
1287-
reveal_type(NT) # N: Revealed type is "def [T] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1, fallback=__main__.NT[T`-1]]"
1287+
reveal_type(NT) # N: Revealed type is "def [T] (key: builtins.int, value: T`1) -> Tuple[builtins.int, T`1, fallback=__main__.NT[T`1]]"
12881288

12891289
nts: NT[str]
12901290
reveal_type(nts) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]"

‎test-data/unit/check-typeddict.test‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2550,7 +2550,7 @@ from typing import TypedDict, TypeVar
25502550

25512551
T = TypeVar("T")
25522552
TD = TypedDict("TD", {"key": int, "value": T})
2553-
reveal_type(TD) # N: Revealed type is "def [T] (*, key: builtins.int, value: T`-1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`-1})"
2553+
reveal_type(TD) # N: Revealed type is "def [T] (*, key: builtins.int, value: T`1) -> TypedDict('__main__.TD', {'key': builtins.int, 'value': T`1})"
25542554

25552555
tds: TD[str]
25562556
reveal_type(tds) # N: Revealed type is "TypedDict('__main__.TD', {'key': builtins.int, 'value': builtins.str})"

0 commit comments

Comments
 (0)