Skip to content

Commit 55ee0bd

Browse files
ilevkivskyiilevkivskyi
authored
Fix type variable defaults depending on previous variables (#21526)
Fixes #19192 Fixes #20027 Fixes few TODOs in tests (don't know if there are relevant issues) Closes #19382 Core idea is simple: don't do bizarre things :-) More precisely, instead of various in-place modifications (which is a big no-no for types), simply expand defaults iteratively as we build the environment for the type. Couple additional things: * Update defaults in `freshen_function_type_vars()` this is required for consistency when updating `TypeVarId`s. * Use `fix_instance()` in `checkexpr.py` for type applications. This will apply _all_ type arguments in cases like `x = Foo[int](...)`. The PEP and the spec say this is how it should be.
1 parent cde4779 commit 55ee0bd

8 files changed

Lines changed: 94 additions & 111 deletions

File tree

‎mypy/applytype.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,12 @@ def get_target_type(
3737
report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None],
3838
context: Context,
3939
skip_unsatisfied: bool,
40+
id_to_type: dict[TypeVarId, Type],
4041
) -> Type | None:
4142
p_type = get_proper_type(type)
4243
if isinstance(p_type, UninhabitedType) and p_type.ambiguous and tvar.has_default():
43-
return tvar.default
44+
# Gradually expand defaults, as they may depend on previous type variables.
45+
return expand_type(tvar.default, id_to_type)
4446
if isinstance(tvar, ParamSpecType):
4547
return type
4648
if isinstance(tvar, TypeVarTupleType):
@@ -113,7 +115,13 @@ def apply_generic_arguments(
113115
continue
114116

115117
target_type = get_target_type(
116-
tvar, type, callable, report_incompatible_typevar_value, context, skip_unsatisfied
118+
tvar,
119+
type,
120+
callable,
121+
report_incompatible_typevar_value,
122+
context,
123+
skip_unsatisfied,
124+
id_to_type,
117125
)
118126
if target_type is not None:
119127
id_to_type[tvar.id] = target_type

‎mypy/checkexpr.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5079,11 +5079,7 @@ class C(Generic[T, Unpack[Ts]]): ...
50795079
return [AnyType(TypeOfAny.from_error)] * len(vars)
50805080

50815081
# TODO: in future we may want to support type application to variadic functions.
5082-
if (
5083-
not vars
5084-
or not any(isinstance(v, TypeVarTupleType) for v in vars)
5085-
or not t.is_type_obj()
5086-
):
5082+
if not vars or not t.is_type_obj() or t.type_object().fullname == "builtins.tuple":
50875083
return list(args)
50885084
info = t.type_object()
50895085
# We reuse the logic from semanal phase to reduce code duplication.
@@ -5097,6 +5093,9 @@ class C(Generic[T, Unpack[Ts]]): ...
50975093
)
50985094
args = list(fake.args)
50995095

5096+
if not any(isinstance(v, TypeVarTupleType) for v in vars):
5097+
return args
5098+
51005099
prefix = next(i for (i, v) in enumerate(vars) if isinstance(v, TypeVarTupleType))
51015100
suffix = len(vars) - prefix - 1
51025101
tvt = vars[prefix]

‎mypy/expandtype.py‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ def freshen_function_type_vars(callee: F) -> F:
129129
tv = v.new_unification_variable(v)
130130
tvs.append(tv)
131131
tvmap[v.id] = tv
132+
if tv.has_default():
133+
# Point to fresh ids in case defaults depend on previous variables.
134+
tv.default = expand_type(tv.default, tvmap)
132135
fresh = expand_type(callee, tvmap).copy_modified(variables=tvs)
133136
return cast(F, fresh)
134137
else:
@@ -182,7 +185,6 @@ class ExpandTypeVisitor(TrivialSyntheticTypeTranslator):
182185
def __init__(self, variables: Mapping[TypeVarId, Type]) -> None:
183186
super().__init__()
184187
self.variables = variables
185-
self.recursive_tvar_guard: dict[TypeVarId, Type | None] | None = None
186188

187189
def visit_unbound_type(self, t: UnboundType) -> Type:
188190
return t
@@ -245,16 +247,6 @@ def visit_type_var(self, t: TypeVarType) -> Type:
245247
# TODO: do we really need to do this?
246248
# If I try to remove this special-casing ~40 tests fail on reveal_type().
247249
return repl.copy_modified(last_known_value=None)
248-
if isinstance(repl, TypeVarType) and repl.has_default():
249-
if self.recursive_tvar_guard is None:
250-
self.recursive_tvar_guard = {}
251-
if (tvar_id := repl.id) in self.recursive_tvar_guard:
252-
return self.recursive_tvar_guard[tvar_id] or repl
253-
self.recursive_tvar_guard[tvar_id] = None
254-
repl.default = repl.default.accept(self)
255-
expanded = repl.accept(self) # Note: `expanded is repl` may be true.
256-
repl = repl if isinstance(expanded, TypeVarType) else expanded
257-
self.recursive_tvar_guard[tvar_id] = repl
258250
return repl
259251

260252
def visit_param_spec(self, t: ParamSpecType) -> Type:

‎mypy/semanal.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2466,9 +2466,6 @@ def tvar_defs_from_tvars(
24662466
tvar_defs: list[TypeVarLikeType] = []
24672467
last_tvar_name_with_default: str | None = None
24682468
for name, tvar_expr in tvars:
2469-
tvar_expr.default = tvar_expr.default.accept(
2470-
TypeVarDefaultTranslator(self, tvar_expr.name, context)
2471-
)
24722469
# PEP-695 type variables that are redeclared in an inner scope are warned
24732470
# about elsewhere.
24742471
if not tvar_expr.is_new_style and not self.tvar_scope.allow_binding(
@@ -2478,6 +2475,11 @@ def tvar_defs_from_tvars(
24782475
message_registry.TYPE_VAR_REDECLARED_IN_NESTED_CLASS.format(name), context
24792476
)
24802477
tvar_def = self.tvar_scope.bind_new(name, tvar_expr, self.fail, context)
2478+
# Fix any residual UnboundTypes in the generated TypeVarLike, keep
2479+
# TypeVarLikeExpr untouched as it may be shared by multiple classes.
2480+
tvar_def.default = tvar_def.default.accept(
2481+
TypeVarDefaultTranslator(self, tvar_expr.name, context)
2482+
)
24812483
if last_tvar_name_with_default is not None and not tvar_def.has_default():
24822484
self.msg.tvar_without_default_type(
24832485
tvar_def.name, last_tvar_name_with_default, context

‎mypy/tvar_scope.py‎

Lines changed: 3 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,8 @@
1212
TypeVarTupleExpr,
1313
)
1414
from mypy.types import (
15-
AnyType,
1615
ParamSpecFlavor,
1716
ParamSpecType,
18-
TrivialSyntheticTypeTranslator,
19-
Type,
20-
TypeAliasType,
21-
TypeOfAny,
2217
TypeVarId,
2318
TypeVarLikeType,
2419
TypeVarTupleType,
@@ -28,54 +23,6 @@
2823
FailFunc: _TypeAlias = Callable[[str, Context], None]
2924

3025

31-
class TypeVarLikeDefaultFixer(TrivialSyntheticTypeTranslator):
32-
"""Set namespace for all TypeVarLikeTypes types."""
33-
34-
def __init__(
35-
self,
36-
scope: TypeVarLikeScope,
37-
fail_func: FailFunc,
38-
source_tv: TypeVarLikeExpr,
39-
context: Context,
40-
) -> None:
41-
self.scope = scope
42-
self.fail_func = fail_func
43-
self.source_tv = source_tv
44-
self.context = context
45-
super().__init__()
46-
47-
def visit_type_var(self, t: TypeVarType) -> Type:
48-
existing = self.scope.get_binding(t.fullname)
49-
if existing is None:
50-
self._report_unbound_tvar(t)
51-
return AnyType(TypeOfAny.from_error)
52-
return existing
53-
54-
def visit_param_spec(self, t: ParamSpecType) -> Type:
55-
existing = self.scope.get_binding(t.fullname)
56-
if existing is None:
57-
self._report_unbound_tvar(t)
58-
return AnyType(TypeOfAny.from_error)
59-
return existing
60-
61-
def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type:
62-
existing = self.scope.get_binding(t.fullname)
63-
if existing is None:
64-
self._report_unbound_tvar(t)
65-
return AnyType(TypeOfAny.from_error)
66-
return existing
67-
68-
def visit_type_alias_type(self, t: TypeAliasType) -> Type:
69-
return t
70-
71-
def _report_unbound_tvar(self, tvar: TypeVarLikeType) -> None:
72-
self.fail_func(
73-
f"Type variable {tvar.name} referenced in the default"
74-
f" of {self.source_tv.name} is unbound",
75-
self.context,
76-
)
77-
78-
7926
class TypeVarLikeScope:
8027
"""Scope that holds bindings for type variables and parameter specifications.
8128
@@ -148,23 +95,14 @@ def bind_new(
14895
i = self.func_id
14996
namespace = self.namespace
15097

151-
# Defaults may reference other type variables. That is only valid when the
152-
# referenced variable is already in scope (textually precedes the definition we're
153-
# processing now).
154-
default = tvar_expr.default.accept(
155-
TypeVarLikeDefaultFixer(
156-
self, fail_func=fail_func, source_tv=tvar_expr, context=context
157-
)
158-
)
159-
16098
if isinstance(tvar_expr, TypeVarExpr):
16199
tvar_def: TypeVarLikeType = TypeVarType(
162100
name=name,
163101
fullname=tvar_expr.fullname,
164102
id=TypeVarId(i, namespace=namespace),
165103
values=tvar_expr.values,
166104
upper_bound=tvar_expr.upper_bound,
167-
default=default,
105+
default=tvar_expr.default,
168106
variance=tvar_expr.variance,
169107
line=tvar_expr.line,
170108
column=tvar_expr.column,
@@ -176,7 +114,7 @@ def bind_new(
176114
id=TypeVarId(i, namespace=namespace),
177115
flavor=ParamSpecFlavor.BARE,
178116
upper_bound=tvar_expr.upper_bound,
179-
default=default,
117+
default=tvar_expr.default,
180118
line=tvar_expr.line,
181119
column=tvar_expr.column,
182120
)
@@ -187,7 +125,7 @@ def bind_new(
187125
id=TypeVarId(i, namespace=namespace),
188126
upper_bound=tvar_expr.upper_bound,
189127
tuple_fallback=tvar_expr.tuple_fallback,
190-
default=default,
128+
default=tvar_expr.default,
191129
line=tvar_expr.line,
192130
column=tvar_expr.column,
193131
)

‎mypy/typeanal.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2098,15 +2098,16 @@ def fix_instance(
20982098
disallow_any, fail, note, t, options, fullname, unexpanded_type
20992099
)
21002100
arg = any_type
2101+
with state.strict_optional_set(options.strict_optional):
2102+
# Gradually expand defaults, as they may depend on previous variables.
2103+
if tv.has_default():
2104+
arg = expand_type(arg, env)
2105+
env[tv.id] = arg
21012106
args.append(arg)
2102-
env[tv.id] = arg
2107+
else:
2108+
env[tv.id] = arg
21032109
t.args = tuple(args)
21042110
fix_type_var_tuple_argument(t)
2105-
if not t.type.has_type_var_tuple_type:
2106-
with state.strict_optional_set(True):
2107-
fixed = expand_type(t, env)
2108-
assert isinstance(fixed, Instance)
2109-
t.args = fixed.args
21102111

21112112

21122113
def instantiate_type_alias(
@@ -2288,7 +2289,6 @@ def set_any_tvars(
22882289

22892290
env: dict[TypeVarId, Type] = {}
22902291
used_any_type = False
2291-
has_type_var_tuple_type = False
22922292
for tv, arg in itertools.zip_longest(node.alias_tvars, args, fillvalue=None):
22932293
if tv is None:
22942294
continue
@@ -2300,16 +2300,16 @@ def set_any_tvars(
23002300
used_any_type = True
23012301
if isinstance(tv, TypeVarTupleType):
23022302
# TODO Handle TypeVarTuple defaults
2303-
has_type_var_tuple_type = True
23042303
arg = UnpackType(Instance(tv.tuple_fallback.type, [any_type]))
2304+
with state.strict_optional_set(options.strict_optional):
2305+
# Gradually expand defaults, as they may depend on previous variables.
2306+
if tv.has_default():
2307+
arg = expand_type(arg, env)
2308+
env[tv.id] = arg
23052309
args.append(arg)
2306-
env[tv.id] = arg
2310+
else:
2311+
env[tv.id] = arg
23072312
t = TypeAliasType(node, args, newline, newcolumn)
2308-
if not has_type_var_tuple_type:
2309-
with state.strict_optional_set(options.strict_optional):
2310-
fixed = expand_type(t, env)
2311-
assert isinstance(fixed, TypeAliasType)
2312-
t.args = fixed.args
23132313

23142314
if used_any_type and disallow_any and node.alias_tvars:
23152315
assert fail is not None

‎test-data/unit/check-python313.test‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ reveal_type(A2().x) # N: Revealed type is "builtins.int"
291291
reveal_type(A3().x) # N: Revealed type is "builtins.int"
292292
[builtins fixtures/tuple.pyi]
293293

294-
[case testTypeVarDefaultToAnotherTypeVar]
294+
[case testTypeVarDefaultToAnotherTypeVarClass]
295295
class A[X, Y = X, Z = Y]:
296296
x: X
297297
y: Y
@@ -300,8 +300,7 @@ class A[X, Y = X, Z = Y]:
300300
a1: A[int]
301301
reveal_type(a1.x) # N: Revealed type is "builtins.int"
302302
reveal_type(a1.y) # N: Revealed type is "builtins.int"
303-
# TODO: this must reveal `int` as well:
304-
reveal_type(a1.z) # N: Revealed type is "X`1"
303+
reveal_type(a1.z) # N: Revealed type is "builtins.int"
305304

306305
a2: A[int, str]
307306
reveal_type(a2.x) # N: Revealed type is "builtins.int"
@@ -314,6 +313,35 @@ reveal_type(a3.y) # N: Revealed type is "builtins.str"
314313
reveal_type(a3.z) # N: Revealed type is "builtins.bool"
315314
[builtins fixtures/tuple.pyi]
316315

316+
[case testTypeVarDefaultToAnotherTypeVarAlias]
317+
type A[X, Y = X, Z = Y] = tuple[X, Y, Z]
318+
319+
a1: A[int]
320+
reveal_type(a1[0]) # N: Revealed type is "builtins.int"
321+
reveal_type(a1[1]) # N: Revealed type is "builtins.int"
322+
reveal_type(a1[2]) # N: Revealed type is "builtins.int"
323+
324+
a2: A[int, str]
325+
reveal_type(a2[0]) # N: Revealed type is "builtins.int"
326+
reveal_type(a2[1]) # N: Revealed type is "builtins.str"
327+
reveal_type(a2[2]) # N: Revealed type is "builtins.str"
328+
329+
a3: A[int, str, bool]
330+
reveal_type(a3[0]) # N: Revealed type is "builtins.int"
331+
reveal_type(a3[1]) # N: Revealed type is "builtins.str"
332+
reveal_type(a3[2]) # N: Revealed type is "builtins.bool"
333+
[builtins fixtures/tuple.pyi]
334+
335+
[case testTypeVarDefaultAssertTypeType]
336+
from typing import assert_type
337+
338+
class NoNonDefaults[T = str, S = int]: ...
339+
340+
assert_type(NoNonDefaults[()], type[NoNonDefaults[str, int]])
341+
assert_type(NoNonDefaults[str], type[NoNonDefaults[str, int]])
342+
assert_type(NoNonDefaults[str, int], type[NoNonDefaults[str, int]])
343+
[builtins fixtures/tuple.pyi]
344+
317345
[case testTypeVarDefaultToAnotherTypeVarWrong]
318346
class A[Y = X, X = int]: ... # E: Name "X" is not defined
319347

‎test-data/unit/check-typevar-defaults.test‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,7 @@ def func_error_alias1(
137137
reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, Any]"
138138
reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
139139

140-
TERR2 = Dict[T4, T3] # TODO should be an error \
141-
# Type parameter "T4" has a default type that refers to one or more type variables that are out of scope
140+
TERR2 = Dict[T4, T3] # E: Type parameter "T4" has a default type that refers to one or more type variables that are out of scope
142141

143142
def func_error_alias2(
144143
a: TERR2,
@@ -518,8 +517,8 @@ def func_d3(
518517
reveal_type(b) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
519518
reveal_type(c) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]"
520519

521-
# k = ClassD3()
522-
# reveal_type(k) # Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]" # TODO
520+
k = ClassD3()
521+
reveal_type(k) # N: Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]"
523522
l = ClassD3[int]()
524523
reveal_type(l) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
525524
m = ClassD3[int, float]()
@@ -904,6 +903,19 @@ Alias: TypeAlias = "MyClass[T1, T2]"
904903
class MyClass(Generic["T1", "T2"]): ...
905904
[builtins fixtures/tuple.pyi]
906905

906+
[case testTypeVariableDefaultNotSticky]
907+
from typing import Generic, TypeVar
908+
909+
T1 = TypeVar("T1")
910+
T2 = TypeVar("T2", default=T1)
911+
912+
class MyClass(Generic[T1, T2]): ...
913+
914+
reveal_type(MyClass[int]) # N: Revealed type is "def () -> __main__.MyClass[builtins.int, builtins.int]"
915+
reveal_type(MyClass[str]) # N: Revealed type is "def () -> __main__.MyClass[builtins.str, builtins.str]"
916+
reveal_type(MyClass[bytes]) # N: Revealed type is "def () -> __main__.MyClass[builtins.bytes, builtins.bytes]"
917+
[builtins fixtures/tuple.pyi]
918+
907919
[case testDefaultsApplicationInAliasNoCrashNested]
908920
from typing import Generic, TypeVar
909921
from typing_extensions import TypeAlias
@@ -928,19 +940,23 @@ T3 = TypeVar("T3", default=T2)
928940
class A(Generic[T1, T2, T3]): ...
929941
reveal_type(A) # N: Revealed type is "def [T1, T2 = T1`1, T3 = T2`2 = T1`1] () -> __main__.A[T1`1, T2`2 = T1`1, T3`3 = T2`2 = T1`1]"
930942
a: A[int]
931-
reveal_type(a) # N: Revealed type is "__main__.A[builtins.int, builtins.int, T1`1]"
943+
reveal_type(a) # N: Revealed type is "__main__.A[builtins.int, builtins.int, builtins.int]"
944+
945+
AA = tuple[T1, T2, T3]
946+
aa: AA[str]
947+
reveal_type(aa) # N: Revealed type is "tuple[builtins.str, builtins.str, builtins.str]"
932948

933-
class B(Generic[T1, T3]): ... # E: Type variable T2 referenced in the default of T3 is unbound
949+
class B(Generic[T1, T3]): ... # E: Type parameter "T3" has a default type that refers to one or more type variables that are out of scope
934950
reveal_type(B) # N: Revealed type is "def [T1, T3 = Any] () -> __main__.B[T1`1, T3`2 = Any]"
935951
b: B[int]
936952
reveal_type(b) # N: Revealed type is "__main__.B[builtins.int, Any]"
937953

938-
class C(Generic[T2]): ... # E: Type variable T1 referenced in the default of T2 is unbound
954+
class C(Generic[T2]): ... # E: Type parameter "T2" has a default type that refers to one or more type variables that are out of scope
939955
reveal_type(C) # N: Revealed type is "def [T2 = Any] () -> __main__.C[T2`1 = Any]"
940956
c: C
941957
reveal_type(c) # N: Revealed type is "__main__.C[Any]"
942958

943-
class D(Generic[T2, T1]): ... # E: Type variable T1 referenced in the default of T2 is unbound \
959+
class D(Generic[T2, T1]): ... # E: Type parameter "T2" has a default type that refers to one or more type variables that are out of scope \
944960
# E: "T1" cannot appear after "T2" in type parameter list because it has no default type
945961
reveal_type(D) # N: Revealed type is "def [T2 = Any, T1 = Any] () -> __main__.D[T2`1 = Any, T1`2 = Any]"
946962
d: D

0 commit comments

Comments
 (0)