Skip to content

Commit 1d6905a

Browse files
authored
Fix various bugs in TypeVarTuples with defaults (#21544)
Fixes a bunch of TODOs in tests. Fixes #20449 Couple non-trivial things here: First, I prohibit (regular) type variables with defaults that appear after a `TypeVarTuple`. The PEP and the spec are clear that these are ambiguous and invalid. But, because they are ambiguous, we can't simply give the error, since otherwise various parts of code will behave in weirdly inconsistent ways. Therefore, I completely erase ambiguous type variables out of existence, 1984 style. Second, couple commented-out tests cases suggested that `TypeVarTuple` defaults should be used in cases where `*tuple[()]` would be used as a value otherwise, plus `Foo[X, Y]` and `Foo[X, Y, *tuple[()]]` should be interpreted differently. I don't see anything about this in the PEP/spec, and I don't like this. I think the default should be used _only_ in cases where `Any` or `Never` would be used otherwise.
1 parent 8f4c840 commit 1d6905a

5 files changed

Lines changed: 125 additions & 56 deletions

File tree

‎mypy/message_registry.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
181181
'Implicit generic "Any". Use "{}" and specify generic parameters'
182182
)
183183
NO_CYCLIC_DEFAULT: Final = "Cyclic type variable defaults are not supported"
184+
NO_DEFAULT_AFTER_TYPEVAR_TUPLE: Final = "A type variable with default cannot follow TypeVarTuple"
184185
INVALID_UNPACK: Final = "{} cannot be unpacked (must be tuple or TypeVarTuple)"
185186
INVALID_UNPACK_POSITION: Final = "Unpack is only valid in a variadic position"
186187
INVALID_PARAM_SPEC_LOCATION: Final = "Invalid location for ParamSpec {}"

‎mypy/semanal.py‎

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -546,8 +546,13 @@ def __init__(
546546
# to create the set lazily.
547547
self.types_fixed: set[TypeInfo | TypeAlias] | None = None
548548

549+
# Stack of type variables that have been removed from current class because they
550+
# cannot be bound unambiguously. This can happen if a (regular) type variable
551+
# with a default follows a type variable tuple.
552+
self.removed_type_vars: list[list[TypeVarType]] = [[]]
553+
549554
# mypyc doesn't properly handle implementing an abstractproperty
550-
# with a regular attribute so we make them properties
555+
# with a regular attribute, so we make them properties
551556
@property
552557
def type(self) -> TypeInfo | None:
553558
return self._type
@@ -1836,8 +1841,9 @@ def visit_class_def(self, defn: ClassDef) -> None:
18361841
if self.push_type_args(defn.type_args, defn) is None:
18371842
self.mark_incomplete(defn.name, defn)
18381843
return
1839-
1844+
self.removed_type_vars.append([])
18401845
self.analyze_class(defn)
1846+
self.removed_type_vars.pop()
18411847
self.pop_type_args(defn.type_args)
18421848
self.incomplete_type_stack.pop()
18431849

@@ -2084,7 +2090,21 @@ def check_type_alias_bases(self, bases: list[Expression]) -> None:
20842090
)
20852091

20862092
def setup_type_vars(self, defn: ClassDef, tvar_defs: list[TypeVarLikeType]) -> None:
2087-
defn.type_vars = tvar_defs
2093+
seen_tvt = False
2094+
valid_tvar_defs = []
2095+
for tv in tvar_defs:
2096+
if seen_tvt and isinstance(tv, TypeVarType) and tv.has_default():
2097+
self.fail(
2098+
message_registry.NO_DEFAULT_AFTER_TYPEVAR_TUPLE, defn, code=codes.TYPE_VAR
2099+
)
2100+
# Remove the ambiguous type variable, and record it, so that we can replace
2101+
# all its uses with Any.
2102+
self.removed_type_vars[-1].append(tv)
2103+
continue
2104+
if isinstance(tv, TypeVarTupleType):
2105+
seen_tvt = True
2106+
valid_tvar_defs.append(tv)
2107+
defn.type_vars = valid_tvar_defs
20882108
defn.info.type_vars = []
20892109
# we want to make sure any additional logic in add_type_vars gets run
20902110
defn.info.add_type_vars()
@@ -4017,6 +4037,28 @@ def analyze_alias(
40174037
with self.allow_unbound_tvars_set():
40184038
rvalue.accept(self)
40194039

4040+
new_tvar_defs = []
4041+
erase_tvar_defs = []
4042+
variadic = False
4043+
for td in tvar_defs:
4044+
if variadic and isinstance(td, TypeVarType) and td.has_default():
4045+
self.fail(
4046+
message_registry.NO_DEFAULT_AFTER_TYPEVAR_TUPLE,
4047+
rvalue,
4048+
code=codes.TYPE_VAR,
4049+
)
4050+
# Remove the ambiguous type variable, and record it, so that we can
4051+
# replace all its uses with Any.
4052+
erase_tvar_defs.append(td)
4053+
continue
4054+
if isinstance(td, TypeVarTupleType):
4055+
# There can be only one variadic variable at most,
4056+
# the error is reported elsewhere.
4057+
if variadic:
4058+
continue
4059+
variadic = True
4060+
new_tvar_defs.append(td)
4061+
40204062
analyzed, depends_on = analyze_type_alias(
40214063
typ,
40224064
self,
@@ -4029,20 +4071,11 @@ def analyze_alias(
40294071
in_dynamic_func=dynamic,
40304072
global_scope=global_scope,
40314073
allowed_alias_tvars=tvar_defs,
4074+
erase_tvar_defs=erase_tvar_defs,
40324075
alias_type_params_names=all_declared_type_params_names,
40334076
python_3_12_type_alias=python_3_12_type_alias,
40344077
)
40354078

4036-
# There can be only one variadic variable at most, the error is reported elsewhere.
4037-
new_tvar_defs = []
4038-
variadic = False
4039-
for td in tvar_defs:
4040-
if isinstance(td, TypeVarTupleType):
4041-
if variadic:
4042-
continue
4043-
variadic = True
4044-
new_tvar_defs.append(td)
4045-
40464079
indexed = bool(isinstance(typ, UnboundType) and (typ.args or typ.empty_tuple_index))
40474080
default_depends = {}
40484081
for _, tv in alias_type_vars:
@@ -7792,6 +7825,7 @@ def type_analyzer(
77927825
prohibit_special_class_field_types=prohibit_special_class_field_types,
77937826
allow_type_any=allow_type_any,
77947827
analyzing_tvar_def=analyzing_tvar_def,
7828+
erase_tvar_defs=self.removed_type_vars[-1],
77957829
)
77967830
tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic())
77977831
tpan.global_scope = not self.type and not self.function_stack

‎mypy/typeanal.py‎

Lines changed: 54 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@ def analyze_type_alias(
154154
in_dynamic_func: bool = False,
155155
global_scope: bool = True,
156156
allowed_alias_tvars: list[TypeVarLikeType] | None = None,
157+
erase_tvar_defs: list[TypeVarType] | None = None,
157158
alias_type_params_names: list[str] | None = None,
158159
python_3_12_type_alias: bool = False,
159160
) -> tuple[Type, set[str]]:
@@ -174,6 +175,7 @@ def analyze_type_alias(
174175
allow_placeholder=allow_placeholder,
175176
prohibit_self_type="type alias target",
176177
allowed_alias_tvars=allowed_alias_tvars,
178+
erase_tvar_defs=erase_tvar_defs,
177179
alias_type_params_names=alias_type_params_names,
178180
python_3_12_type_alias=python_3_12_type_alias,
179181
)
@@ -220,6 +222,7 @@ def __init__(
220222
prohibit_self_type: str | None = None,
221223
prohibit_special_class_field_types: str | None = None,
222224
allowed_alias_tvars: list[TypeVarLikeType] | None = None,
225+
erase_tvar_defs: list[TypeVarType] | None = None,
223226
allow_type_any: bool = False,
224227
alias_type_params_names: list[str] | None = None,
225228
analyzing_tvar_def: bool = False,
@@ -240,6 +243,11 @@ def __init__(
240243
if allowed_alias_tvars is None:
241244
allowed_alias_tvars = []
242245
self.allowed_alias_tvars = allowed_alias_tvars
246+
# Should we erase some type variables? This can be used to mass-erase type
247+
# variables that were found to be invalid at the class/alias definition.
248+
if erase_tvar_defs is None:
249+
erase_tvar_defs = []
250+
self.erase_tvar_defs = erase_tvar_defs
243251
self.alias_type_params_names = alias_type_params_names
244252
# If false, record incomplete ref if we generate PlaceholderType.
245253
self.allow_placeholder = allow_placeholder
@@ -403,6 +411,13 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool)
403411
msg = f'Can\'t use bound type variable "{t.name}" to define generic alias'
404412
self.fail(msg, t, code=codes.VALID_TYPE)
405413
return AnyType(TypeOfAny.from_error)
414+
if (
415+
isinstance(sym.node, TypeVarExpr)
416+
and tvar_def is not None
417+
and tvar_def in self.erase_tvar_defs
418+
):
419+
# The caller should have already given a relevant error.
420+
return AnyType(TypeOfAny.from_error)
406421
if isinstance(sym.node, TypeVarExpr) and tvar_def is not None:
407422
assert isinstance(tvar_def, TypeVarType)
408423
if len(t.args) > 0:
@@ -2108,7 +2123,10 @@ def fix_instance(
21082123
"""
21092124
used_default = False
21102125
arg_count = len(t.args)
2111-
min_tv_count = sum(not tv.has_default() for tv in t.type.defn.type_vars)
2126+
min_tv_count = sum(
2127+
not tv.has_default() and not isinstance(tv, TypeVarTupleType)
2128+
for tv in t.type.defn.type_vars
2129+
)
21122130
max_tv_count = len(t.type.type_vars)
21132131
if arg_count < min_tv_count or arg_count > max_tv_count:
21142132
# Don't use existing args if arg_count doesn't match
@@ -2117,9 +2135,10 @@ def fix_instance(
21172135
disallow_any = False
21182136
t.args = ()
21192137

2120-
args: list[Type] = [*(t.args[:max_tv_count])]
2138+
args: list[Type] = list(t.args)
21212139
any_type: AnyType | None = None
21222140
env: dict[TypeVarId, Type] = {}
2141+
tvt_no_default = False
21232142

21242143
for tv, arg in itertools.zip_longest(t.type.defn.type_vars, t.args, fillvalue=None):
21252144
if tv is None:
@@ -2152,16 +2171,27 @@ def fix_instance(
21522171
arg = any_type
21532172
else:
21542173
assert arg is not None
2155-
with state.strict_optional_set(options.strict_optional):
2156-
# Gradually expand defaults, as they may depend on previous variables.
2157-
if tv.has_default():
2158-
arg = expand_type(arg, env)
2159-
env[tv.id] = arg
2160-
args.append(arg)
2174+
if use_any and isinstance(tv, TypeVarTupleType):
2175+
tvt_no_default = True
2176+
# Default such as *tuple[int, str] should be unpacked into individual items.
2177+
if isinstance(arg, UnpackType) and isinstance(
2178+
unpack := get_proper_type(arg.type), TupleType
2179+
):
2180+
unpacked = unpack.items
2181+
else:
2182+
unpacked = [arg]
2183+
for arg in unpacked:
2184+
with state.strict_optional_set(options.strict_optional):
2185+
# Gradually expand defaults, as they may depend on previous variables.
2186+
if tv.has_default():
2187+
arg = expand_type(arg, env)
2188+
env[tv.id] = arg
2189+
args.append(arg)
21612190
else:
21622191
env[tv.id] = arg
21632192
t.args = tuple(args)
2164-
fix_type_var_tuple_argument(t)
2193+
if tvt_no_default:
2194+
fix_type_var_tuple_argument(t)
21652195
return used_default
21662196

21672197

@@ -2380,15 +2410,22 @@ def set_any_tvars(
23802410
else:
23812411
arg = any_type
23822412
used_any_type = True
2383-
if isinstance(tv, TypeVarTupleType):
2384-
# TODO Handle TypeVarTuple defaults
2413+
if used_any_type and isinstance(tv, TypeVarTupleType):
23852414
arg = UnpackType(Instance(tv.tuple_fallback.type, [any_type]))
2386-
with state.strict_optional_set(options.strict_optional):
2387-
# Gradually expand defaults, as they may depend on previous variables.
2388-
if tv.has_default():
2389-
arg = expand_type(arg, env)
2390-
env[tv.id] = arg
2391-
args.append(arg)
2415+
# Default such as *tuple[int, str] should be unpacked into individual items.
2416+
if isinstance(arg, UnpackType) and isinstance(
2417+
unpack := get_proper_type(arg.type), TupleType
2418+
):
2419+
unpacked = unpack.items
2420+
else:
2421+
unpacked = [arg]
2422+
for arg in unpacked:
2423+
with state.strict_optional_set(options.strict_optional):
2424+
# Gradually expand defaults, as they may depend on previous variables.
2425+
if tv.has_default():
2426+
arg = expand_type(arg, env)
2427+
env[tv.id] = arg
2428+
args.append(arg)
23922429
else:
23932430
env[tv.id] = arg
23942431
t = TypeAliasType(node, args, newline, newcolumn)

‎test-data/unit/check-python313.test‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ reveal_type(func_b1(callback1)) # N: Revealed type is "def (x: builtins.str)"
140140
reveal_type(func_b1(2)) # N: Revealed type is "def (builtins.int, builtins.str)"
141141

142142
def func_c1[*Ts = *tuple[int, str]](x: int | Callable[[*Ts], None]) -> tuple[*Ts]: ...
143-
# reveal_type(func_c1(callback1)) # Revealed type is "Tuple[str]" # TODO
143+
reveal_type(func_c1(callback1)) # N: Revealed type is "tuple[builtins.str]"
144144
reveal_type(func_c1(2)) # N: Revealed type is "tuple[builtins.int, builtins.str]"
145145
[builtins fixtures/tuple.pyi]
146146

@@ -195,7 +195,7 @@ def func_c1(
195195
a: ClassC1,
196196
b: ClassC1[float],
197197
) -> None:
198-
# reveal_type(a) # Revealed type is "__main__.ClassC1[builtins.int, builtins.str]" # TODO
198+
reveal_type(a) # N: Revealed type is "__main__.ClassC1[builtins.int, builtins.str]"
199199
reveal_type(b) # N: Revealed type is "__main__.ClassC1[builtins.float]"
200200

201201
k = ClassC1()
@@ -250,7 +250,7 @@ def func_c1(
250250
a: TC1,
251251
b: TC1[float],
252252
) -> None:
253-
# reveal_type(a) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO
253+
reveal_type(a) # N: Revealed type is "tuple[builtins.int, builtins.str]"
254254
reveal_type(b) # N: Revealed type is "tuple[builtins.float]"
255255

256256
[builtins fixtures/tuple.pyi]

0 commit comments

Comments
 (0)