Skip to content

Commit ed82143

Browse files
ilevkivskyiilevkivskyi
authored
Handle edge case in overload subtyping (#22007)
Fixes #495 Fixes #12379 The solution is straightforward: we already have an existing (decent) method to union similar callables, so I simply refactor the code so that it can be used in `subtypes.py` to handle the relevant edge case.
1 parent 4f4877e commit ed82143

3 files changed

Lines changed: 155 additions & 104 deletions

File tree

‎mypy/checkexpr.py‎

Lines changed: 15 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,9 @@
126126
find_member,
127127
is_same_type,
128128
is_subtype,
129+
merge_typevars_in_callables_by_name,
129130
non_method_protocol_members,
131+
union_function_signatures,
130132
)
131133
from mypy.traverser import (
132134
all_name_and_member_expressions,
@@ -3298,75 +3300,30 @@ def type_overrides_set(
32983300

32993301
def combine_function_signatures(self, types: list[ProperType]) -> AnyType | CallableType:
33003302
"""Accepts a list of function signatures and attempts to combine them together into a
3301-
new CallableType consisting of the union of all of the given arguments and return types.
3303+
new CallableType consisting of the union of all the given arguments and return types.
33023304
33033305
If there is at least one non-callable type, return Any (this can happen if there is
33043306
an ambiguity because of Any in arguments).
33053307
"""
33063308
assert types, "Trying to merge no callables"
33073309
if not all(isinstance(c, CallableType) for c in types):
33083310
return AnyType(TypeOfAny.special_form)
3309-
33103311
callables = cast("list[CallableType]", types)
3311-
if len(callables) == 1:
3312-
return callables[0]
3313-
3314-
# Note: we are assuming here that if a user uses some TypeVar 'T' in
3315-
# two different functions, they meant for that TypeVar to mean the
3316-
# same thing.
3317-
#
3318-
# This function will make sure that all instances of that TypeVar 'T'
3319-
# refer to the same underlying TypeVarType objects to simplify the union-ing
3320-
# logic below.
3321-
#
3322-
# (If the user did *not* mean for 'T' to be consistently bound to the
3323-
# same type in their overloads, well, their code is probably too
3324-
# confusing and ought to be re-written anyways.)
3325-
callables, variables = merge_typevars_in_callables_by_name(callables)
33263312

3327-
new_args: list[list[Type]] = [[] for _ in callables[0].arg_types]
3328-
new_kinds = list(callables[0].arg_kinds)
3329-
new_returns: list[Type] = []
3330-
too_complex = False
3331-
3332-
for target in callables:
3333-
# We fall back to Callable[..., Union[<returns>]] if the functions do not have
3334-
# the exact same signature. The only exception is if one arg is optional and
3335-
# the other is positional: in that case, we continue unioning (and expect a
3336-
# positional arg).
3337-
# TODO: Enhance the merging logic to handle a wider variety of signatures.
3338-
if len(new_kinds) != len(target.arg_kinds):
3339-
too_complex = True
3340-
break
3341-
for i, (new_kind, target_kind) in enumerate(zip(new_kinds, target.arg_kinds)):
3342-
if new_kind == target_kind:
3343-
continue
3344-
if new_kind.is_positional() and target_kind.is_positional():
3345-
new_kinds[i] = ARG_POS
3346-
else:
3347-
too_complex = True
3348-
break
3349-
if too_complex:
3350-
break
3351-
for i, arg in enumerate(target.arg_types):
3352-
new_args[i].append(arg)
3353-
new_returns.append(target.ret_type)
3354-
3355-
union_return = make_simplified_union(new_returns)
3356-
if too_complex:
3357-
any = AnyType(TypeOfAny.special_form)
3358-
return callables[0].copy_modified(
3359-
arg_types=[any, any],
3360-
arg_kinds=[ARG_STAR, ARG_STAR2],
3361-
arg_names=[None, None],
3362-
ret_type=union_return,
3363-
variables=variables,
3364-
implicit=True,
3365-
)
3313+
combined = union_function_signatures(callables)
3314+
if combined is not None:
3315+
return combined
33663316

3317+
# We fall back to Callable[..., Union[<returns>]] if the functions do not have
3318+
# the exact same signature. The only exception is if one arg is optional and
3319+
# the other is positional (and expect a positional arg).
3320+
callables, variables = merge_typevars_in_callables_by_name(callables)
3321+
union_return = make_simplified_union([c.ret_type for c in callables])
3322+
any = AnyType(TypeOfAny.special_form)
33673323
return callables[0].copy_modified(
3368-
arg_types=[make_simplified_union(args) for args in new_args],
3369-
arg_kinds=new_kinds,
3324+
arg_types=[any, any],
3325+
arg_kinds=[ARG_STAR, ARG_STAR2],
3326+
arg_names=[None, None],
33703327
ret_type=union_return,
33713328
variables=variables,
33723329
implicit=True,
@@ -6965,51 +6922,6 @@ def all_same_types(types: list[Type]) -> bool:
69656922
return all(is_same_type(t, types[0]) for t in types[1:])
69666923

69676924

6968-
def merge_typevars_in_callables_by_name(
6969-
callables: Sequence[CallableType],
6970-
) -> tuple[list[CallableType], list[TypeVarType]]:
6971-
"""Takes all the typevars present in the callables and 'combines' the ones with the same name.
6972-
6973-
For example, suppose we have two callables with signatures "f(x: T, y: S) -> T" and
6974-
"f(x: List[Tuple[T, S]]) -> Tuple[T, S]". Both callables use typevars named "T" and
6975-
"S", but we treat them as distinct, unrelated typevars. (E.g. they could both have
6976-
distinct ids.)
6977-
6978-
If we pass in both callables into this function, it returns a list containing two
6979-
new callables that are identical in signature, but use the same underlying TypeVarType
6980-
for T and S.
6981-
6982-
This is useful if we want to take the output lists and "merge" them into one callable
6983-
in some way -- for example, when unioning together overloads.
6984-
6985-
Returns both the new list of callables and a list of all distinct TypeVarType objects used.
6986-
"""
6987-
output: list[CallableType] = []
6988-
unique_typevars: dict[str, TypeVarType] = {}
6989-
variables: list[TypeVarType] = []
6990-
6991-
for target in callables:
6992-
if target.is_generic():
6993-
target = freshen_function_type_vars(target)
6994-
6995-
rename = {} # Dict[TypeVarId, TypeVar]
6996-
for tv in target.variables:
6997-
name = tv.fullname
6998-
if name not in unique_typevars:
6999-
# TODO: support ParamSpecType and TypeVarTuple.
7000-
if isinstance(tv, (ParamSpecType, TypeVarTupleType)):
7001-
continue
7002-
assert isinstance(tv, TypeVarType)
7003-
unique_typevars[name] = tv
7004-
variables.append(tv)
7005-
rename[tv.id] = unique_typevars[name]
7006-
7007-
target = expand_type(target, rename)
7008-
output.append(target)
7009-
7010-
return output, variables
7011-
7012-
70136925
def try_getting_literal(typ: Type) -> ProperType:
70146926
"""If possible, get a more precise literal type for a given type."""
70156927
typ = get_proper_type(typ)

‎mypy/subtypes.py‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from __future__ import annotations
22

3-
from collections.abc import Callable, Iterable, Iterator
3+
from collections.abc import Callable, Iterable, Iterator, Sequence
44
from contextlib import contextmanager
55
from typing import Any, Final, TypeAlias as _TypeAlias, TypeVar, cast
66

@@ -20,6 +20,7 @@
2020
# Circular import; done in the function instead.
2121
# import mypy.solve
2222
from mypy.nodes import (
23+
ARG_POS,
2324
ARG_STAR,
2425
ARG_STAR2,
2526
CONTRAVARIANT,
@@ -60,6 +61,7 @@
6061
TypedDictType,
6162
TypeOfAny,
6263
TypeType,
64+
TypeVarLikeType,
6365
TypeVarTupleType,
6466
TypeVarType,
6567
TypeVisitor,
@@ -1069,6 +1071,12 @@ def visit_overloaded(self, left: Overloaded) -> bool:
10691071
for item in left.items:
10701072
if self._is_subtype(item, right):
10711073
return True
1074+
# If simple logic failed, check a (somewhat ad hoc but important)
1075+
# edge case: Overloaded(def (int) -> int, def (str) -> str) is
1076+
# a subtype of def (int | str) -> int | str.
1077+
combined = union_function_signatures(left.items)
1078+
if combined is not None and self._is_subtype(combined, right):
1079+
return True
10721080
return False
10731081
elif isinstance(right, Overloaded):
10741082
if left == self.right:
@@ -2203,6 +2211,92 @@ def report(*args: Any) -> None:
22032211
return cast(NormalizedCallableType, applied)
22042212

22052213

2214+
def union_function_signatures(callables: list[CallableType]) -> CallableType | None:
2215+
"""Combine a list of functions by taking the union of all the arguments and return types."""
2216+
if len(callables) == 1:
2217+
return callables[0]
2218+
2219+
# Note: we are assuming here that if a user uses some TypeVar 'T' in
2220+
# two different functions, they meant for that TypeVar to mean the
2221+
# same thing.
2222+
#
2223+
# This function will make sure that all instances of that TypeVar 'T'
2224+
# refer to the same underlying TypeVarType objects to simplify the union-ing
2225+
# logic below.
2226+
#
2227+
# (If the user did *not* mean for 'T' to be consistently bound to the
2228+
# same type in their overloads, well, their code is probably too
2229+
# confusing and ought to be re-written anyway.)
2230+
callables, variables = merge_typevars_in_callables_by_name(callables)
2231+
2232+
new_args: list[list[Type]] = [[] for _ in callables[0].arg_types]
2233+
new_kinds = list(callables[0].arg_kinds)
2234+
new_returns: list[Type] = []
2235+
2236+
for target in callables:
2237+
# TODO: Enhance the merging logic to handle a wider variety of signatures.
2238+
if len(new_kinds) != len(target.arg_kinds):
2239+
return None
2240+
for i, (new_kind, target_kind) in enumerate(zip(new_kinds, target.arg_kinds)):
2241+
if new_kind == target_kind:
2242+
continue
2243+
if new_kind.is_positional() and target_kind.is_positional():
2244+
new_kinds[i] = ARG_POS
2245+
else:
2246+
return None
2247+
2248+
for i, arg in enumerate(target.arg_types):
2249+
new_args[i].append(arg)
2250+
new_returns.append(target.ret_type)
2251+
2252+
return callables[0].copy_modified(
2253+
arg_types=[mypy.typeops.make_simplified_union(args) for args in new_args],
2254+
arg_kinds=new_kinds,
2255+
ret_type=mypy.typeops.make_simplified_union(new_returns),
2256+
variables=variables,
2257+
implicit=True,
2258+
)
2259+
2260+
2261+
def merge_typevars_in_callables_by_name(
2262+
callables: Sequence[CallableType],
2263+
) -> tuple[list[CallableType], list[TypeVarLikeType]]:
2264+
"""Takes all the typevars present in the callables and 'combines' the ones with the same name.
2265+
2266+
For example, suppose we have two callables with signatures "f(x: T, y: S) -> T" and
2267+
"f(x: List[Tuple[T, S]]) -> Tuple[T, S]". Both callables use typevars named "T" and
2268+
"S", but we treat them as distinct, unrelated typevars. (E.g. they could both have
2269+
distinct ids.)
2270+
2271+
If we pass in both callables into this function, it returns a list containing two
2272+
new callables that are identical in signature, but use the same underlying TypeVarType
2273+
for T and S.
2274+
2275+
This is useful if we want to take the output lists and "merge" them into one callable
2276+
in some way -- for example, when unioning together overloads.
2277+
2278+
Returns both the new list of callables and a list of all distinct TypeVarType objects used.
2279+
"""
2280+
output: list[CallableType] = []
2281+
unique_typevars: dict[str, TypeVarLikeType] = {}
2282+
variables: list[TypeVarLikeType] = []
2283+
2284+
for target in callables:
2285+
if target.is_generic():
2286+
rename = {} # Mapping TypeVarId -> TypeVar
2287+
for tv in target.variables:
2288+
name = tv.fullname
2289+
if name not in unique_typevars:
2290+
unique_typevars[name] = tv
2291+
variables.append(tv)
2292+
rename[tv.id] = unique_typevars[name]
2293+
2294+
target = expand_type(target, rename)
2295+
output.append(target)
2296+
2297+
return output, variables
2298+
2299+
22062300
def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None:
22072301
"""Return the items of t, excluding any occurrence of s, if and only if
22082302
- t only contains simple literals

‎test-data/unit/check-overloading.test‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6856,3 +6856,48 @@ if isinstance(headers, dict):
68566856

68576857
reveal_type(headers) # N: Revealed type is "__main__.Headers | typing.Iterable[tuple[builtins.bytes, builtins.bytes]]"
68586858
[builtins fixtures/isinstancelist.pyi]
6859+
6860+
[case testOverloadUnionCallableSubtype]
6861+
from typing import Callable, Union, overload
6862+
6863+
def f(fn: Callable[[Union[int, str]], Union[int, str]]) -> None: pass
6864+
def f1(fn: Callable[[Union[int, str, bytes]], Union[int, str]]) -> None: pass
6865+
def f2(fn: Callable[[Union[int, str]], int]) -> None: pass
6866+
6867+
@overload
6868+
def g(x: int) -> int: ...
6869+
@overload
6870+
def g(x: str) -> str: ...
6871+
def g(x): ...
6872+
6873+
f(g)
6874+
f1(g) # E: Argument 1 to "f1" has incompatible type overloaded function; expected "Callable[[int | str | bytes], int | str]"
6875+
f2(g) # E: Argument 1 to "f2" has incompatible type overloaded function; expected "Callable[[int | str], int]"
6876+
6877+
[case testOverloadOverridesUnionCallable]
6878+
from typing import overload, Union
6879+
6880+
class A:
6881+
def f(self, x: Union[int, str]) -> None: ...
6882+
6883+
class B(A):
6884+
@overload
6885+
def f(self, x: int) -> None: ...
6886+
@overload
6887+
def f(self, x: str) -> None: ...
6888+
def f(self, x: Union[int, str]) -> None: ...
6889+
6890+
[case testOverloadOverridesUnionCallableGeneric]
6891+
from typing import overload, Optional, TypeVar
6892+
6893+
T = TypeVar("T")
6894+
6895+
class A:
6896+
def f(self, x: Optional[T]) -> Optional[T]: ...
6897+
6898+
class B(A):
6899+
@overload
6900+
def f(self, x: None) -> None: ...
6901+
@overload
6902+
def f(self, x: T) -> T: ...
6903+
def f(self, x: Optional[T]) -> Optional[T]: ...

0 commit comments

Comments
 (0)