Skip to content

Commit 879da56

Browse files
authored
Make joins with Any more precise (python#22336)
Fixes python#9186 Fixes python#3194 Currently, we have `join(Any, X) = Any`, but this causes various false negatives. Some of these false negatives will be exacerbated by python#22286. This PR makes `join(Any, X) = X | Any` (with obvious exceptions for `object`, `Never`, etc). Note I also simplify/unify logic for callables by deleting `combine_similar_callables()`.
1 parent d027c7c commit 879da56

16 files changed

Lines changed: 134 additions & 73 deletions

‎mypy/argmap.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ def expand_actual_type(
233233
item = fallback.args[0]
234234
return item
235235
elif isinstance(actual_type, ParamSpecType):
236-
# ParamSpec is valid in *args but it can't be unpacked.
236+
# ParamSpec is valid in *args, but it can't be unpacked.
237237
return actual_type
238238
else:
239239
return AnyType(TypeOfAny.from_error)
@@ -260,7 +260,7 @@ def expand_actual_type(
260260
1
261261
]
262262
elif isinstance(actual_type, ParamSpecType):
263-
# ParamSpec is valid in **kwargs but it can't be unpacked.
263+
# ParamSpec is valid in **kwargs, but it can't be unpacked.
264264
return actual_type
265265
else:
266266
return AnyType(TypeOfAny.from_error)

‎mypy/checkexpr.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5591,6 +5591,19 @@ def check_typeddict_literal_in_context(
55915591
return typeddict_context.copy_modified()
55925592

55935593
def visit_dict_expr(self, e: DictExpr) -> Type:
5594+
old_allow_any_join = type_state.allow_any_join
5595+
try:
5596+
ctx = self.type_context[-1]
5597+
# This is somewhat ad hoc, but we want to cover situations where a dict
5598+
# appears inside another dict or list literal as well.
5599+
type_state.allow_any_join = (
5600+
ctx is None or isinstance(ctx, TypeVarType) and ctx.id.is_meta_var()
5601+
)
5602+
return self._visit_dict_expr(e)
5603+
finally:
5604+
type_state.allow_any_join = old_allow_any_join
5605+
5606+
def _visit_dict_expr(self, e: DictExpr) -> Type:
55945607
"""Type check a dict expression.
55955608
55965609
Translate it into a call to dict(), with provisions for **expr.

‎mypy/join.py‎

Lines changed: 19 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
get_proper_types,
6363
split_with_prefix_and_suffix,
6464
)
65+
from mypy.typestate import type_state
6566

6667

6768
class InstanceJoiner:
@@ -245,7 +246,9 @@ def join_types(s: Type, t: Type, instance_joiner: InstanceJoiner | None = None)
245246
s, t = t, s
246247

247248
if isinstance(s, AnyType):
248-
return s
249+
if isinstance(t, AnyType):
250+
return s
251+
s, t = t, s
249252

250253
if isinstance(s, ErasedType):
251254
return t
@@ -284,13 +287,22 @@ def visit_union_type(self, t: UnionType) -> ProperType:
284287
return mypy.typeops.make_simplified_union([self.s, t])
285288

286289
def visit_any(self, t: AnyType) -> ProperType:
287-
return t
290+
if type_state.allow_any_join:
291+
return t
292+
if isinstance(self.s, Instance) and self.s.type.fullname == "builtins.object":
293+
return self.s
294+
if isinstance(self.s, UnionType) and any(
295+
isinstance(get_proper_type(it), AnyType) for it in self.s.items
296+
):
297+
# Avoid multiple Anys in a union type.
298+
return self.s
299+
return UnionType([self.s, t])
288300

289301
def visit_none_type(self, t: NoneType) -> ProperType:
290302
if state.strict_optional:
291303
if isinstance(self.s, (NoneType, UninhabitedType)):
292304
return t
293-
elif isinstance(self.s, (UnboundType, AnyType)):
305+
elif isinstance(self.s, UnboundType):
294306
return AnyType(TypeOfAny.special_form)
295307
else:
296308
return mypy.typeops.make_simplified_union([self.s, t])
@@ -390,8 +402,6 @@ def visit_instance(self, t: Instance) -> ProperType:
390402
def visit_callable_type(self, t: CallableType) -> ProperType:
391403
if isinstance(self.s, CallableType):
392404
if is_similar_callables(t, self.s):
393-
if is_equivalent(t, self.s):
394-
return combine_similar_callables(t, self.s)
395405
result = join_similar_callables(t, self.s)
396406
if any(
397407
isinstance(tp, (NoneType, UninhabitedType))
@@ -462,7 +472,7 @@ def visit_overloaded(self, t: Overloaded) -> ProperType:
462472
for s_item in s.items:
463473
if is_similar_callables(t_item, s_item):
464474
if is_equivalent(t_item, s_item):
465-
result.append(combine_similar_callables(t_item, s_item))
475+
result.append(join_similar_callables(t_item, s_item))
466476
elif is_subtype(t_item, s_item):
467477
result.append(s_item)
468478
if result:
@@ -824,7 +834,8 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
824834
arg_types.append(s.arg_types[i])
825835
else:
826836
arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i]))
827-
# TODO in combine_similar_callables also applies here (names and kinds; user metaclasses)
837+
# TODO Handle more kinds and different argument names.
838+
# TODO what should happen if one fallback is 'type' and the other is a custom metaclass?
828839
# The fallback type can be either 'function', 'type', or some user-provided metaclass.
829840
# The result should always use 'function' as a fallback if either operands are using it.
830841
if t.fallback.type.fullname == "builtins.function":
@@ -849,7 +860,7 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
849860

850861

851862
def safe_join(t: Type, s: Type) -> Type:
852-
# This is a temporary solution to prevent crashes in combine_similar_callables() etc.,
863+
# This is a temporary solution to prevent crashes in join_similar_callables() etc.,
853864
# until relevant TODOs on handling arg_kinds will be addressed there.
854865
if not isinstance(t, UnpackType) and not isinstance(s, UnpackType):
855866
return join_types(t, s)
@@ -880,36 +891,6 @@ def safe_meet(t: Type, s: Type) -> Type:
880891
return UninhabitedType()
881892

882893

883-
def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType:
884-
t, s = match_generic_callables(t, s)
885-
arg_types: list[Type] = []
886-
for i in range(len(t.arg_types)):
887-
arg_types.append(safe_join(t.arg_types[i], s.arg_types[i]))
888-
# TODO kinds and argument names
889-
# TODO what should happen if one fallback is 'type' and the other is a user-provided metaclass?
890-
# The fallback type can be either 'function', 'type', or some user-provided metaclass.
891-
# The result should always use 'function' as a fallback if either operands are using it.
892-
if t.fallback.type.fullname == "builtins.function":
893-
fallback = t.fallback
894-
elif s.fallback.type.fullname == "builtins.function":
895-
fallback = s.fallback
896-
else:
897-
fallback = InstanceJoiner().join_instances(t.fallback, s.fallback)
898-
assert isinstance(fallback, Instance)
899-
instance_type = None
900-
if t.instance_type is not None and s.instance_type is not None:
901-
instance_type = join_types(t.instance_type, s.instance_type)
902-
return t.copy_modified(
903-
arg_types=arg_types,
904-
arg_kinds=combine_arg_kinds(t, s),
905-
arg_names=combine_arg_names(t, s),
906-
ret_type=join_types(t.ret_type, s.ret_type),
907-
instance_type=instance_type,
908-
fallback=fallback,
909-
name=None,
910-
)
911-
912-
913894
def combine_arg_names(
914895
t: CallableType | Parameters, s: CallableType | Parameters
915896
) -> list[str | None]:

‎mypy/meet.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -961,8 +961,6 @@ def visit_callable_type(self, t: CallableType) -> ProperType:
961961
if isinstance(self.s, CallableType) and join.is_similar_callables(
962962
t, self.s, for_meet=True
963963
):
964-
if is_equivalent(t, self.s):
965-
return join.combine_similar_callables(t, self.s)
966964
result = meet_similar_callables(t, self.s)
967965
# We set the from_type_type flag to suppress error when a collection of
968966
# concrete class objects gets inferred as their common abstract superclass.
@@ -1247,7 +1245,7 @@ def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType:
12471245
arg_types: list[Type] = []
12481246
for i in range(len(t.arg_types)):
12491247
arg_types.append(safe_join(t.arg_types[i], s.arg_types[i]))
1250-
# TODO in combine_similar_callables also applies here (names and kinds)
1248+
# TODOs in join_similar_callables() also applies here.
12511249
# The fallback type can be either 'function' or 'type'. The result should have 'function' as
12521250
# fallback only if both operands have it as 'function'.
12531251
if t.fallback.type.fullname != "builtins.function":

‎mypy/test/testsolve.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from mypy.solve import Bounds, Graph, solve_constraints, transitive_closure
77
from mypy.test.helpers import Suite, assert_equal
88
from mypy.test.typefixture import TypeFixture
9-
from mypy.types import Type, TypeVarId, TypeVarLikeType, TypeVarType
9+
from mypy.types import Type, TypeVarId, TypeVarLikeType, TypeVarType, UnionType
1010

1111

1212
class SolveSuite(Suite):
@@ -84,7 +84,7 @@ def test_simple_constraints_with_dynamic_type(self) -> None:
8484
self.assert_solve(
8585
[self.fx.t],
8686
[self.supc(self.fx.t, self.fx.anyt), self.supc(self.fx.t, self.fx.a)],
87-
[self.fx.anyt],
87+
[UnionType([self.fx.a, self.fx.anyt])],
8888
)
8989

9090
self.assert_solve([self.fx.t], [self.subc(self.fx.t, self.fx.anyt)], [self.fx.anyt])

‎mypy/test/testtypes.py‎

Lines changed: 44 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -840,21 +840,45 @@ def test_unbound_type(self) -> None:
840840
self.assert_join(t, UnboundType("X"), self.fx.anyt)
841841

842842
def test_any_type(self) -> None:
843-
# Join against 'Any' type always results in 'Any'.
844843
with state.strict_optional_set(False):
845844
self.assert_join(NoneType(), self.fx.anyt, self.fx.anyt)
846845

847846
for t in [
848-
self.fx.anyt,
849847
self.fx.a,
850-
self.fx.o,
851-
NoneType(),
852848
UnboundType("x"),
853849
self.fx.t,
854850
self.tuple(),
855851
self.callable(self.fx.a, self.fx.b),
856852
]:
857-
self.assert_join(t, self.fx.anyt, self.fx.anyt)
853+
self.assert_join(t, self.fx.anyt, UnionType([t, self.fx.anyt]))
854+
self.assert_join(self.fx.anyt, t, UnionType([t, self.fx.anyt]))
855+
856+
self.assert_join(self.fx.anyt, self.fx.anyt, self.fx.anyt)
857+
858+
self.assert_join(self.fx.o, self.fx.anyt, self.fx.o)
859+
self.assert_join(NoneType(), self.fx.anyt, UnionType([self.fx.anyt, NoneType()]))
860+
self.assert_join(UninhabitedType(), self.fx.anyt, self.fx.anyt)
861+
862+
self.assert_join(self.fx.anyt, self.fx.o, self.fx.o)
863+
self.assert_join(self.fx.anyt, NoneType(), UnionType([self.fx.anyt, NoneType()]))
864+
self.assert_join(self.fx.anyt, UninhabitedType(), self.fx.anyt)
865+
866+
def test_any_type_with_union(self) -> None:
867+
self.assert_join(
868+
UnionType([self.fx.a, self.fx.anyt]),
869+
self.fx.anyt,
870+
UnionType([self.fx.a, self.fx.anyt]),
871+
)
872+
self.assert_join(
873+
self.fx.anyt,
874+
UnionType([self.fx.a, self.fx.anyt]),
875+
UnionType([self.fx.a, self.fx.anyt]),
876+
)
877+
self.assert_join(
878+
UnionType([self.fx.a, self.fx.anyt]),
879+
UnionType([self.fx.a, self.fx.anyt]),
880+
UnionType([self.fx.a, self.fx.anyt]),
881+
)
858882

859883
def test_mixed_truth_restricted_type_simple(self) -> None:
860884
# make_simplified_union against differently restricted truthiness types drops restrictions.
@@ -885,7 +909,7 @@ def test_simple_generics(self) -> None:
885909
with state.strict_optional_set(True):
886910
self.assert_join(self.fx.ga, self.fx.nonet, UnionType([self.fx.ga, NoneType()]))
887911

888-
self.assert_join(self.fx.ga, self.fx.anyt, self.fx.anyt)
912+
self.assert_join(self.fx.ga, self.fx.anyt, UnionType([self.fx.ga, self.fx.anyt]))
889913

890914
for t in [
891915
self.fx.a,
@@ -938,7 +962,9 @@ def test_callables_with_any(self) -> None:
938962
self.assert_join(
939963
self.callable(self.fx.a, self.fx.a, self.fx.anyt, self.fx.a),
940964
self.callable(self.fx.a, self.fx.anyt, self.fx.a, self.fx.anyt),
941-
self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, self.fx.anyt),
965+
self.callable(
966+
self.fx.a, self.fx.anyt, self.fx.anyt, UnionType([self.fx.a, self.fx.anyt])
967+
),
942968
)
943969

944970
def test_overloaded(self) -> None:
@@ -1016,12 +1042,14 @@ def test_simple_type_objects(self) -> None:
10161042

10171043
def test_type_type(self) -> None:
10181044
self.assert_join(self.fx.type_a, self.fx.type_b, self.fx.type_a)
1019-
self.assert_join(self.fx.type_b, self.fx.type_any, self.fx.type_any)
1045+
self.assert_join(
1046+
self.fx.type_b, self.fx.type_any, UnionType([self.fx.type_b, self.fx.type_any])
1047+
)
10201048
self.assert_join(self.fx.type_b, self.fx.type_type, self.fx.type_type)
10211049
self.assert_join(self.fx.type_b, self.fx.type_c, self.fx.type_a)
10221050
self.assert_join(self.fx.type_c, self.fx.type_d, TypeType.make_normalized(self.fx.o))
10231051
self.assert_join(self.fx.type_type, self.fx.type_any, self.fx.type_type)
1024-
self.assert_join(self.fx.type_b, self.fx.anyt, self.fx.anyt)
1052+
self.assert_join(self.fx.type_b, self.fx.anyt, UnionType([self.fx.type_b, self.fx.anyt]))
10251053

10261054
def test_literal_type(self) -> None:
10271055
a = self.fx.a
@@ -1035,7 +1063,7 @@ def test_literal_type(self) -> None:
10351063
self.assert_join(lit1, d, self.fx.o)
10361064
self.assert_join(lit1, lit2, a)
10371065
self.assert_join(lit1, lit3, self.fx.o)
1038-
self.assert_join(lit1, self.fx.anyt, self.fx.anyt)
1066+
self.assert_join(lit1, self.fx.anyt, UnionType([lit1, self.fx.anyt]))
10391067
self.assert_join(UnionType([lit1, lit2]), lit2, UnionType([lit1, lit2]))
10401068
self.assert_join(UnionType([lit1, lit2]), a, a)
10411069
self.assert_join(UnionType([lit1, lit3]), a, UnionType([a, lit3]))
@@ -1324,7 +1352,12 @@ def test_callables_with_dynamic(self) -> None:
13241352
self.assert_meet(
13251353
self.callable(self.fx.a, self.fx.a, self.fx.anyt, self.fx.a),
13261354
self.callable(self.fx.a, self.fx.anyt, self.fx.a, self.fx.anyt),
1327-
self.callable(self.fx.a, self.fx.anyt, self.fx.anyt, self.fx.anyt),
1355+
self.callable(
1356+
self.fx.a,
1357+
UnionType([self.fx.a, self.fx.anyt]),
1358+
UnionType([self.fx.a, self.fx.anyt]),
1359+
self.fx.a,
1360+
),
13281361
)
13291362

13301363
def test_meet_interface_types(self) -> None:

‎mypy/typestate.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,15 @@ class TypeState:
113113
# Whether to use new type inference algorithm that can infer polymorphic types.
114114
# This is temporary and will be removed soon when new algorithm is more polished.
115115
infer_polymorphic: bool
116-
# No not simplify/normalize unions during type inference in this flag is set.
116+
# Do not simplify/normalize unions during type inference in this flag is set.
117117
# Technically, union simplification gives strictly better type inference, but it
118118
# may have bad performance (since it is quadratic). So we may want to disable
119119
# it situations where performance is important.
120120
keep_unions: bool
121+
# Infer join(X, Any) as Any (instead of X | Any) when this flag is set.
122+
# This is used to preserve backwards compatibility for situations like
123+
# testDictWithUnionAnySpecialCaseInStar2. This is needed until #5382 is fixed.
124+
allow_any_join: bool
121125

122126
# N.B: We do all the accesses to these properties through
123127
# TypeState, instead of making these classmethods and accessing
@@ -137,6 +141,7 @@ def __init__(self) -> None:
137141
self.infer_unions = False
138142
self.infer_polymorphic = False
139143
self.keep_unions = False
144+
self.allow_any_join = False
140145
self.bind_self_cache = {}
141146

142147
def is_assumed_subtype(self, left: Type, right: Type) -> bool:

‎mypyc/test-data/irbuild-lists.test‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -986,7 +986,8 @@ def test():
986986
r19 :: list
987987
r20, r21 :: native_int
988988
r22 :: bit
989-
r23, x :: object
989+
r23 :: object
990+
x :: union[str, tuple[str, str], object]
990991
r24 :: str
991992
r25 :: native_int
992993
a :: list

‎mypyc/test-data/irbuild-tuple.test‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1042,7 +1042,8 @@ def test():
10421042
r19 :: tuple
10431043
r20, r21 :: native_int
10441044
r22 :: bit
1045-
r23, x :: object
1045+
r23 :: object
1046+
x :: union[str, tuple[str, str], object]
10461047
r24 :: str
10471048
r25 :: native_int
10481049
a :: tuple

‎test-data/unit/check-classes.test‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9844,7 +9844,9 @@ fs = [f, lambda x: 1, f]
98449844
reveal_type(fs) # N: Revealed type is "builtins.list[def (x: Any) -> builtins.int]"
98459845

98469846
[case testLambdaInferredInCollection2]
9847-
def f(x: int, y: int)-> int: ...
9847+
def f(x: int, y: int) -> int: ...
98489848

98499849
fs = [lambda df, x: df.foo, f]
9850-
reveal_type(fs) # N: Revealed type is "builtins.list[def (Any, Any) -> Any]"
9850+
reveal_type(fs) # N: Revealed type is "builtins.list[def (Any, Any) -> builtins.int | Any]"
9851+
fs2 = [f, lambda df, x: df.foo]
9852+
reveal_type(fs2) # N: Revealed type is "builtins.list[def (Any, Any) -> builtins.int | Any]"

0 commit comments

Comments
 (0)