Skip to content

Commit 595760f

Browse files
committed
Remove RawLiteralType synthetic type
This diff changes how we track raw literal types in the semantic analysis phase. It makes the following changes: 1. Removes the `RawLiteralType` synthetic type. 2. Adds a new `TypeOfAny`: `TypeOfAny.invalid_type` as suggested in python#4030. 3. Modifies `AnyType` so it can optionally contain a new `RawLiteral` class. This class contains information about the underlying literal that produced that particular `TypeOfAny`. 4. Adjusts mypy to stop recommending using `Literal[...]` when doing `A = NewType('A', 4)` or `T = TypeVar('T', bound=4)`. (The former suggestion is a bad one: you can't create a NewType of a Literal[...] type. The latter suggestion is a valid but stupid one: `T = TypeVar('T', bound=Literal[4])` is basically the same thing as `T = Literal[4]`.) This resolves python#5989. The net effect of this diff is that: 1. RawLiteralTypes no longer leak during fine-grained mode, which should partially help unblock python#6075. 2. The way mypy handles literal expressions in types is "inverted". Previously, we by default assumed literal expressions would belong inside `Literal[...]` and tacked on some logic to make them convert into error `AnyTypes`. Now, we do the reverse: we start with an error `AnyType` and convert those into `Literal[...]`s as needed. This more closely mirrors the way mypy *used* to work before we started work on Literal types. It should also hopefully help reduce some of the cognitive burden of working on other parts of the semantic analysis code, since we no longer need to worry about the `RawLiteralType` synthetic type. 3. We now have more flexibility in how we choose to handle invalid types: since they're just `Anys`, we have more opportunities to intercept and customize the exact way in which we handle errors. Also see python#4030 for additional context. (This diff lays out some of the foundation work for that diff).
1 parent fcee66d commit 595760f

14 files changed

Lines changed: 266 additions & 161 deletions

‎mypy/exprtotype.py‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from mypy.fastparse import parse_type_string
99
from mypy.types import (
1010
Type, UnboundType, TypeList, EllipsisType, AnyType, Optional, CallableArgument, TypeOfAny,
11-
RawLiteralType,
11+
RawLiteral,
1212
)
1313

1414

@@ -39,9 +39,19 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No
3939
if isinstance(expr, NameExpr):
4040
name = expr.name
4141
if name == 'True':
42-
return RawLiteralType(True, 'builtins.bool', line=expr.line, column=expr.column)
42+
return AnyType(
43+
TypeOfAny.invalid_type,
44+
raw_literal=RawLiteral(True, 'builtins.bool'),
45+
line=expr.line,
46+
column=expr.column,
47+
)
4348
elif name == 'False':
44-
return RawLiteralType(False, 'builtins.bool', line=expr.line, column=expr.column)
49+
return AnyType(
50+
TypeOfAny.invalid_type,
51+
raw_literal=RawLiteral(False, 'builtins.bool'),
52+
line=expr.line,
53+
column=expr.column,
54+
)
4555
else:
4656
return UnboundType(name, line=expr.line, column=expr.column)
4757
elif isinstance(expr, MemberExpr):
@@ -122,17 +132,27 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No
122132
assume_str_is_unicode=True)
123133
elif isinstance(expr, UnaryExpr):
124134
typ = expr_to_unanalyzed_type(expr.expr)
125-
if isinstance(typ, RawLiteralType) and isinstance(typ.value, int) and expr.op == '-':
126-
typ.value *= -1
127-
return typ
128-
else:
129-
raise TypeTranslationError()
135+
if isinstance(typ, AnyType) and typ.raw_literal is not None:
136+
if isinstance(typ.raw_literal.value, int) and expr.op == '-':
137+
typ.raw_literal.value *= -1
138+
return typ
139+
raise TypeTranslationError()
130140
elif isinstance(expr, IntExpr):
131-
return RawLiteralType(expr.value, 'builtins.int', line=expr.line, column=expr.column)
141+
return AnyType(
142+
TypeOfAny.invalid_type,
143+
raw_literal=RawLiteral(expr.value, 'builtins.int'),
144+
line=expr.line,
145+
column=expr.column,
146+
)
132147
elif isinstance(expr, FloatExpr):
133148
# Floats are not valid parameters for RawLiteralType, so we just
134149
# pass in 'None' for now. We'll report the appropriate error at a later stage.
135-
return RawLiteralType(None, 'builtins.float', line=expr.line, column=expr.column)
150+
return AnyType(
151+
TypeOfAny.invalid_type,
152+
raw_literal=RawLiteral(None, 'builtins.float'),
153+
line=expr.line,
154+
column=expr.column,
155+
)
136156
elif isinstance(expr, EllipsisExpr):
137157
return EllipsisType(expr.line)
138158
else:

‎mypy/fastparse.py‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
)
3232
from mypy.types import (
3333
Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, CallableArgument,
34-
TypeOfAny, Instance, RawLiteralType,
34+
TypeOfAny, Instance, RawLiteral,
3535
)
3636
from mypy import defaults
3737
from mypy import messages
@@ -184,11 +184,21 @@ def parse_type_string(expr_string: str, expr_fallback_name: str,
184184
node.original_str_fallback = expr_fallback_name
185185
return node
186186
else:
187-
return RawLiteralType(expr_string, expr_fallback_name, line, column)
187+
return AnyType(
188+
TypeOfAny.invalid_type,
189+
raw_literal=RawLiteral(expr_string, expr_fallback_name),
190+
line=line,
191+
column=column,
192+
)
188193
except (SyntaxError, ValueError):
189194
# Note: the parser will raise a `ValueError` instead of a SyntaxError if
190195
# the string happens to contain things like \x00.
191-
return RawLiteralType(expr_string, expr_fallback_name, line, column)
196+
return AnyType(
197+
TypeOfAny.invalid_type,
198+
raw_literal=RawLiteral(expr_string, expr_fallback_name),
199+
line=line,
200+
column=column,
201+
)
192202

193203

194204
def is_no_type_check_decorator(expr: ast3.expr) -> bool:
@@ -1183,7 +1193,11 @@ def visit_Name(self, n: Name) -> Type:
11831193

11841194
def visit_NameConstant(self, n: NameConstant) -> Type:
11851195
if isinstance(n.value, bool):
1186-
return RawLiteralType(n.value, 'builtins.bool', line=self.line)
1196+
return AnyType(
1197+
TypeOfAny.invalid_type,
1198+
raw_literal=RawLiteral(n.value, 'builtins.bool'),
1199+
line=self.line,
1200+
)
11871201
else:
11881202
return UnboundType(str(n.value), line=self.line)
11891203

@@ -1192,9 +1206,9 @@ def visit_UnaryOp(self, n: UnaryOp) -> Type:
11921206
# We support specifically Literal[-4] and nothing else.
11931207
# For example, Literal[+4] or Literal[~6] is not supported.
11941208
typ = self.visit(n.operand)
1195-
if isinstance(typ, RawLiteralType) and isinstance(n.op, USub):
1196-
if isinstance(typ.value, int):
1197-
typ.value *= -1
1209+
if isinstance(typ, AnyType) and typ.raw_literal is not None:
1210+
if isinstance(typ.raw_literal.value, int) and isinstance(n.op, USub):
1211+
typ.raw_literal.value *= -1
11981212
return typ
11991213
self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1))
12001214
return AnyType(TypeOfAny.from_error)
@@ -1204,11 +1218,19 @@ def visit_Num(self, n: Num) -> Type:
12041218
# Could be either float or int
12051219
numeric_value = n.n
12061220
if isinstance(numeric_value, int):
1207-
return RawLiteralType(numeric_value, 'builtins.int', line=self.line)
1221+
return AnyType(
1222+
TypeOfAny.invalid_type,
1223+
raw_literal=RawLiteral(numeric_value, 'builtins.int'),
1224+
line=self.line,
1225+
)
12081226
elif isinstance(numeric_value, float):
12091227
# Floats and other numbers are not valid parameters for RawLiteralType, so we just
12101228
# pass in 'None' for now. We'll report the appropriate error at a later stage.
1211-
return RawLiteralType(None, 'builtins.float', line=self.line)
1229+
return AnyType(
1230+
TypeOfAny.invalid_type,
1231+
raw_literal=RawLiteral(None, 'builtins.float'),
1232+
line=self.line,
1233+
)
12121234
else:
12131235
self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1))
12141236
return AnyType(TypeOfAny.from_error)
@@ -1230,7 +1252,12 @@ def visit_Str(self, n: Str) -> Type:
12301252
# Bytes(bytes s)
12311253
def visit_Bytes(self, n: Bytes) -> Type:
12321254
contents = bytes_to_human_readable_repr(n.s)
1233-
return RawLiteralType(contents, 'builtins.bytes', self.line, column=n.col_offset)
1255+
return AnyType(
1256+
TypeOfAny.invalid_type,
1257+
raw_literal=RawLiteral(contents, 'builtins.bytes'),
1258+
line=self.line,
1259+
column=n.col_offset,
1260+
)
12341261

12351262
# Subscript(expr value, slice slice, expr_context ctx)
12361263
def visit_Subscript(self, n: ast3.Subscript) -> Type:

‎mypy/indirection.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,6 @@ def visit_tuple_type(self, t: types.TupleType) -> Set[str]:
9090
def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]:
9191
return self._visit(t.items.values()) | self._visit(t.fallback)
9292

93-
def visit_raw_literal_type(self, t: types.RawLiteralType) -> Set[str]:
94-
assert False, "Unexpected RawLiteralType after semantic analysis phase"
95-
9693
def visit_literal_type(self, t: types.LiteralType) -> Set[str]:
9794
return self._visit(t.fallback)
9895

‎mypy/plugin.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ def anal_type(self, t: Type, *,
180180
tvar_scope: Optional[TypeVarScope] = None,
181181
allow_tuple_literal: bool = False,
182182
allow_unbound_tvars: bool = False,
183+
report_invalid_types: bool = True,
183184
third_pass: bool = False) -> Type:
184185
"""Analyze an unbound type."""
185186
raise NotImplementedError

‎mypy/semanal.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1283,7 +1283,7 @@ def update_metaclass(self, defn: ClassDef) -> None:
12831283
return
12841284
defn.metaclass = metas.pop()
12851285

1286-
def expr_to_analyzed_type(self, expr: Expression) -> Type:
1286+
def expr_to_analyzed_type(self, expr: Expression, report_invalid_types: bool = True) -> Type:
12871287
if isinstance(expr, CallExpr):
12881288
expr.accept(self)
12891289
info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope())
@@ -1295,7 +1295,7 @@ def expr_to_analyzed_type(self, expr: Expression) -> Type:
12951295
fallback = Instance(info, [])
12961296
return TupleType(info.tuple_type.items, fallback=fallback)
12971297
typ = expr_to_unanalyzed_type(expr)
1298-
return self.anal_type(typ)
1298+
return self.anal_type(typ, report_invalid_types=report_invalid_types)
12991299

13001300
def verify_base_classes(self, defn: ClassDef) -> bool:
13011301
info = defn.info
@@ -1686,6 +1686,7 @@ def type_analyzer(self, *,
16861686
tvar_scope: Optional[TypeVarScope] = None,
16871687
allow_tuple_literal: bool = False,
16881688
allow_unbound_tvars: bool = False,
1689+
report_invalid_types: bool = True,
16891690
third_pass: bool = False) -> TypeAnalyser:
16901691
if tvar_scope is None:
16911692
tvar_scope = self.tvar_scope
@@ -1696,6 +1697,7 @@ def type_analyzer(self, *,
16961697
self.is_typeshed_stub_file,
16971698
allow_unbound_tvars=allow_unbound_tvars,
16981699
allow_tuple_literal=allow_tuple_literal,
1700+
report_invalid_types=report_invalid_types,
16991701
allow_unnormalized=self.is_stub_file,
17001702
third_pass=third_pass)
17011703
tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic())
@@ -1706,10 +1708,12 @@ def anal_type(self, t: Type, *,
17061708
tvar_scope: Optional[TypeVarScope] = None,
17071709
allow_tuple_literal: bool = False,
17081710
allow_unbound_tvars: bool = False,
1711+
report_invalid_types: bool = True,
17091712
third_pass: bool = False) -> Type:
17101713
a = self.type_analyzer(tvar_scope=tvar_scope,
17111714
allow_unbound_tvars=allow_unbound_tvars,
17121715
allow_tuple_literal=allow_tuple_literal,
1716+
report_invalid_types=report_invalid_types,
17131717
third_pass=third_pass)
17141718
typ = t.accept(a)
17151719
self.add_type_alias_deps(a.aliases_used)
@@ -2394,7 +2398,12 @@ def process_typevar_parameters(self, args: List[Expression],
23942398
self.fail("TypeVar cannot have both values and an upper bound", context)
23952399
return None
23962400
try:
2397-
upper_bound = self.expr_to_analyzed_type(param_value)
2401+
upper_bound = self.expr_to_analyzed_type(param_value,
2402+
report_invalid_types=False)
2403+
if isinstance(upper_bound, AnyType) and upper_bound.from_invalid_type:
2404+
self.fail("TypeVar 'bound' must be a type", param_value)
2405+
# Note: we do not return 'None' here: we want to continue using the
2406+
# AnyType as the upper bound.
23982407
except TypeTranslationError:
23992408
self.fail("TypeVar 'bound' must be a type", param_value)
24002409
return None

‎mypy/semanal_newtype.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,12 @@ def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Opt
114114
self.fail(msg, context)
115115
return None
116116

117-
old_type = self.api.anal_type(unanalyzed_type)
117+
old_type = self.api.anal_type(unanalyzed_type, report_invalid_types=False)
118118

119119
# The caller of this function assumes that if we return a Type, it's always
120120
# a valid one. So, we translate AnyTypes created from errors into None.
121-
if isinstance(old_type, AnyType) and old_type.type_of_any == TypeOfAny.from_error:
121+
bad_anys = (TypeOfAny.from_error, TypeOfAny.invalid_type)
122+
if isinstance(old_type, AnyType) and old_type.type_of_any in bad_anys:
122123
self.fail(msg, context)
123124
return None
124125

‎mypy/semanal_shared.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ def anal_type(self, t: Type, *,
9191
tvar_scope: Optional[TypeVarScope] = None,
9292
allow_tuple_literal: bool = False,
9393
allow_unbound_tvars: bool = False,
94+
report_invalid_types: bool = True,
9495
third_pass: bool = False) -> Type:
9596
raise NotImplementedError
9697

‎mypy/server/astdiff.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,18 @@ def visit_unbound_type(self, typ: UnboundType) -> SnapshotItem:
267267
snapshot_types(typ.args))
268268

269269
def visit_any(self, typ: AnyType) -> SnapshotItem:
270-
return snapshot_simple_type(typ)
270+
if typ.raw_literal:
271+
return (
272+
'Any',
273+
typ.type_of_any,
274+
typ.raw_literal.value,
275+
typ.raw_literal.base_type_name,
276+
)
277+
else:
278+
return (
279+
'Any',
280+
typ.type_of_any,
281+
)
271282

272283
def visit_none_type(self, typ: NoneTyp) -> SnapshotItem:
273284
return snapshot_simple_type(typ)

‎mypy/server/astmerge.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@
5959
Type, SyntheticTypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType,
6060
TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
6161
Overloaded, TypeVarDef, TypeList, CallableArgument, EllipsisType, StarType, LiteralType,
62-
RawLiteralType,
6362
)
6463
from mypy.util import get_prefix, replace_object_state
6564
from mypy.typestate import TypeState
@@ -392,9 +391,6 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None:
392391
value_type.accept(self)
393392
typ.fallback.accept(self)
394393

395-
def visit_raw_literal_type(self, t: RawLiteralType) -> None:
396-
assert False, "Unexpected RawLiteralType after semantic analysis phase"
397-
398394
def visit_literal_type(self, typ: LiteralType) -> None:
399395
typ.fallback.accept(self)
400396

‎mypy/type_visitor.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
from mypy.types import (
2222
Type, AnyType, CallableType, Overloaded, TupleType, TypedDictType, LiteralType,
23-
RawLiteralType, Instance, NoneTyp, TypeType,
23+
Instance, NoneTyp, TypeType,
2424
UnionType, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef,
2525
UnboundType, ErasedType, ForwardRef, StarType, EllipsisType, TypeList, CallableArgument,
2626
)
@@ -127,10 +127,6 @@ def visit_callable_argument(self, t: CallableArgument) -> T:
127127
def visit_ellipsis_type(self, t: EllipsisType) -> T:
128128
pass
129129

130-
@abstractmethod
131-
def visit_raw_literal_type(self, t: RawLiteralType) -> T:
132-
pass
133-
134130

135131
@trait
136132
class TypeTranslator(TypeVisitor[Type]):
@@ -282,9 +278,6 @@ def visit_tuple_type(self, t: TupleType) -> T:
282278
def visit_typeddict_type(self, t: TypedDictType) -> T:
283279
return self.query_types(t.items.values())
284280

285-
def visit_raw_literal_type(self, t: RawLiteralType) -> T:
286-
return self.strategy([])
287-
288281
def visit_literal_type(self, t: LiteralType) -> T:
289282
return self.strategy([])
290283

0 commit comments

Comments
 (0)