Skip to content

Commit 027b86d

Browse files
ilevkivskyiilevkivskyi
authored
New semantic analyzer: Fix (de)serialization of broken named tuples (#6421)
Fixes #6413 In my original PR I missed the fact that "broken" named tuples (those where variable name and named tuple name are different) are stored under both names (one to actually access it, and other for (de)serialization). I also extend the semantic analyzer API, and clean-up/clarify the symbol adding logic a bit.
1 parent 84f8c9e commit 027b86d

5 files changed

Lines changed: 117 additions & 9 deletions

File tree

‎mypy/build.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1794,8 +1794,13 @@ def patch_dependency_parents(self) -> None:
17941794
semantic analyzer will perform this patch for us when processing stale
17951795
SCCs.
17961796
"""
1797+
Analyzer = Union[SemanticAnalyzerPass2, NewSemanticAnalyzer] # noqa
1798+
if self.manager.options.new_semantic_analyzer:
1799+
analyzer = self.manager.new_semantic_analyzer # type: Analyzer
1800+
else:
1801+
analyzer = self.manager.semantic_analyzer
17971802
for dep in self.dependencies:
1798-
self.manager.semantic_analyzer.add_submodules_to_parent_modules(dep, True)
1803+
analyzer.add_submodules_to_parent_modules(dep, True)
17991804

18001805
def fix_suppressed_dependencies(self, graph: Graph) -> None:
18011806
"""Corrects whether dependencies are considered stale in silent mode.

‎mypy/newsemanal/semanal.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3955,6 +3955,28 @@ def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
39553955
module_hidden=module_hidden)
39563956
return self.add_symbol_table_node(name, symbol, context)
39573957

3958+
def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None:
3959+
"""Same as above, but skipping the local namespace.
3960+
3961+
This doesn't check for previous definition and is only used
3962+
for serialization of method-level classes.
3963+
3964+
Classes defined within methods can be exposed through an
3965+
attribute type, but method-level symbol tables aren't serialized.
3966+
This method can be used to add such classes to an enclosing,
3967+
serialized symbol table.
3968+
"""
3969+
# TODO: currently this is only used by named tuples. Use this method
3970+
# also by typed dicts and normal classes, see issue #6422.
3971+
if self.type is not None:
3972+
names = self.type.names
3973+
kind = MDEF
3974+
else:
3975+
names = self.globals
3976+
kind = GDEF
3977+
symbol = SymbolTableNode(kind, node)
3978+
names[name] = symbol
3979+
39583980
def current_symbol_table(self) -> SymbolTable:
39593981
if self.is_func_scope():
39603982
assert self.locals[-1] is not None

‎mypy/newsemanal/semanal_namedtuple.py‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,18 @@ def check_namedtuple(self,
179179
return True, info
180180
name = cast(Union[StrExpr, BytesExpr, UnicodeExpr], call.args[0]).value
181181
if name != var_name or is_func_scope:
182-
# Give it a unique name derived from the line number.
182+
# There are three special cases where need to give it a unique name derived
183+
# from the line number:
184+
# * There is a name mismatch with l.h.s., therefore we need to disambiguate
185+
# situations like:
186+
# A = NamedTuple('Same', [('x', int)])
187+
# B = NamedTuple('Same', [('y', str)])
188+
# * This is a base class expression, since it often matches the class name:
189+
# class NT(NamedTuple('NT', [...])):
190+
# ...
191+
# * This is a local (function or method level) named tuple, since
192+
# two methods of a class can define a named tuple with the same name,
193+
# and they will be stored in the same namespace (see below).
183194
name += '@' + str(call.line)
184195
if len(defaults) > 0:
185196
default_items = {
@@ -189,15 +200,31 @@ def check_namedtuple(self,
189200
else:
190201
default_items = {}
191202
info = self.build_namedtuple_typeinfo(name, items, types, default_items)
192-
# Store it as a global just in case it would remain anonymous.
193-
# (Or in the nearest class if there is one.)
194-
self.store_namedtuple_info(info, var_name or name, call, is_typed)
203+
# If var_name is not None (i.e. this is not a base class expression), we always
204+
# store the generated TypeInfo under var_name in the current scope, so that
205+
# other definitions can use it.
206+
if var_name:
207+
self.store_namedtuple_info(info, var_name, call, is_typed)
208+
# There are three cases where we need to store the generated TypeInfo
209+
# second time (for the purpose of serialization):
210+
# * If there is a name mismatch like One = NamedTuple('Other', [...])
211+
# we also store the info under name 'Other@lineno', this is needed
212+
# because classes are (de)serialized using their actual fullname, not
213+
# the name of l.h.s.
214+
# * If this is a method level named tuple. It can leak from the method
215+
# via assignment to self attribute and therefore needs to be serialized
216+
# (local namespaces are not serialized).
217+
# * If it is a base class expression. It was not stored above, since
218+
# there is no var_name (but it still needs to be serialized
219+
# since it is in MRO of some class).
220+
if name != var_name or is_func_scope:
221+
# NOTE: we skip local namespaces since they are not serialized.
222+
self.api.add_symbol_skip_local(name, info)
195223
return True, info
196224

197225
def store_namedtuple_info(self, info: TypeInfo, name: str,
198226
call: CallExpr, is_typed: bool) -> None:
199-
stnode = SymbolTableNode(GDEF, info)
200-
self.api.add_symbol_table_node(name, stnode)
227+
self.api.add_symbol(name, info, call)
201228
call.analyzed = NamedTupleExpr(info, is_typed=is_typed)
202229
call.analyzed.set_line(call.line, call.column)
203230

‎mypy/newsemanal/semanal_shared.py‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
from mypy_extensions import trait
66

77
from mypy.nodes import (
8-
Context, SymbolTableNode, MypyFile, ImportedName, FuncDef, Node, TypeInfo, Expression, GDEF
8+
Context, SymbolTableNode, MypyFile, ImportedName, FuncDef, Node, TypeInfo, Expression, GDEF,
9+
SymbolNode
910
)
1011
from mypy.util import correct_relative_import
1112
from mypy.types import Type, FunctionLike, Instance
@@ -122,7 +123,23 @@ def schedule_patch(self, priority: int, fn: Callable[[], None]) -> None:
122123

123124
@abstractmethod
124125
def add_symbol_table_node(self, name: str, stnode: SymbolTableNode) -> bool:
125-
"""Add node to global symbol table (or to nearest class if there is one)."""
126+
"""Add node to the current symbol table."""
127+
raise NotImplementedError
128+
129+
@abstractmethod
130+
def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
131+
module_public: bool = True, module_hidden: bool = False) -> bool:
132+
"""Add symbol to the current symbol table."""
133+
raise NotImplementedError
134+
135+
@abstractmethod
136+
def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None:
137+
"""Add symbol to the current symbol table, skipping locals.
138+
139+
This is used to store symbol nodes in a symbol table that
140+
is going to be serialized (local namespaces are not serialized).
141+
See implementation docstring for more details.
142+
"""
126143
raise NotImplementedError
127144

128145
@abstractmethod

‎test-data/unit/check-incremental.test‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4931,3 +4931,40 @@ import pack.mod
49314931
import a
49324932
[out]
49334933
[out2]
4934+
4935+
[case testNewAnalyzerIncrementalBrokenNamedTuple]
4936+
# flags: --new-semantic-analyzer
4937+
import a
4938+
[file a.py]
4939+
from b import NT
4940+
x: NT
4941+
[file a.py.2]
4942+
from b import NT
4943+
x: NT
4944+
reveal_type(x)
4945+
[file b.py]
4946+
from typing import NamedTuple
4947+
NT = NamedTuple('BadName', [('x', int)])
4948+
[out]
4949+
[out2]
4950+
tmp/a.py:3: error: Revealed type is 'Tuple[builtins.int, fallback=b.BadName@2]'
4951+
4952+
[case testNewAnalyzerIncrementalMethodNamedTuple]
4953+
# flags: --new-semantic-analyzer
4954+
import a
4955+
[file a.py]
4956+
from b import C
4957+
x: C
4958+
[file a.py.2]
4959+
from b import C
4960+
x: C
4961+
reveal_type(x.h)
4962+
[file b.py]
4963+
from typing import NamedTuple
4964+
class C:
4965+
def __init__(self) -> None:
4966+
self.h: Hidden
4967+
Hidden = NamedTuple('Hidden', [('x', int)])
4968+
[out]
4969+
[out2]
4970+
tmp/a.py:3: error: Revealed type is 'Tuple[builtins.int, fallback=b.C.Hidden@5]'

0 commit comments

Comments
 (0)