Skip to content

Commit a9928e3

Browse files
authored
[mypyc] Skip recursive setup for frame-private nested functions (python#22187)
The recursive setup could cause `AttributeError` at runtime when a function contains both a frame-private as well as a captured nested function. The setup would add the frame-private function object as an attribute to the env class and access it even though it was never initialized. Fix by omitting the recursive setup since it's not needed - a frame-private nested function can't call itself. Also fix possibly inconsistent storage for a redefined nested function that is captured in one branch and not captured in another by using `original_def` in capture/free variable checks.
1 parent a645d53 commit a9928e3

7 files changed

Lines changed: 146 additions & 19 deletions

File tree

‎mypyc/irbuild/env_class.py‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def g() -> int:
3131
from mypyc.irbuild.builder import IRBuilder, SymbolTarget
3232
from mypyc.irbuild.context import FuncInfo, GeneratorClass, ImplicitClass
3333
from mypyc.irbuild.targets import AssignmentTargetAttr
34+
from mypyc.irbuild.util import get_func_target_symbol
3435

3536

3637
def setup_env_class(builder: IRBuilder) -> ClassIR:
@@ -141,7 +142,7 @@ def load_env_registers(builder: IRBuilder, prefix: str = "") -> None:
141142
# If this is a FuncDef, then make sure to load the FuncDef into its own environment
142143
# class so that the function can be called recursively.
143144
if isinstance(fitem, FuncDef) and fn_info.add_nested_funcs_to_env:
144-
setup_func_for_recursive_call(builder, fitem, fn_info.callable_class, prefix=prefix)
145+
setup_func_for_recursive_call(builder, fitem, fn_info.callable_class)
145146

146147

147148
def load_outer_env(
@@ -298,8 +299,13 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None:
298299
)
299300

300301
if fn_info.fitem in builder.encapsulating_funcs:
302+
added: set[SymbolNode] = set()
301303
for nested_fn in builder.encapsulating_funcs[fn_info.fitem]:
302304
if isinstance(nested_fn, FuncDef):
305+
target_symbol = get_func_target_symbol(nested_fn)
306+
if target_symbol in added:
307+
continue
308+
added.add(target_symbol)
303309
# The return type is 'object' instead of an RInstance of the
304310
# callable class because differently defined functions with
305311
# the same name and signature across conditional blocks
@@ -308,18 +314,18 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None:
308314
nested_prefix = prefix
309315
if nested_fn.is_generator or nested_fn.is_coroutine:
310316
nested_prefix = GENERATOR_ATTRIBUTE_PREFIX
311-
keep_alive = is_free_variable(builder, nested_fn)
312-
if fn_info.is_generator and not builder.is_captured_by_nested_func(nested_fn):
317+
keep_alive = is_free_variable(builder, target_symbol)
318+
if fn_info.is_generator and not builder.is_captured_by_nested_func(target_symbol):
313319
builder.add_var_to_generator_frame(
314-
nested_fn,
320+
target_symbol,
315321
object_rprimitive,
316322
fn_info.generator_class.self_reg,
317323
reassign=False,
318324
keep_alive_on_completion=keep_alive,
319325
)
320326
else:
321327
builder.add_var_to_env_class(
322-
nested_fn,
328+
target_symbol,
323329
object_rprimitive,
324330
env_for_func,
325331
reassign=False,
@@ -328,20 +334,26 @@ def add_vars_to_env(builder: IRBuilder, prefix: str = "") -> None:
328334
)
329335

330336

331-
def setup_func_for_recursive_call(
332-
builder: IRBuilder, fdef: FuncDef, base: ImplicitClass, prefix: str = ""
333-
) -> None:
337+
def setup_func_for_recursive_call(builder: IRBuilder, fdef: FuncDef, base: ImplicitClass) -> None:
334338
"""Enable calling a nested function (with a callable class) recursively.
335339
336340
Adds the instance of the callable class representing the given
337341
FuncDef to a register in the environment so that the function can
338342
be called recursively. Note that this needs to be done only for
339343
nested functions.
340344
"""
341-
# First, set the attribute of the environment class so that GetAttr can be called on it.
342345
prev_env = builder.fn_infos[-2].env_class
343-
attr_name = prefix + fdef.name
344-
prev_env.attributes[attr_name] = builder.type_to_rtype(fdef.type)
346+
target_symbol = get_func_target_symbol(fdef)
347+
target = builder.symtables[-2].get(target_symbol)
348+
if (
349+
not isinstance(target_symbol, FuncDef)
350+
or not isinstance(target, AssignmentTargetAttr)
351+
or not isinstance(target.obj.type, RInstance)
352+
or target.obj.type.class_ir is not prev_env
353+
):
354+
return
355+
356+
attr_name = target.attr
345357
line = fdef.line
346358

347359
if isinstance(base, GeneratorClass):
@@ -355,7 +367,7 @@ def setup_func_for_recursive_call(
355367
# Obtain the instance of the callable class representing the FuncDef, and add it to the
356368
# current environment.
357369
val = builder.add(GetAttr(prev_env_reg, attr_name, line))
358-
target = builder.add_local_reg(fdef, object_rprimitive)
370+
target = builder.add_local_reg(target_symbol, object_rprimitive)
359371
builder.assign(target, val, line)
360372

361373

‎mypyc/irbuild/function.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
)
7979
from mypyc.irbuild.generator import gen_generator_func, gen_generator_func_body
8080
from mypyc.irbuild.targets import AssignmentTarget
81+
from mypyc.irbuild.util import get_func_target_symbol
8182
from mypyc.primitives.dict_ops import (
8283
dict_get_method_with_none,
8384
dict_new_op,
@@ -976,11 +977,10 @@ def get_func_target(builder: IRBuilder, fdef: FuncDef) -> AssignmentTarget:
976977
If the function was not already defined somewhere, then define it
977978
and add it to the current environment.
978979
"""
979-
if orig := fdef.original_def:
980-
if isinstance(orig, Decorator):
981-
orig = orig.func
980+
target_symbol = get_func_target_symbol(fdef)
981+
if target_symbol is not fdef:
982982
# Get the target associated with the previously defined FuncDef.
983-
return builder.lookup(orig)
983+
return builder.lookup(target_symbol)
984984

985985
if builder.fn_info.is_generator or builder.fn_info.add_nested_funcs_to_env:
986986
return builder.lookup(fdef)

‎mypyc/irbuild/generator.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,7 @@ class that implements the function (each function gets a separate class).
111111
and top_level
112112
and top_level.add_nested_funcs_to_env
113113
):
114-
setup_func_for_recursive_call(
115-
builder, fitem, builder.fn_info.generator_class, prefix=GENERATOR_ATTRIBUTE_PREFIX
116-
)
114+
setup_func_for_recursive_call(builder, fitem, builder.fn_info.generator_class)
117115
cleanup_on_error = BasicBlock()
118116
builder.builder.push_error_handler(cleanup_on_error)
119117
create_switch_for_generator_class(builder)

‎mypyc/irbuild/util.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,14 @@ def get_func_def(op: FuncDef | Decorator | OverloadedFuncDef) -> FuncDef:
355355
return op
356356

357357

358+
def get_func_target_symbol(fdef: FuncDef) -> FuncDef | Var:
359+
"""Return the symbol used to store a possibly redefined function."""
360+
original = fdef.original_def
361+
if isinstance(original, Decorator):
362+
return original.func
363+
return original or fdef
364+
365+
358366
def concrete_arg_kind(kind: ArgKind) -> ArgKind:
359367
"""Find the concrete version of an arg kind that is being passed."""
360368
if kind == ARG_OPT:

‎mypyc/test-data/run-functions.test‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1399,6 +1399,33 @@ print(f(1))
13991399
[out]
14001400
2
14011401

1402+
[case testReassignedNestedFunction]
1403+
from typing import Callable, Generator
1404+
1405+
def outer(flag: bool) -> list[int]:
1406+
out: list[int] = []
1407+
f: Callable[[int], None] = lambda n: out.append(-1)
1408+
1409+
def swap() -> None:
1410+
nonlocal f
1411+
f = lambda n: out.append(100 + n)
1412+
1413+
if flag:
1414+
def f(n: int) -> None:
1415+
def k(m: int) -> int:
1416+
return k(m - 1) if m else 0
1417+
1418+
out.append(n + k(2))
1419+
if n == 0:
1420+
swap()
1421+
f(1) # should call the lambda installed by swap()
1422+
f(0)
1423+
return out
1424+
1425+
def test_reassigned_nested_function() -> None:
1426+
assert outer(False) == [-1]
1427+
assert outer(True) == [0, 101]
1428+
14021429
[case testStarArgFastPaths]
14031430
from typing import Any, Mapping
14041431
def fn(x: str, y: int) -> str:

‎mypyc/test-data/run-generators.test‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -907,6 +907,38 @@ def test_nested_generator_private_spill() -> None:
907907
assert yields == ("right",)
908908
assert value == "capturedleftsent"
909909

910+
[case testNestedGeneratorPrivateHelper]
911+
from typing import Iterator
912+
913+
def outer() -> Iterator[int]:
914+
def captured() -> Iterator[int]:
915+
yield 1
916+
917+
def private() -> Iterator[int]:
918+
yield from captured()
919+
920+
yield from private()
921+
922+
def test_nested_generator_private_helper() -> None:
923+
assert list(outer()) == [1]
924+
925+
[case testRedefinedNestedGeneratorHelper]
926+
from typing import Iterator
927+
928+
def redefined_nested_helper(flag: bool) -> Iterator[int]:
929+
if flag:
930+
def helper(n: int) -> int:
931+
return 1
932+
else:
933+
def helper(n: int) -> int:
934+
return helper(1) if not n else 2
935+
936+
yield helper(0)
937+
938+
def test_redefined_nested_generator_helper() -> None:
939+
assert list(redefined_nested_helper(True)) == [1]
940+
assert list(redefined_nested_helper(False)) == [2]
941+
910942
[case testGeneratorPrivateAndEscapedBindings]
911943
from typing import Any, Callable, Generator, Iterator
912944

‎mypyc/test/test_generator_spills.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,56 @@ def get() -> str:
184184
)
185185
assert environment is not frame
186186

187+
def test_nested_generator_private_helper_stays_on_frame(self) -> None:
188+
module, _, _, _ = build_ir_for_single_file2("""\
189+
from typing import Iterator
190+
191+
def outer() -> Iterator[int]:
192+
def captured() -> Iterator[int]:
193+
yield 1
194+
def private() -> Iterator[int]:
195+
yield from captured()
196+
yield from private()
197+
""".splitlines())
198+
outer_frame = next(cl for cl in module.classes if cl.name == "outer_gen")
199+
environment_type = outer_frame.attributes[ENV_ATTR_NAME]
200+
assert isinstance(environment_type, RInstance)
201+
environment = environment_type.class_ir
202+
203+
assert GENERATOR_ATTRIBUTE_PREFIX + "captured" in environment.attributes
204+
assert "private" in frame_variables(outer_frame)
205+
assert GENERATOR_ATTRIBUTE_PREFIX + "private" not in environment.attributes
206+
207+
private_frame = next(cl for cl in module.classes if cl.name.startswith("private_gen___"))
208+
private_helper = private_frame.env_user_function
209+
assert private_helper is not None
210+
assert not any(
211+
isinstance(op, GetAttr) and op.attr == GENERATOR_ATTRIBUTE_PREFIX + "private"
212+
for block in private_helper.blocks
213+
for op in block.ops
214+
)
215+
216+
def test_redefined_nested_helper_has_single_storage_location(self) -> None:
217+
module, _, _, _ = build_ir_for_single_file2("""\
218+
from typing import Iterator
219+
220+
def outer(flag: bool) -> Iterator[int]:
221+
if flag:
222+
def helper(n: int) -> int:
223+
return 1
224+
else:
225+
def helper(n: int) -> int:
226+
return helper(1) if not n else 2
227+
yield helper(0)
228+
""".splitlines())
229+
outer_frame = next(cl for cl in module.classes if cl.name == "outer_gen")
230+
environment_type = outer_frame.attributes[ENV_ATTR_NAME]
231+
assert isinstance(environment_type, RInstance)
232+
environment = environment_type.class_ir
233+
234+
assert GENERATOR_ATTRIBUTE_PREFIX + "helper" in environment.attributes
235+
assert "helper" not in frame_variables(outer_frame)
236+
187237
def test_nested_finally_promotes_each_saved_exception(self) -> None:
188238
cl = generator_class(
189239
"""\

0 commit comments

Comments
 (0)