Skip to content

Commit 51e9a9f

Browse files
Merge pull request #14447 from RonnyPfannschmidt/ronny/fix-14445-assert-walrus
fix(rewrite): prevent walrus operator double evaluation in assertions
2 parents e984a9b + eb47e44 commit 51e9a9f

4 files changed

Lines changed: 329 additions & 259 deletions

File tree

‎changelog/14445.bugfix.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Assertion rewriting no longer evaluates walrus operator (``:=``) expressions more than once, which reported results the test never computed when the expression had side effects.
2+
3+
An operand evaluated before a later walrus is now reported with the value it actually saw, so ``assert value != identity(value := value.lower())`` keeps Python's left-to-right order.

‎src/_pytest/assertion/rewrite.py‎

Lines changed: 80 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
from __future__ import annotations
44

55
import ast
6-
from collections import defaultdict
76
from collections.abc import Callable
87
from collections.abc import Iterable
98
from collections.abc import Iterator
@@ -57,20 +56,13 @@
5756
from _pytest.assertion import AssertionState
5857

5958

60-
class Sentinel:
61-
pass
62-
63-
6459
assertstate_key = StashKey["AssertionState"]()
6560

6661
# pytest caches rewritten pycs in pycache dirs
6762
PYTEST_TAG = f"{sys.implementation.cache_tag}-pytest-{version}"
6863
PYC_EXT = ".py" + ((__debug__ and "c") or "o")
6964
PYC_TAIL = "." + PYTEST_TAG + PYC_EXT
7065

71-
# Special marker that denotes we have just left a scope definition
72-
_SCOPE_END_MARKER = Sentinel()
73-
7466

7567
class AssertionRewritingHook(importlib.abc.MetaPathFinder, importlib.abc.Loader):
7668
"""PEP302/PEP451 import hook which rewrites asserts."""
@@ -538,6 +530,16 @@ def traverse_node(node: ast.AST) -> Iterator[ast.AST]:
538530
yield from traverse_node(child)
539531

540532

533+
def _walrus_targets(nodes: Iterable[ast.expr]) -> set[str]:
534+
"""Return the names any walrus operator in *nodes* rebinds."""
535+
return {
536+
sub.target.id
537+
for node in nodes
538+
for sub in ast.walk(node)
539+
if isinstance(sub, ast.NamedExpr)
540+
}
541+
542+
541543
@functools.lru_cache(maxsize=1)
542544
def _get_assertion_exprs(src: bytes) -> dict[int, str]:
543545
"""Return a mapping from {lineno: "assertion test expression"}."""
@@ -642,14 +644,8 @@ class AssertionRewriter(ast.NodeVisitor):
642644
.push_format_context() and .pop_format_context() which allows
643645
to build another %-formatted string while already building one.
644646
645-
:scope: A tuple containing the current scope used for variables_overwrite.
646-
647-
:variables_overwrite: A dict filled with references to variables
648-
that change value within an assert. This happens when a variable is
649-
reassigned with the walrus operator
650-
651-
This state, except the variables_overwrite, is reset on every new assert
652-
statement visited and used by the other visitors.
647+
This state is reset on every new assert statement visited and used by
648+
the other visitors.
653649
"""
654650

655651
def __init__(
@@ -665,10 +661,6 @@ def __init__(
665661
else:
666662
self.enable_assertion_pass_hook = False
667663
self.source = source
668-
self.scope: tuple[ast.AST, ...] = ()
669-
self.variables_overwrite: defaultdict[tuple[ast.AST, ...], dict[str, str]] = (
670-
defaultdict(dict)
671-
)
672664

673665
def run(self, mod: ast.Module) -> None:
674666
"""Find all assert statements in *mod* and rewrite them."""
@@ -718,16 +710,9 @@ def run(self, mod: ast.Module) -> None:
718710
mod.body[pos:pos] = imports
719711

720712
# Collect asserts.
721-
self.scope = (mod,)
722-
nodes: list[ast.AST | Sentinel] = [mod]
713+
nodes: list[ast.AST] = [mod]
723714
while nodes:
724715
node = nodes.pop()
725-
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef):
726-
self.scope = (*self.scope, node)
727-
nodes.append(_SCOPE_END_MARKER)
728-
if node == _SCOPE_END_MARKER:
729-
self.scope = self.scope[:-1]
730-
continue
731716
assert isinstance(node, ast.AST)
732717
for name, field in ast.iter_fields(node):
733718
if isinstance(field, list):
@@ -954,15 +939,16 @@ def visit_Assert(self, assert_: ast.Assert) -> list[ast.stmt]:
954939
return self.statements
955940

956941
def visit_NamedExpr(self, name: ast.NamedExpr) -> tuple[ast.NamedExpr, str]:
957-
# This method handles the 'walrus operator' repr of the target
958-
# name if it's a local variable or _should_repr_global_name()
959-
# thinks it's acceptable.
942+
# Return the NamedExpr as-is so it evaluates in its natural position
943+
# (preserving left-to-right evaluation order in function calls, etc.).
944+
# For the explanation, reference the target variable.
960945
locs = ast.Call(self.builtin("locals"), [], [])
961946
target_id = name.target.id
947+
target_name = ast.Name(target_id, ast.Load())
962948
inlocs = ast.Compare(ast.Constant(target_id), [ast.In()], [locs])
963-
dorepr = self.helper("_should_repr_global_name", name)
949+
dorepr = self.helper("_should_repr_global_name", target_name)
964950
test = ast.BoolOp(ast.Or(), [inlocs, dorepr])
965-
expr = ast.IfExp(test, self.display(name), ast.Constant(target_id))
951+
expr = ast.IfExp(test, self.display(target_name), ast.Constant(target_id))
966952
return name, self.explanation_param(expr)
967953

968954
def visit_Name(self, name: ast.Name) -> tuple[ast.Name, str]:
@@ -975,6 +961,27 @@ def visit_Name(self, name: ast.Name) -> tuple[ast.Name, str]:
975961
expr = ast.IfExp(test, self.display(name), ast.Constant(name.id))
976962
return name, self.explanation_param(expr)
977963

964+
def visit_operand(
965+
self, operand: ast.expr, later: Sequence[ast.expr]
966+
) -> tuple[ast.expr, str]:
967+
"""Visit an operand, freezing it against walrus operators in *later*.
968+
969+
Operands are rewritten into statements that run in source order, but
970+
a plain name is left as a bare load evaluated at the very end, when
971+
the enclosing expression is assembled. A walrus operator in a later
972+
operand rebinds that name in between, so both the value used and the
973+
value reported would be the post-walrus one -- Python evaluates the
974+
earlier operand first. Copy the value into a temporary instead.
975+
"""
976+
specifiers = set(self.explanation_specifiers)
977+
res, expl = self.visit(operand)
978+
if isinstance(res, ast.Name) and res.id in _walrus_targets(later):
979+
snapshot = self.assign(res)
980+
for key in set(self.explanation_specifiers) - specifiers:
981+
self.explanation_specifiers[key] = self.display(snapshot)
982+
res = snapshot
983+
return res, expl
984+
978985
def visit_BoolOp(self, boolop: ast.BoolOp) -> tuple[ast.Name, str]:
979986
res_var = self.variable()
980987
expl_list = self.assign(ast.List([], ast.Load()))
@@ -983,32 +990,43 @@ def visit_BoolOp(self, boolop: ast.BoolOp) -> tuple[ast.Name, str]:
983990
body = save = self.statements
984991
fail_save = self.expl_stmts
985992
levels = len(boolop.values) - 1
993+
later_walrus_targets = [
994+
_walrus_targets(boolop.values[idx + 1 :])
995+
for idx in range(len(boolop.values))
996+
]
986997
self.push_format_context()
987-
# Process each operand, short-circuiting if needed.
998+
# Process each operand, short-circuiting as needed.
988999
for i, v in enumerate(boolop.values):
9891000
if i:
9901001
fail_inner: list[ast.stmt] = []
9911002
# cond is set in a prior loop iteration below
9921003
self.expl_stmts.append(ast.If(cond, fail_inner, [])) # noqa: F821
9931004
self.expl_stmts = fail_inner
994-
match v:
995-
# Check if the left operand is an ast.NamedExpr and the value has already been visited
996-
case ast.Compare(
997-
left=ast.NamedExpr(target=ast.Name(id=target_id))
998-
) if target_id in [
999-
e.id for e in boolop.values[:i] if hasattr(e, "id")
1000-
]:
1001-
pytest_temp = self.variable()
1002-
self.variables_overwrite[self.scope][target_id] = v.left # type:ignore[assignment]
1003-
# mypy's false positive, we're checking that the 'target' attribute exists.
1004-
v.left.target.id = pytest_temp # type:ignore[attr-defined]
10051005
self.push_format_context()
10061006
res, expl = self.visit(v)
10071007
body.append(ast.Assign([ast.Name(res_var, ast.Store())], res))
1008+
# Snapshot when the raw ``res`` node would be unsafe to reuse
1009+
# as a condition or explanation reference:
1010+
# - NamedExpr (non-last): reusing the node re-evaluates the
1011+
# walrus expression including any side effects.
1012+
# - Name whose variable a later walrus overwrites: the
1013+
# explanation would show the post-walrus value.
1014+
needs_snapshot = (isinstance(v, ast.NamedExpr) and i < levels) or (
1015+
isinstance(v, ast.Name) and v.id in later_walrus_targets[i]
1016+
)
1017+
if needs_snapshot:
1018+
snapshot = self.assign(ast.Name(res_var, ast.Load()))
1019+
res = snapshot
1020+
for key in self.stack[-1]:
1021+
self.stack[-1][key] = self.display(snapshot)
10081022
expl_format = self.pop_format_context(ast.Constant(expl))
10091023
call = ast.Call(app, [expl_format], [])
10101024
self.expl_stmts.append(ast.Expr(call))
10111025
if i < levels:
1026+
# Short-circuit: and → continue if truthy; or → if falsy.
1027+
# ``res`` is a stable reference (Name vars are only
1028+
# snapshotted when a later walrus would corrupt them;
1029+
# calls/compares return @py_assert vars from assign()).
10121030
cond: ast.expr = res
10131031
if is_or:
10141032
cond = ast.UnaryOp(ast.Not(), cond)
@@ -1029,7 +1047,7 @@ def visit_UnaryOp(self, unary: ast.UnaryOp) -> tuple[ast.Name, str]:
10291047

10301048
def visit_BinOp(self, binop: ast.BinOp) -> tuple[ast.Name, str]:
10311049
symbol = BINOP_MAP[binop.op.__class__]
1032-
left_expr, left_expl = self.visit(binop.left)
1050+
left_expr, left_expl = self.visit_operand(binop.left, [binop.right])
10331051
right_expr, right_expl = self.visit(binop.right)
10341052
explanation = f"({left_expl} {symbol} {right_expl})"
10351053
res = self.assign(
@@ -1038,25 +1056,19 @@ def visit_BinOp(self, binop: ast.BinOp) -> tuple[ast.Name, str]:
10381056
return res, explanation
10391057

10401058
def visit_Call(self, call: ast.Call) -> tuple[ast.Name, str]:
1041-
new_func, func_expl = self.visit(call.func)
1059+
# The callee and every argument are evaluated left to right, so each of
1060+
# them has to be frozen against walrus operators in what follows.
1061+
operands = [*call.args, *(keyword.value for keyword in call.keywords)]
1062+
new_func, func_expl = self.visit_operand(call.func, operands)
10421063
arg_expls = []
10431064
new_args = []
10441065
new_kwargs = []
1045-
for arg in call.args:
1046-
if isinstance(arg, ast.Name) and arg.id in self.variables_overwrite.get(
1047-
self.scope, {}
1048-
):
1049-
arg = self.variables_overwrite[self.scope][arg.id] # type:ignore[assignment]
1050-
res, expl = self.visit(arg)
1066+
for i, arg in enumerate(call.args):
1067+
res, expl = self.visit_operand(arg, operands[i + 1 :])
10511068
arg_expls.append(expl)
10521069
new_args.append(res)
1053-
for keyword in call.keywords:
1054-
match keyword.value:
1055-
case ast.Name(id=id) if id in self.variables_overwrite.get(
1056-
self.scope, {}
1057-
):
1058-
keyword.value = self.variables_overwrite[self.scope][id] # type:ignore[assignment]
1059-
res, expl = self.visit(keyword.value)
1070+
for i, keyword in enumerate(call.keywords, start=len(call.args)):
1071+
res, expl = self.visit_operand(keyword.value, operands[i + 1 :])
10601072
new_kwargs.append(ast.keyword(keyword.arg, res))
10611073
if keyword.arg:
10621074
arg_expls.append(keyword.arg + "=" + expl)
@@ -1090,17 +1102,11 @@ def visit_Attribute(self, attr: ast.Attribute) -> tuple[ast.Name, str]:
10901102

10911103
def visit_Compare(self, comp: ast.Compare) -> tuple[ast.expr, str]:
10921104
self.push_format_context()
1093-
# We first check if we have overwritten a variable in the previous assert
1094-
match comp.left:
1095-
case ast.Name(id=name_id) if name_id in self.variables_overwrite.get(
1096-
self.scope, {}
1097-
):
1098-
comp.left = self.variables_overwrite[self.scope][name_id] # type: ignore[assignment]
1099-
case ast.NamedExpr(target=ast.Name(id=target_id)):
1100-
self.variables_overwrite[self.scope][target_id] = comp.left # type: ignore[assignment]
1101-
left_res, left_expl = self.visit(comp.left)
1105+
left_res, left_expl = self.visit_operand(comp.left, comp.comparators)
11021106
if isinstance(comp.left, ast.Compare | ast.BoolOp):
11031107
left_expl = f"({left_expl})"
1108+
if isinstance(left_res, ast.NamedExpr):
1109+
left_res = self.assign(left_res)
11041110
res_variables = [self.variable() for i in range(len(comp.ops))]
11051111
load_names: list[ast.expr] = [ast.Name(v, ast.Load()) for v in res_variables]
11061112
store_names = [ast.Name(v, ast.Store()) for v in res_variables]
@@ -1109,17 +1115,13 @@ def visit_Compare(self, comp: ast.Compare) -> tuple[ast.expr, str]:
11091115
syms: list[ast.expr] = []
11101116
results = [left_res]
11111117
for i, op, next_operand in it:
1112-
match (next_operand, left_res):
1113-
case (
1114-
ast.NamedExpr(target=ast.Name(id=target_id)),
1115-
ast.Name(id=name_id),
1116-
) if target_id == name_id:
1117-
next_operand.target.id = self.variable()
1118-
self.variables_overwrite[self.scope][name_id] = next_operand # type: ignore[assignment]
1119-
1120-
next_res, next_expl = self.visit(next_operand)
1118+
next_res, next_expl = self.visit_operand(
1119+
next_operand, comp.comparators[i + 1 :]
1120+
)
11211121
if isinstance(next_operand, ast.Compare | ast.BoolOp):
11221122
next_expl = f"({next_expl})"
1123+
if isinstance(next_res, ast.NamedExpr):
1124+
next_res = self.assign(next_res)
11231125
results.append(next_res)
11241126
sym = BINOP_MAP[op.__class__]
11251127
syms.append(ast.Constant(sym))

0 commit comments

Comments
 (0)