Skip to content

Commit 718e771

Browse files
committed
fix(e2): close environment bindings only on rebindings that surely run
A rebinding now closes only the bindings made in its own block or in blocks nested inside it. One inside an if, try, with, match or loop body leaves the outer binding open, and later uses count against both. Nothing closes where a raise, break or continue can skip the rebinding (try, except, with and loop bodies), or through a walrus. A read anywhere in a loop counts against every binding made in that loop, which covers the next pass reading this pass's copy. This stays linear: visiting loop bodies twice would double the work at every nesting level, and the open bindings per name are capped so a crafted file cannot make the pass quadratic. pop and setdefault count as in-place edits when their value is discarded or comes from one literal key; popitem only when discarded. A walrus result, a class attribute, a class-body read that falls through to globals, and any name in a file that calls globals(), locals(), vars(), eval() or exec() keep the finding. A RecursionError in the flow pass falls back to reporting every full read instead of dropping the file's E1-E5 findings. Signed-off-by: Malin Fossum <[email protected]>
1 parent dbf917f commit 718e771

2 files changed

Lines changed: 484 additions & 46 deletions

File tree

‎src/skillspector/nodes/analyzers/static_patterns_data_exfiltration.py‎

Lines changed: 209 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@
1818
from __future__ import annotations
1919

2020
import ast
21+
import itertools
2122
import re
2223
import sys
24+
from collections.abc import Sequence
2325
from dataclasses import dataclass, field
2426

2527
from skillspector.logging_config import get_logger
@@ -203,20 +205,87 @@ def _is_dynamic_copy_call(call: ast.Call, aliases: dict[str, str]) -> bool:
203205
)
204206

205207

206-
# In-place edits of a mapping that keep the values on the host.
207-
_ENVIRONMENT_MUTATION_METHODS = frozenset({"update", "pop", "popitem", "setdefault", "clear"})
208+
# In-place edits of a mapping that keep the values on the host. ``update`` and
209+
# ``clear`` return ``None``. The extraction methods hand a value back, so they
210+
# only count as edits when that value is thrown away, or when it is the value of
211+
# one literal key: a targeted lookup, which E2 does not treat as harvesting.
212+
_ENVIRONMENT_MUTATION_METHODS = frozenset({"update", "clear"})
213+
_ENVIRONMENT_EXTRACTION_METHODS = frozenset({"pop", "popitem", "setdefault"})
214+
_ENVIRONMENT_SINGLE_KEY_METHODS = frozenset({"pop", "setdefault"})
215+
216+
# Statement lists a raise, ``break`` or ``continue`` can leave before a later
217+
# rebinding runs, while code that still sees the older binding goes on to run:
218+
# a handler, a ``finally``, a context manager that swallows the error, or the
219+
# code after the loop. Loop bodies are handled in ``_visit_loop``.
220+
_PROTECTED_BODIES = frozenset(
221+
{
222+
(ast.Try, "body"),
223+
(ast.Try, "orelse"),
224+
(ast.TryStar, "body"),
225+
(ast.TryStar, "orelse"),
226+
(ast.ExceptHandler, "body"),
227+
(ast.With, "body"),
228+
(ast.AsyncWith, "body"),
229+
}
230+
)
231+
232+
# Builtins that read variables by a name only known at run time.
233+
_DYNAMIC_NAMESPACE_CALLS = frozenset({"globals", "locals", "vars", "eval", "exec"})
234+
235+
# Bindings of one name that may all still be live, past which the name is
236+
# reported rather than followed. Keeps crafted files linear.
237+
_MAX_OPEN_BINDINGS = 32
238+
239+
240+
def _method_receiver(node: ast.AST, methods: frozenset[str]) -> ast.Name | None:
241+
"""Return the bare name a ``name.method(...)`` call is made on, for *methods*."""
242+
if (
243+
isinstance(node, ast.Call)
244+
and isinstance(node.func, ast.Attribute)
245+
and node.func.attr in methods
246+
and isinstance(node.func.value, ast.Name)
247+
):
248+
return node.func.value
249+
return None
250+
251+
252+
def _edit_receiver(node: ast.AST) -> ast.Name | None:
253+
"""Return the name a call edits in place without taking values off the mapping."""
254+
receiver = _method_receiver(node, _ENVIRONMENT_MUTATION_METHODS)
255+
if receiver is None and isinstance(node, ast.Expr):
256+
receiver = _method_receiver(node.value, _ENVIRONMENT_EXTRACTION_METHODS)
257+
if receiver is None and isinstance(node, ast.Call):
258+
single_key = _method_receiver(node, _ENVIRONMENT_SINGLE_KEY_METHODS)
259+
if single_key is not None and (
260+
node.args
261+
and isinstance(node.args[0], ast.Constant)
262+
and isinstance(node.args[0].value, str)
263+
):
264+
receiver = single_key
265+
return receiver
208266

209267

210268
@dataclass
211269
class _EnvironmentBinding:
212-
"""One name bound to a candidate expression, tracked until the name is rebound."""
270+
"""One name bound to a candidate expression, tracked until a rebinding that surely runs."""
213271

214272
value_id: int
273+
# The statement block the binding was made in, as a path of block numbers.
274+
block: tuple[int, ...]
215275
reached_child_process: bool = False
216276
escaped: bool = False
217277

218278

219279
@dataclass
280+
class _Loop:
281+
"""Names read and bindings made in one loop, whose body may run again."""
282+
283+
reads: set[tuple[_Scope, str]] = field(default_factory=set)
284+
bindings: list[tuple[_Scope, str, _EnvironmentBinding]] = field(default_factory=list)
285+
286+
287+
# Compared by identity, so a scope can key the per-loop read set.
288+
@dataclass(eq=False)
220289
class _Scope:
221290
"""Open bindings of one lexical scope: module, function, lambda, class or comprehension."""
222291

@@ -227,14 +296,15 @@ class _Scope:
227296
deferred: bool = False
228297
is_class: bool = False
229298
is_comprehension: bool = False
230-
open: dict[str, _EnvironmentBinding] = field(default_factory=dict)
299+
# Several bindings of one name stay open when a rebinding may not have run.
300+
open: dict[str, list[_EnvironmentBinding]] = field(default_factory=dict)
231301
read_later: set[str] = field(default_factory=set)
232302

233303
def owns(self, name: str) -> bool:
234304
return self.local_names is None or name in self.local_names
235305

236306

237-
def _scope_declarations(body: list[ast.AST]) -> tuple[set[str], set[str]]:
307+
def _scope_declarations(body: Sequence[ast.AST]) -> tuple[set[str], set[str]]:
238308
"""Return the names a scope binds itself and the names it declares ``global``.
239309
240310
Nested functions, lambdas and classes are scopes of their own, so only their
@@ -295,11 +365,24 @@ class _EnvironmentFlowVisitor(ast.NodeVisitor):
295365
"""Decide which environment mappings only ever reach a child-process ``env=``.
296366
297367
Names are followed per lexical scope, in evaluation order (assignment values
298-
before their targets), until they are rebound. A later ``env = {}`` handed to
299-
a launcher therefore cannot vouch for an earlier ``env = os.environ.copy()``,
300-
and a nested scope's own ``env`` cannot close the outer one. A binding passes
301-
through only if every use up to the rebinding is a child-process ``env=``
302-
argument or an in-place edit of the mapping; any other use keeps the finding.
368+
before their targets), until a rebinding that surely runs. A later ``env = {}``
369+
handed to a launcher therefore cannot vouch for an earlier
370+
``env = os.environ.copy()``, and a nested scope's own ``env`` cannot close the
371+
outer one. A binding passes through only if every use is a child-process
372+
``env=`` argument or an in-place edit of the mapping; any other use keeps the
373+
finding.
374+
375+
Control flow errs toward keeping the finding. A rebinding closes only the
376+
bindings made in its own block or in blocks nested inside it, so one inside an
377+
``if``, ``try``, ``with``, ``match`` or loop body leaves the outer binding open
378+
next to the new one, and later uses count against both. A rebinding never
379+
closes anything where a raise, ``break`` or ``continue`` can skip it (``try``,
380+
``except``, ``with`` and loop bodies), or as a walrus, which can sit in a
381+
short-circuit. A loop body may run again, so a read anywhere in a loop counts
382+
against every binding made in that loop. A walrus result, a class attribute
383+
and any name in a file that reads names dynamically (``globals()``,
384+
``eval``) keep the finding, as does a name with too many live bindings.
385+
303386
A function or lambda body runs at some later time, so its reads of an
304387
enclosing name count against every binding of that name that is open when the
305388
body is defined or made after it. Its writes to an enclosing name through
@@ -310,18 +393,19 @@ class _EnvironmentFlowVisitor(ast.NodeVisitor):
310393
def __init__(self, tree: ast.AST, aliases: dict[str, str]) -> None:
311394
self._env_arguments: set[int] = set()
312395
self._mutated_names: set[int] = set()
396+
self._dynamic_reads = False
313397
for node in ast.walk(tree):
314-
if isinstance(node, ast.Call):
315-
if resolve_call_name(node, aliases) in _CHILD_PROCESS_ENV_CALLS:
398+
receiver = _edit_receiver(node)
399+
if receiver is not None:
400+
self._mutated_names.add(id(receiver))
401+
elif isinstance(node, ast.Call):
402+
call_name = resolve_call_name(node, aliases)
403+
if call_name in _CHILD_PROCESS_ENV_CALLS:
316404
self._env_arguments.update(
317405
id(keyword.value) for keyword in node.keywords if keyword.arg == "env"
318406
)
319-
elif (
320-
isinstance(node.func, ast.Attribute)
321-
and node.func.attr in _ENVIRONMENT_MUTATION_METHODS
322-
and isinstance(node.func.value, ast.Name)
323-
):
324-
self._mutated_names.add(id(node.func.value))
407+
elif call_name in _DYNAMIC_NAMESPACE_CALLS:
408+
self._dynamic_reads = True
325409
elif (
326410
isinstance(node, ast.Subscript)
327411
and isinstance(node.ctx, (ast.Store, ast.Del))
@@ -333,20 +417,36 @@ def __init__(self, tree: ast.AST, aliases: dict[str, str]) -> None:
333417
self._scopes: list[_Scope] = [_Scope(local_names=None)]
334418
self._passed_through: set[int] = set()
335419
self._escaped: set[int] = set()
420+
self._block_numbers = itertools.count()
421+
self._block: tuple[int, ...] = ()
422+
# How many protected bodies of the current scope enclose this point.
423+
self._protected = 0
424+
self._loops: list[_Loop] = []
336425

337426
def passthrough_ids(self) -> set[int]:
338427
"""Ids of expressions whose only destination is a child-process ``env=``."""
339428
self._close_scope(self._scopes[0])
429+
if self._dynamic_reads:
430+
# Any name may be read back by a string, so only inline arguments are safe.
431+
return set(self._env_arguments)
340432
return (self._passed_through - self._escaped) | self._env_arguments
341433

342-
def _close(self, scope: _Scope, name: str) -> None:
343-
binding = scope.open.pop(name, None)
344-
if binding is None:
345-
return
346-
if binding.escaped:
347-
self._escaped.add(binding.value_id)
348-
elif binding.reached_child_process:
349-
self._passed_through.add(binding.value_id)
434+
def _close(self, scope: _Scope, name: str, block: tuple[int, ...] | None = None) -> None:
435+
"""Close the bindings of ``name`` made in ``block`` or in blocks nested inside it.
436+
437+
Those are the bindings a rebinding in ``block`` surely replaces; ``None``
438+
closes them all.
439+
"""
440+
still_open: list[_EnvironmentBinding] = []
441+
for binding in scope.open.pop(name, []):
442+
if block is not None and binding.block[: len(block)] != block:
443+
still_open.append(binding)
444+
elif binding.escaped:
445+
self._escaped.add(binding.value_id)
446+
elif binding.reached_child_process:
447+
self._passed_through.add(binding.value_id)
448+
if still_open:
449+
scope.open[name] = still_open
350450

351451
def _close_scope(self, scope: _Scope) -> None:
352452
for name in list(scope.open):
@@ -374,21 +474,70 @@ def _store(self, name: str, value_id: int | None, *, walrus: bool = False) -> No
374474
scope, deferred = self._resolve(name, walrus=walrus)
375475
if deferred:
376476
return
377-
self._close(scope, name)
378-
if value_id is not None:
379-
scope.open[name] = _EnvironmentBinding(value_id, escaped=name in scope.read_later)
477+
if not walrus and not self._protected:
478+
self._close(scope, name, self._block)
479+
if value_id is None:
480+
return
481+
# A walrus also hands the value to the enclosing expression, and a class
482+
# attribute outlives the class body.
483+
escaped = walrus or scope.is_class or name in scope.read_later
484+
binding = _EnvironmentBinding(value_id, self._block, escaped=escaped)
485+
bindings = scope.open.setdefault(name, [])
486+
bindings.append(binding)
487+
for loop in self._loops:
488+
loop.bindings.append((scope, name, binding))
489+
if len(bindings) > _MAX_OPEN_BINDINGS:
490+
self._escaped.update(open_binding.value_id for open_binding in bindings)
491+
del scope.open[name]
380492

381493
def _bind(self, target: ast.expr, value_id: int, *, walrus: bool = False) -> None:
382494
if isinstance(target, ast.Name):
383495
self._store(target.id, value_id, walrus=walrus)
384496
else:
385497
self.visit(target)
386498

499+
def _visit_block(self, nodes: Sequence[ast.AST], *, protected: bool = False) -> None:
500+
"""Visit statements that may run zero, one or several times as one block."""
501+
outer = self._block
502+
self._block = (*outer, next(self._block_numbers))
503+
self._protected += protected
504+
for node in nodes:
505+
self.visit(node)
506+
self._protected -= protected
507+
self._block = outer
508+
509+
def _visit_loop(self, header: Sequence[ast.AST], body: list[ast.stmt]) -> None:
510+
"""Visit a loop's repeated part, then count its reads against its bindings."""
511+
loop = _Loop()
512+
self._loops.append(loop)
513+
# ``break`` and ``continue`` can skip any rebinding in the body.
514+
self._visit_block([*header, *body], protected=True)
515+
self._loops.pop()
516+
# The next pass reads what this pass bound, whatever the source order.
517+
for scope, name, binding in loop.bindings:
518+
if (scope, name) in loop.reads:
519+
self._escaped.add(binding.value_id)
520+
521+
def generic_visit(self, node: ast.AST) -> None:
522+
"""Visit children in order, giving every nested statement list its own block."""
523+
for field_name, value in ast.iter_fields(node):
524+
if isinstance(value, list) and value and isinstance(value[0], ast.stmt):
525+
protected = (type(node), field_name) in _PROTECTED_BODIES
526+
self._visit_block(value, protected=protected)
527+
elif isinstance(value, list):
528+
for item in value:
529+
if isinstance(item, ast.AST):
530+
self.visit(item)
531+
elif isinstance(value, ast.AST):
532+
self.visit(value)
533+
387534
def _visit_scope(self, scope: _Scope, body: list[ast.stmt] | list[ast.expr]) -> None:
535+
# A ``try`` around a definition cannot see the locals of its body.
536+
outer_protected, self._protected = self._protected, 0
388537
self._scopes.append(scope)
389-
for statement in body:
390-
self.visit(statement)
538+
self._visit_block(body)
391539
self._close_scope(self._scopes.pop())
540+
self._protected = outer_protected
392541

393542
def _visit_defaults(self, arguments: ast.arguments) -> None:
394543
for default in (*arguments.defaults, *arguments.kw_defaults):
@@ -437,14 +586,13 @@ def _visit_comprehension(
437586
}
438587
deferred = isinstance(node, ast.GeneratorExp)
439588
self._scopes.append(_Scope(targets, deferred=deferred, is_comprehension=True))
589+
repeated: list[ast.AST] = []
440590
for index, generator in enumerate(node.generators):
441591
if index:
442-
self.visit(generator.iter)
443-
self.visit(generator.target)
444-
for condition in generator.ifs:
445-
self.visit(condition)
446-
for result in results:
447-
self.visit(result)
592+
repeated.append(generator.iter)
593+
repeated.append(generator.target)
594+
repeated.extend(generator.ifs)
595+
self._visit_loop([*repeated, *results], [])
448596
self._close_scope(self._scopes.pop())
449597

450598
def visit_ListComp(self, node: ast.ListComp) -> None:
@@ -480,28 +628,38 @@ def visit_AugAssign(self, node: ast.AugAssign) -> None:
480628

481629
def visit_For(self, node: ast.For | ast.AsyncFor) -> None:
482630
self.visit(node.iter)
483-
self.visit(node.target)
484-
for statement in node.body + node.orelse:
485-
self.visit(statement)
631+
# The target is rebound on every pass, and not at all for an empty iterable.
632+
self._visit_loop([node.target], node.body)
633+
self._visit_block(node.orelse)
486634

487635
def visit_AsyncFor(self, node: ast.AsyncFor) -> None:
488636
self.visit_For(node)
489637

638+
def visit_While(self, node: ast.While) -> None:
639+
# The test runs again before every pass.
640+
self._visit_loop([node.test], node.body)
641+
self._visit_block(node.orelse)
642+
490643
def visit_Name(self, node: ast.Name) -> None:
491644
node_id = id(node)
492645
if not isinstance(node.ctx, ast.Load) and node_id not in self._mutated_names:
493646
self._store(node.id, None)
494647
return
495648
scope, deferred = self._resolve(node.id)
496-
binding = scope.open.get(node.id)
649+
bindings = scope.open.get(node.id, [])
497650
if node_id in self._env_arguments:
498-
if binding is not None:
651+
for binding in bindings:
499652
binding.reached_child_process = True
500653
elif node_id not in self._mutated_names:
501654
if deferred:
502655
scope.read_later.add(node.id)
503-
if binding is not None:
656+
if scope.is_class:
657+
# Before the class body binds the name, the read falls through to globals.
658+
bindings = [*bindings, *self._scopes[0].open.get(node.id, [])]
659+
for binding in bindings:
504660
binding.escaped = True
661+
for loop in self._loops:
662+
loop.reads.add((scope, node.id))
505663

506664

507665
def _analyze_python_environment_reads(
@@ -529,9 +687,14 @@ def _analyze_python_environment_reads(
529687

530688
aliases = python_ast.import_aliases
531689
lines = python_ast.lines
532-
flow = _EnvironmentFlowVisitor(tree, aliases)
533-
flow.visit(tree)
534-
child_process_passthroughs = flow.passthrough_ids()
690+
try:
691+
flow = _EnvironmentFlowVisitor(tree, aliases)
692+
flow.visit(tree)
693+
child_process_passthroughs = flow.passthrough_ids()
694+
except RecursionError:
695+
# Too deep for the recursive flow pass: report every full read, as before
696+
# the child-process exemption existed, instead of losing the whole file.
697+
child_process_passthroughs = set()
535698
findings: list[AnalyzerFinding] = []
536699
emitted: set[int] = set()
537700
tag = [PatternCategory.DATA_EXFILTRATION.value]

0 commit comments

Comments
 (0)