1818from __future__ import annotations
1919
2020import ast
21+ import itertools
2122import re
2223import sys
24+ from collections .abc import Sequence
2325from dataclasses import dataclass , field
2426
2527from skillspector .logging_config import get_logger
@@ -203,20 +205,87 @@ def _is_dynamic_copy_call(call: ast.Call, aliases: dict[str, str]) -> bool:
203205 )
204206
205207
206- # In-place edits of a mapping that keep the values on the host.
207- _ENVIRONMENT_MUTATION_METHODS = frozenset ({"update" , "pop" , "popitem" , "setdefault" , "clear" })
208+ # In-place edits of a mapping that keep the values on the host. ``update`` and
209+ # ``clear`` return ``None``. The extraction methods hand a value back, so they
210+ # only count as edits when that value is thrown away, or when it is the value of
211+ # one literal key: a targeted lookup, which E2 does not treat as harvesting.
212+ _ENVIRONMENT_MUTATION_METHODS = frozenset ({"update" , "clear" })
213+ _ENVIRONMENT_EXTRACTION_METHODS = frozenset ({"pop" , "popitem" , "setdefault" })
214+ _ENVIRONMENT_SINGLE_KEY_METHODS = frozenset ({"pop" , "setdefault" })
215+
216+ # Statement lists a raise, ``break`` or ``continue`` can leave before a later
217+ # rebinding runs, while code that still sees the older binding goes on to run:
218+ # a handler, a ``finally``, a context manager that swallows the error, or the
219+ # code after the loop. Loop bodies are handled in ``_visit_loop``.
220+ _PROTECTED_BODIES = frozenset (
221+ {
222+ (ast .Try , "body" ),
223+ (ast .Try , "orelse" ),
224+ (ast .TryStar , "body" ),
225+ (ast .TryStar , "orelse" ),
226+ (ast .ExceptHandler , "body" ),
227+ (ast .With , "body" ),
228+ (ast .AsyncWith , "body" ),
229+ }
230+ )
231+
232+ # Builtins that read variables by a name only known at run time.
233+ _DYNAMIC_NAMESPACE_CALLS = frozenset ({"globals" , "locals" , "vars" , "eval" , "exec" })
234+
235+ # Bindings of one name that may all still be live, past which the name is
236+ # reported rather than followed. Keeps crafted files linear.
237+ _MAX_OPEN_BINDINGS = 32
238+
239+
240+ def _method_receiver (node : ast .AST , methods : frozenset [str ]) -> ast .Name | None :
241+ """Return the bare name a ``name.method(...)`` call is made on, for *methods*."""
242+ if (
243+ isinstance (node , ast .Call )
244+ and isinstance (node .func , ast .Attribute )
245+ and node .func .attr in methods
246+ and isinstance (node .func .value , ast .Name )
247+ ):
248+ return node .func .value
249+ return None
250+
251+
252+ def _edit_receiver (node : ast .AST ) -> ast .Name | None :
253+ """Return the name a call edits in place without taking values off the mapping."""
254+ receiver = _method_receiver (node , _ENVIRONMENT_MUTATION_METHODS )
255+ if receiver is None and isinstance (node , ast .Expr ):
256+ receiver = _method_receiver (node .value , _ENVIRONMENT_EXTRACTION_METHODS )
257+ if receiver is None and isinstance (node , ast .Call ):
258+ single_key = _method_receiver (node , _ENVIRONMENT_SINGLE_KEY_METHODS )
259+ if single_key is not None and (
260+ node .args
261+ and isinstance (node .args [0 ], ast .Constant )
262+ and isinstance (node .args [0 ].value , str )
263+ ):
264+ receiver = single_key
265+ return receiver
208266
209267
210268@dataclass
211269class _EnvironmentBinding :
212- """One name bound to a candidate expression, tracked until the name is rebound ."""
270+ """One name bound to a candidate expression, tracked until a rebinding that surely runs ."""
213271
214272 value_id : int
273+ # The statement block the binding was made in, as a path of block numbers.
274+ block : tuple [int , ...]
215275 reached_child_process : bool = False
216276 escaped : bool = False
217277
218278
219279@dataclass
280+ class _Loop :
281+ """Names read and bindings made in one loop, whose body may run again."""
282+
283+ reads : set [tuple [_Scope , str ]] = field (default_factory = set )
284+ bindings : list [tuple [_Scope , str , _EnvironmentBinding ]] = field (default_factory = list )
285+
286+
287+ # Compared by identity, so a scope can key the per-loop read set.
288+ @dataclass (eq = False )
220289class _Scope :
221290 """Open bindings of one lexical scope: module, function, lambda, class or comprehension."""
222291
@@ -227,14 +296,15 @@ class _Scope:
227296 deferred : bool = False
228297 is_class : bool = False
229298 is_comprehension : bool = False
230- open : dict [str , _EnvironmentBinding ] = field (default_factory = dict )
299+ # Several bindings of one name stay open when a rebinding may not have run.
300+ open : dict [str , list [_EnvironmentBinding ]] = field (default_factory = dict )
231301 read_later : set [str ] = field (default_factory = set )
232302
233303 def owns (self , name : str ) -> bool :
234304 return self .local_names is None or name in self .local_names
235305
236306
237- def _scope_declarations (body : list [ast .AST ]) -> tuple [set [str ], set [str ]]:
307+ def _scope_declarations (body : Sequence [ast .AST ]) -> tuple [set [str ], set [str ]]:
238308 """Return the names a scope binds itself and the names it declares ``global``.
239309
240310 Nested functions, lambdas and classes are scopes of their own, so only their
@@ -295,11 +365,24 @@ class _EnvironmentFlowVisitor(ast.NodeVisitor):
295365 """Decide which environment mappings only ever reach a child-process ``env=``.
296366
297367 Names are followed per lexical scope, in evaluation order (assignment values
298- before their targets), until they are rebound. A later ``env = {}`` handed to
299- a launcher therefore cannot vouch for an earlier ``env = os.environ.copy()``,
300- and a nested scope's own ``env`` cannot close the outer one. A binding passes
301- through only if every use up to the rebinding is a child-process ``env=``
302- argument or an in-place edit of the mapping; any other use keeps the finding.
368+ before their targets), until a rebinding that surely runs. A later ``env = {}``
369+ handed to a launcher therefore cannot vouch for an earlier
370+ ``env = os.environ.copy()``, and a nested scope's own ``env`` cannot close the
371+ outer one. A binding passes through only if every use is a child-process
372+ ``env=`` argument or an in-place edit of the mapping; any other use keeps the
373+ finding.
374+
375+ Control flow errs toward keeping the finding. A rebinding closes only the
376+ bindings made in its own block or in blocks nested inside it, so one inside an
377+ ``if``, ``try``, ``with``, ``match`` or loop body leaves the outer binding open
378+ next to the new one, and later uses count against both. A rebinding never
379+ closes anything where a raise, ``break`` or ``continue`` can skip it (``try``,
380+ ``except``, ``with`` and loop bodies), or as a walrus, which can sit in a
381+ short-circuit. A loop body may run again, so a read anywhere in a loop counts
382+ against every binding made in that loop. A walrus result, a class attribute
383+ and any name in a file that reads names dynamically (``globals()``,
384+ ``eval``) keep the finding, as does a name with too many live bindings.
385+
303386 A function or lambda body runs at some later time, so its reads of an
304387 enclosing name count against every binding of that name that is open when the
305388 body is defined or made after it. Its writes to an enclosing name through
@@ -310,18 +393,19 @@ class _EnvironmentFlowVisitor(ast.NodeVisitor):
310393 def __init__ (self , tree : ast .AST , aliases : dict [str , str ]) -> None :
311394 self ._env_arguments : set [int ] = set ()
312395 self ._mutated_names : set [int ] = set ()
396+ self ._dynamic_reads = False
313397 for node in ast .walk (tree ):
314- if isinstance (node , ast .Call ):
315- if resolve_call_name (node , aliases ) in _CHILD_PROCESS_ENV_CALLS :
398+ receiver = _edit_receiver (node )
399+ if receiver is not None :
400+ self ._mutated_names .add (id (receiver ))
401+ elif isinstance (node , ast .Call ):
402+ call_name = resolve_call_name (node , aliases )
403+ if call_name in _CHILD_PROCESS_ENV_CALLS :
316404 self ._env_arguments .update (
317405 id (keyword .value ) for keyword in node .keywords if keyword .arg == "env"
318406 )
319- elif (
320- isinstance (node .func , ast .Attribute )
321- and node .func .attr in _ENVIRONMENT_MUTATION_METHODS
322- and isinstance (node .func .value , ast .Name )
323- ):
324- self ._mutated_names .add (id (node .func .value ))
407+ elif call_name in _DYNAMIC_NAMESPACE_CALLS :
408+ self ._dynamic_reads = True
325409 elif (
326410 isinstance (node , ast .Subscript )
327411 and isinstance (node .ctx , (ast .Store , ast .Del ))
@@ -333,20 +417,36 @@ def __init__(self, tree: ast.AST, aliases: dict[str, str]) -> None:
333417 self ._scopes : list [_Scope ] = [_Scope (local_names = None )]
334418 self ._passed_through : set [int ] = set ()
335419 self ._escaped : set [int ] = set ()
420+ self ._block_numbers = itertools .count ()
421+ self ._block : tuple [int , ...] = ()
422+ # How many protected bodies of the current scope enclose this point.
423+ self ._protected = 0
424+ self ._loops : list [_Loop ] = []
336425
337426 def passthrough_ids (self ) -> set [int ]:
338427 """Ids of expressions whose only destination is a child-process ``env=``."""
339428 self ._close_scope (self ._scopes [0 ])
429+ if self ._dynamic_reads :
430+ # Any name may be read back by a string, so only inline arguments are safe.
431+ return set (self ._env_arguments )
340432 return (self ._passed_through - self ._escaped ) | self ._env_arguments
341433
342- def _close (self , scope : _Scope , name : str ) -> None :
343- binding = scope .open .pop (name , None )
344- if binding is None :
345- return
346- if binding .escaped :
347- self ._escaped .add (binding .value_id )
348- elif binding .reached_child_process :
349- self ._passed_through .add (binding .value_id )
434+ def _close (self , scope : _Scope , name : str , block : tuple [int , ...] | None = None ) -> None :
435+ """Close the bindings of ``name`` made in ``block`` or in blocks nested inside it.
436+
437+ Those are the bindings a rebinding in ``block`` surely replaces; ``None``
438+ closes them all.
439+ """
440+ still_open : list [_EnvironmentBinding ] = []
441+ for binding in scope .open .pop (name , []):
442+ if block is not None and binding .block [: len (block )] != block :
443+ still_open .append (binding )
444+ elif binding .escaped :
445+ self ._escaped .add (binding .value_id )
446+ elif binding .reached_child_process :
447+ self ._passed_through .add (binding .value_id )
448+ if still_open :
449+ scope .open [name ] = still_open
350450
351451 def _close_scope (self , scope : _Scope ) -> None :
352452 for name in list (scope .open ):
@@ -374,21 +474,70 @@ def _store(self, name: str, value_id: int | None, *, walrus: bool = False) -> No
374474 scope , deferred = self ._resolve (name , walrus = walrus )
375475 if deferred :
376476 return
377- self ._close (scope , name )
378- if value_id is not None :
379- scope .open [name ] = _EnvironmentBinding (value_id , escaped = name in scope .read_later )
477+ if not walrus and not self ._protected :
478+ self ._close (scope , name , self ._block )
479+ if value_id is None :
480+ return
481+ # A walrus also hands the value to the enclosing expression, and a class
482+ # attribute outlives the class body.
483+ escaped = walrus or scope .is_class or name in scope .read_later
484+ binding = _EnvironmentBinding (value_id , self ._block , escaped = escaped )
485+ bindings = scope .open .setdefault (name , [])
486+ bindings .append (binding )
487+ for loop in self ._loops :
488+ loop .bindings .append ((scope , name , binding ))
489+ if len (bindings ) > _MAX_OPEN_BINDINGS :
490+ self ._escaped .update (open_binding .value_id for open_binding in bindings )
491+ del scope .open [name ]
380492
381493 def _bind (self , target : ast .expr , value_id : int , * , walrus : bool = False ) -> None :
382494 if isinstance (target , ast .Name ):
383495 self ._store (target .id , value_id , walrus = walrus )
384496 else :
385497 self .visit (target )
386498
499+ def _visit_block (self , nodes : Sequence [ast .AST ], * , protected : bool = False ) -> None :
500+ """Visit statements that may run zero, one or several times as one block."""
501+ outer = self ._block
502+ self ._block = (* outer , next (self ._block_numbers ))
503+ self ._protected += protected
504+ for node in nodes :
505+ self .visit (node )
506+ self ._protected -= protected
507+ self ._block = outer
508+
509+ def _visit_loop (self , header : Sequence [ast .AST ], body : list [ast .stmt ]) -> None :
510+ """Visit a loop's repeated part, then count its reads against its bindings."""
511+ loop = _Loop ()
512+ self ._loops .append (loop )
513+ # ``break`` and ``continue`` can skip any rebinding in the body.
514+ self ._visit_block ([* header , * body ], protected = True )
515+ self ._loops .pop ()
516+ # The next pass reads what this pass bound, whatever the source order.
517+ for scope , name , binding in loop .bindings :
518+ if (scope , name ) in loop .reads :
519+ self ._escaped .add (binding .value_id )
520+
521+ def generic_visit (self , node : ast .AST ) -> None :
522+ """Visit children in order, giving every nested statement list its own block."""
523+ for field_name , value in ast .iter_fields (node ):
524+ if isinstance (value , list ) and value and isinstance (value [0 ], ast .stmt ):
525+ protected = (type (node ), field_name ) in _PROTECTED_BODIES
526+ self ._visit_block (value , protected = protected )
527+ elif isinstance (value , list ):
528+ for item in value :
529+ if isinstance (item , ast .AST ):
530+ self .visit (item )
531+ elif isinstance (value , ast .AST ):
532+ self .visit (value )
533+
387534 def _visit_scope (self , scope : _Scope , body : list [ast .stmt ] | list [ast .expr ]) -> None :
535+ # A ``try`` around a definition cannot see the locals of its body.
536+ outer_protected , self ._protected = self ._protected , 0
388537 self ._scopes .append (scope )
389- for statement in body :
390- self .visit (statement )
538+ self ._visit_block (body )
391539 self ._close_scope (self ._scopes .pop ())
540+ self ._protected = outer_protected
392541
393542 def _visit_defaults (self , arguments : ast .arguments ) -> None :
394543 for default in (* arguments .defaults , * arguments .kw_defaults ):
@@ -437,14 +586,13 @@ def _visit_comprehension(
437586 }
438587 deferred = isinstance (node , ast .GeneratorExp )
439588 self ._scopes .append (_Scope (targets , deferred = deferred , is_comprehension = True ))
589+ repeated : list [ast .AST ] = []
440590 for index , generator in enumerate (node .generators ):
441591 if index :
442- self .visit (generator .iter )
443- self .visit (generator .target )
444- for condition in generator .ifs :
445- self .visit (condition )
446- for result in results :
447- self .visit (result )
592+ repeated .append (generator .iter )
593+ repeated .append (generator .target )
594+ repeated .extend (generator .ifs )
595+ self ._visit_loop ([* repeated , * results ], [])
448596 self ._close_scope (self ._scopes .pop ())
449597
450598 def visit_ListComp (self , node : ast .ListComp ) -> None :
@@ -480,28 +628,38 @@ def visit_AugAssign(self, node: ast.AugAssign) -> None:
480628
481629 def visit_For (self , node : ast .For | ast .AsyncFor ) -> None :
482630 self .visit (node .iter )
483- self . visit ( node . target )
484- for statement in node .body + node .orelse :
485- self .visit ( statement )
631+ # The target is rebound on every pass, and not at all for an empty iterable.
632+ self . _visit_loop ([ node .target ], node .body )
633+ self ._visit_block ( node . orelse )
486634
487635 def visit_AsyncFor (self , node : ast .AsyncFor ) -> None :
488636 self .visit_For (node )
489637
638+ def visit_While (self , node : ast .While ) -> None :
639+ # The test runs again before every pass.
640+ self ._visit_loop ([node .test ], node .body )
641+ self ._visit_block (node .orelse )
642+
490643 def visit_Name (self , node : ast .Name ) -> None :
491644 node_id = id (node )
492645 if not isinstance (node .ctx , ast .Load ) and node_id not in self ._mutated_names :
493646 self ._store (node .id , None )
494647 return
495648 scope , deferred = self ._resolve (node .id )
496- binding = scope .open .get (node .id )
649+ bindings = scope .open .get (node .id , [] )
497650 if node_id in self ._env_arguments :
498- if binding is not None :
651+ for binding in bindings :
499652 binding .reached_child_process = True
500653 elif node_id not in self ._mutated_names :
501654 if deferred :
502655 scope .read_later .add (node .id )
503- if binding is not None :
656+ if scope .is_class :
657+ # Before the class body binds the name, the read falls through to globals.
658+ bindings = [* bindings , * self ._scopes [0 ].open .get (node .id , [])]
659+ for binding in bindings :
504660 binding .escaped = True
661+ for loop in self ._loops :
662+ loop .reads .add ((scope , node .id ))
505663
506664
507665def _analyze_python_environment_reads (
@@ -529,9 +687,14 @@ def _analyze_python_environment_reads(
529687
530688 aliases = python_ast .import_aliases
531689 lines = python_ast .lines
532- flow = _EnvironmentFlowVisitor (tree , aliases )
533- flow .visit (tree )
534- child_process_passthroughs = flow .passthrough_ids ()
690+ try :
691+ flow = _EnvironmentFlowVisitor (tree , aliases )
692+ flow .visit (tree )
693+ child_process_passthroughs = flow .passthrough_ids ()
694+ except RecursionError :
695+ # Too deep for the recursive flow pass: report every full read, as before
696+ # the child-process exemption existed, instead of losing the whole file.
697+ child_process_passthroughs = set ()
535698 findings : list [AnalyzerFinding ] = []
536699 emitted : set [int ] = set ()
537700 tag = [PatternCategory .DATA_EXFILTRATION .value ]
0 commit comments