4747from skillspector .python_ast import ParsedPythonFile , parse_python_source
4848from skillspector .python_tokens import PythonLiteralSpans
4949from skillspector .python_tokens import python_literal_spans as _python_literal_spans
50+ from skillspector .python_tokens import python_string_body as _python_string_body
5051from skillspector .security_reconstruction import validated_json_string_spans
5152from skillspector .state import AnalyzerNodeResponse , SkillspectorState
5253
139140# whitespace or a shell control character. A backslash-newline is a line
140141# continuation in both languages, so it does not end the word.
141142_PYTHON_SHELL_WORD_BREAK_RE = re .compile (r"(?<!\\)(?<!\\\r)[\s;|&()<>]" )
143+ # Whitespace and line continuations between Python tokens. Comments are
144+ # tokens of their own.
145+ _PYTHON_TOKEN_GAP_RE = re .compile (r"(?:\s|\\\r?\n)*+" )
146+ # Python operators that can still extend or rewrite a string value.
147+ _PYTHON_STRING_EXTENSION_OPERATORS = frozenset ({"+" , "%" , "." , "*" , "[" })
148+ # One brace opens an f-string replacement field; a doubled brace is literal.
149+ _PYTHON_REPLACEMENT_FIELD_RE = re .compile (r"\{\{|(\{)" )
142150_PRINTF_FORMAT_CONVERSION_RE = re .compile (r"%[-+ #0-9.*']*[A-Za-z%]" )
143151_SHELL_ROOT_TARGET_ESCAPE_RE = re .compile (
144152 r"\\(?:[/~*?]|x(?:2[fF]|7[eE]|2[aA]|3[fF])|"
166174 r"(?P<apostrophe>')t\b[A-Za-z \t,]*[.!?]" ,
167175 re .IGNORECASE ,
168176)
177+ # An English possessive or contraction inside one prose word, such as
178+ # ``reference's`` or ``don't``. The suffix must end the word, so a quote that
179+ # closes a shell word, as in ``r'm'``, never matches.
180+ _MARKDOWN_WORD_APOSTROPHE_RE = re .compile (
181+ r"(?<![\w'/.~-])[A-Za-z]++(?P<apostrophe>')(?:s|t|d|m|re|ve|ll)(?![\w'/])" ,
182+ re .IGNORECASE ,
183+ )
184+ # Shell syntax that opens a quote, an expansion, or a line continuation.
185+ _MARKDOWN_PROSE_SHELL_SYNTAX_RE = re .compile (r"[$`\\\"]" )
169186_ROOT_GLOB_DOCUMENTATION_LINE_RE = re .compile (
170187 r"[ \t]*(?:(?:[-*+]|#{1,6})[ \t]+)?"
171188 r"(?:(?:(?:documentation|note|example)[ \t]*:[ \t]*)"
@@ -2235,12 +2252,23 @@ def _has_shell_command_word_exhaustion(
22352252 ):
22362253 comment = python_source .comment (start ) if python_source is not None else None
22372254 if comment is None :
2238- return True
2255+ # Host code after a Python string's closing quote is not shell
2256+ # text, so a command in the string takes operands only from
2257+ # the rest of the string's value, including literals
2258+ # concatenated with it. A value that code can still extend,
2259+ # for example with ``%`` or ``.format``, stays partial.
2260+ value = (
2261+ python_source .string_value (start , parsed .end )
2262+ if python_source is not None
2263+ else None
2264+ )
2265+ if value is None or _has_runtime_command_operand_exhaustion (* value , check_runtime ):
2266+ return True
2267+ continue
22392268 # A Python comment is prose that no shell receives, so a command
22402269 # named in it can take operands only from the same comment, never
22412270 # from later host code. Recheck that comment as standalone shell
2242- # text. Strings are not confined: concatenation can supply their
2243- # runtime operands.
2271+ # text.
22442272 if comment not in complete_comments :
22452273 if _has_shell_command_word_exhaustion (
22462274 content [comment [0 ] : comment [1 ]],
@@ -2702,6 +2730,22 @@ def _perl_literal_print_shell_text(
27022730 return "" .join (output )
27032731
27042732
2733+ def _python_literal_operands (content : str , token : tuple [int , int ], start : int , end : int ) -> str :
2734+ """Return string literal body text from ``start`` that a shell reads as operands.
2735+
2736+ Code supplies an f-string or t-string replacement field's text at runtime,
2737+ and that text can hold the recursive and force options. Each field also
2738+ counts as them, so a root target beside a field stays undecided.
2739+ """
2740+ text = content [start :end ]
2741+ body_start = _python_string_body (content , * token )[0 ]
2742+ if not {"f" , "t" } & set (content [token [0 ] : body_start ].lower ()):
2743+ return text
2744+ return _PYTHON_REPLACEMENT_FIELD_RE .sub (
2745+ lambda field : field [0 ] if field [1 ] is None else " -rf {" , text
2746+ )
2747+
2748+
27052749class _PythonSourceOwnership :
27062750 """Lazily proven Python string and comment ownership for one source text.
27072751
@@ -2752,6 +2796,82 @@ def comment(self, position: int) -> tuple[int, int] | None:
27522796 return None
27532797 return owner
27542798
2799+ def string_value (self , start : int , end : int ) -> tuple [str , int , int ] | None :
2800+ """Return the proven string value that holds a shell word, and its span.
2801+
2802+ A string literal's quotes are Python syntax: the shell receives only
2803+ its body, so a word ends at the closing quote, and a word starting at
2804+ that quote is not shell text at all. Implicit or ``+`` concatenation
2805+ continues the value with later literal bodies. Return ``None`` for a
2806+ word outside a string literal, or for a value that a format operator,
2807+ method call, subscript, repetition or non-literal operand can still
2808+ extend, also after a closing parenthesis.
2809+ """
2810+ spans = self ._owned_spans ()
2811+ owner = self ._owner (start )
2812+ content = self ._content
2813+ if spans is None or owner is None or content [owner [0 ]] == "#" :
2814+ return None
2815+ body_start , body_end = _python_string_body (content , * owner )
2816+ if start >= body_end :
2817+ return "" , 0 , 0
2818+ word_start = max (start , body_start )
2819+ word_end = max (min (end , body_end ), word_start )
2820+ # Keep the character before the word and its escaping backslashes:
2821+ # they decide whether a quote in the body wraps the command.
2822+ origin = word_start - 1 if word_start > body_start else word_start
2823+ while origin > body_start and content [origin - 1 ] == "\\ " :
2824+ origin -= 1
2825+ # The operand tokenizer reads at most this much after the word.
2826+ limit = word_end + _ROOT_GLOB_COMMAND_CHARS + 1
2827+ pieces = [
2828+ content [origin :word_end ],
2829+ _python_literal_operands (content , owner , word_end , min (body_end , limit )),
2830+ ]
2831+ remaining = limit - body_end
2832+ starts , ends = spans
2833+ index = bisect_right (starts , start ) - 1
2834+
2835+ def next_token (position : int ) -> int :
2836+ # Skip whitespace, line continuations and comment tokens.
2837+ nonlocal index
2838+ while True :
2839+ gap = _PYTHON_TOKEN_GAP_RE .match (content , position )
2840+ position = position if gap is None else gap .end ()
2841+ if (
2842+ index + 1 == len (starts )
2843+ or starts [index + 1 ] != position
2844+ or content [position ] != "#"
2845+ ):
2846+ return position
2847+ index += 1
2848+ position = ends [index ]
2849+
2850+ while remaining > 0 :
2851+ self ._check_runtime ()
2852+ cursor = next_token (ends [index ])
2853+ operator = content [cursor : cursor + 1 ]
2854+ if operator == "+" :
2855+ cursor = next_token (cursor + 1 )
2856+ if index + 1 < len (starts ) and starts [index + 1 ] == cursor :
2857+ index += 1
2858+ literal = (starts [index ], ends [index ])
2859+ literal_start , literal_end = _python_string_body (content , * literal )
2860+ pieces .append (
2861+ _python_literal_operands (content , literal , literal_start , literal_end )
2862+ )
2863+ remaining -= literal_end - literal_start
2864+ continue
2865+ # A parenthesized value, as in ``("$TOOL ") + "-rf /"``, can
2866+ # still be extended after its closing parenthesis.
2867+ while operator == ")" :
2868+ cursor = next_token (cursor + 1 )
2869+ operator = content [cursor : cursor + 1 ]
2870+ if operator in _PYTHON_STRING_EXTENSION_OPERATORS :
2871+ return None
2872+ break
2873+ return "" .join (pieces ), word_start - origin , word_end - origin
2874+
27552875 def word_end (self , start : int ) -> int :
27562876 """Return the end of the host region a shell word at ``start`` can own."""
27572877 spans = self ._owned_spans ()
@@ -4800,13 +4920,19 @@ def _markdown_table_delimiter_columns(
48004920
48014921
48024922def _markdown_shell_text (
4803- content : str , check_runtime : Callable [[], None ], * , complete_context : bool = True
4923+ content : str ,
4924+ check_runtime : Callable [[], None ],
4925+ * ,
4926+ complete_context : bool = True ,
4927+ word_apostrophes : bool = False ,
48044928) -> str :
48054929 """Mask Markdown delimiters while retaining code and exact source offsets.
48064930
48074931 Inline code delimiters are not legacy shell substitutions. Fenced and
48084932 indented code stays literal; longer inline delimiters preserve backticks
48094933 inside their bodies. Pair equal-length runs in linear time.
4934+ ``word_apostrophes`` selects the prose reading, which also masks the
4935+ possessives and contractions of plain prose lines.
48104936 """
48114937 output = list (content )
48124938 prose_apostrophes : set [int ] = set ()
@@ -5034,6 +5160,33 @@ def mask_inline_delimiters() -> None:
50345160 # shell syntax. Fences, indented code, HTML, tables, and
50355161 # pending multiline inline spans never reach this rule.
50365162 prose_apostrophes .add (offset + prefix + prose .start ("apostrophe" ))
5163+ elif (
5164+ word_apostrophes
5165+ and complete_context
5166+ and not runs
5167+ and not heading
5168+ and "'" in leading
5169+ and _MARKDOWN_PROSE_SHELL_SYNTAX_RE .search (leading ) is None
5170+ and (
5171+ line_index == 0
5172+ or (
5173+ lines [line_index - 1 ].endswith ("\n " )
5174+ and not lines [line_index - 1 ].rstrip ("\r \n " ).endswith ("\\ " )
5175+ )
5176+ )
5177+ ):
5178+ # In the prose reading, a prose line owns every
5179+ # possessive and contraction in it. Every quote on the
5180+ # line must be one, the line has no expansion or line
5181+ # continuation, and it starts a new shell line, so no
5182+ # runtime-selected command can take later text as
5183+ # operands through it. The same block exclusions apply.
5184+ apostrophes : list [int ] = []
5185+ for word in _MARKDOWN_WORD_APOSTROPHE_RE .finditer (leading ):
5186+ check_runtime ()
5187+ apostrophes .append (offset + prefix + word .start ("apostrophe" ))
5188+ if len (apostrophes ) == leading .count ("'" ):
5189+ prose_apostrophes .update (apostrophes )
50375190 for match in backtick_runs .finditer (line ):
50385191 check_runtime ()
50395192 runs .append ((offset + match .start (), offset + match .end ()))
@@ -5094,6 +5247,29 @@ def has_bounded_parse_exhaustion(
50945247 complete_context : bool = True ,
50955248) -> bool :
50965249 """Return whether a destructive rm command exceeded the parser's span contract."""
5250+ exhausted = _has_bounded_parse_exhaustion (
5251+ content , check_runtime , file_type = file_type , complete_context = complete_context
5252+ )
5253+ if not exhausted or file_type != "markdown" or not complete_context :
5254+ return exhausted
5255+ # A possessive such as "reference's" in prose is punctuation, yet the
5256+ # lexical reading can leave it open as a shell quote to the end of the
5257+ # file. Retry with the prose reading. Masking an apostrophe can also turn
5258+ # a later closing quote into an opener, so the prose reading may only
5259+ # resolve a limit of the lexical one, never introduce one.
5260+ return _has_bounded_parse_exhaustion (
5261+ content , check_runtime , file_type = file_type , word_apostrophes = True
5262+ )
5263+
5264+
5265+ def _has_bounded_parse_exhaustion (
5266+ content : str ,
5267+ check_runtime : Callable [[], None ],
5268+ * ,
5269+ file_type : str ,
5270+ complete_context : bool = True ,
5271+ word_apostrophes : bool = False ,
5272+ ) -> bool :
50975273 # PowerShell source can invoke another shell with an executable command
50985274 # string. Apply the same bounded checks to those strings; benign PowerShell
50995275 # -replace values have their own narrowly proven expression context.
@@ -5112,7 +5288,12 @@ def has_bounded_parse_exhaustion(
51125288 json_strings = validated_json_string_spans (content , check_runtime )
51135289 structural_quote_closers = {end - 1 for _ , end in json_strings }
51145290 structural_quote_openers = {start for start , _ in json_strings }
5115- content = _markdown_shell_text (content , check_runtime , complete_context = complete_context )
5291+ content = _markdown_shell_text (
5292+ content ,
5293+ check_runtime ,
5294+ complete_context = complete_context ,
5295+ word_apostrophes = word_apostrophes ,
5296+ )
51165297 # Perl ``eval BLOCK`` is recognized from its clause-initial ``eval`` and the
51175298 # adjacent brace alone, so unlike quote ownership it needs no full context.
51185299 if _has_shell_command_word_exhaustion (
0 commit comments