4646from skillspector .python_ast import ParsedPythonFile , parse_python_source
4747from skillspector .python_tokens import PythonLiteralSpans
4848from skillspector .python_tokens import python_literal_spans as _python_literal_spans
49+ from skillspector .python_tokens import python_string_body as _python_string_body
4950from skillspector .security_reconstruction import validated_json_string_spans
5051from skillspector .state import AnalyzerNodeResponse , SkillspectorState
5152
136137# whitespace or a shell control character. A backslash-newline is a line
137138# continuation in both languages, so it does not end the word.
138139_PYTHON_SHELL_WORD_BREAK_RE = re .compile (r"(?<!\\)(?<!\\\r)[\s;|&()<>]" )
140+ # Whitespace and line continuations between Python tokens. Comments are
141+ # tokens of their own.
142+ _PYTHON_TOKEN_GAP_RE = re .compile (r"(?:\s|\\\r?\n)*+" )
143+ # Python operators that can still extend or rewrite a string value.
144+ _PYTHON_STRING_EXTENSION_OPERATORS = frozenset ({"+" , "%" , "." , "*" , "[" })
145+ # One brace opens an f-string replacement field; a doubled brace is literal.
146+ _PYTHON_REPLACEMENT_FIELD_RE = re .compile (r"\{\{|(\{)" )
139147_PRINTF_FORMAT_CONVERSION_RE = re .compile (r"%[-+ #0-9.*']*[A-Za-z%]" )
140148_SHELL_ROOT_TARGET_ESCAPE_RE = re .compile (
141149 r"\\(?:[/~*?]|x(?:2[fF]|7[eE]|2[aA]|3[fF])|"
163171 r"(?P<apostrophe>')t\b[A-Za-z \t,]*[.!?]" ,
164172 re .IGNORECASE ,
165173)
174+ # An English possessive or contraction inside one prose word, such as
175+ # ``reference's`` or ``don't``. The suffix must end the word, so a quote that
176+ # closes a shell word, as in ``r'm'``, never matches.
177+ _MARKDOWN_WORD_APOSTROPHE_RE = re .compile (
178+ r"(?<![\w'/.~-])[A-Za-z]++(?P<apostrophe>')(?:s|t|d|m|re|ve|ll)(?![\w'/])" ,
179+ re .IGNORECASE ,
180+ )
181+ # Shell syntax that opens a quote, an expansion, or a line continuation.
182+ _MARKDOWN_PROSE_SHELL_SYNTAX_RE = re .compile (r"[$`\\\"]" )
166183_ROOT_GLOB_DOCUMENTATION_LINE_RE = re .compile (
167184 r"[ \t]*(?:(?:[-*+]|#{1,6})[ \t]+)?"
168185 r"(?:(?:(?:documentation|note|example)[ \t]*:[ \t]*)"
@@ -2149,12 +2166,23 @@ def _has_shell_command_word_exhaustion(
21492166 ):
21502167 comment = python_source .comment (start ) if python_source is not None else None
21512168 if comment is None :
2152- return True
2169+ # Host code after a Python string's closing quote is not shell
2170+ # text, so a command in the string takes operands only from
2171+ # the rest of the string's value, including literals
2172+ # concatenated with it. A value that code can still extend,
2173+ # for example with ``%`` or ``.format``, stays partial.
2174+ value = (
2175+ python_source .string_value (start , parsed .end )
2176+ if python_source is not None
2177+ else None
2178+ )
2179+ if value is None or _has_runtime_command_operand_exhaustion (* value , check_runtime ):
2180+ return True
2181+ continue
21532182 # A Python comment is prose that no shell receives, so a command
21542183 # named in it can take operands only from the same comment, never
21552184 # from later host code. Recheck that comment as standalone shell
2156- # text. Strings are not confined: concatenation can supply their
2157- # runtime operands.
2185+ # text.
21582186 if comment not in complete_comments :
21592187 if _has_shell_command_word_exhaustion (
21602188 content [comment [0 ] : comment [1 ]],
@@ -2616,6 +2644,22 @@ def _perl_literal_print_shell_text(
26162644 return "" .join (output )
26172645
26182646
2647+ def _python_literal_operands (content : str , token : tuple [int , int ], start : int , end : int ) -> str :
2648+ """Return string literal body text from ``start`` that a shell reads as operands.
2649+
2650+ Code supplies an f-string or t-string replacement field's text at runtime,
2651+ and that text can hold the recursive and force options. Each field also
2652+ counts as them, so a root target beside a field stays undecided.
2653+ """
2654+ text = content [start :end ]
2655+ body_start = _python_string_body (content , * token )[0 ]
2656+ if not {"f" , "t" } & set (content [token [0 ] : body_start ].lower ()):
2657+ return text
2658+ return _PYTHON_REPLACEMENT_FIELD_RE .sub (
2659+ lambda field : field [0 ] if field [1 ] is None else " -rf {" , text
2660+ )
2661+
2662+
26192663class _PythonSourceOwnership :
26202664 """Lazily proven Python string and comment ownership for one source text.
26212665
@@ -2666,6 +2710,82 @@ def comment(self, position: int) -> tuple[int, int] | None:
26662710 return None
26672711 return owner
26682712
2713+ def string_value (self , start : int , end : int ) -> tuple [str , int , int ] | None :
2714+ """Return the proven string value that holds a shell word, and its span.
2715+
2716+ A string literal's quotes are Python syntax: the shell receives only
2717+ its body, so a word ends at the closing quote, and a word starting at
2718+ that quote is not shell text at all. Implicit or ``+`` concatenation
2719+ continues the value with later literal bodies. Return ``None`` for a
2720+ word outside a string literal, or for a value that a format operator,
2721+ method call, subscript, repetition or non-literal operand can still
2722+ extend, also after a closing parenthesis.
2723+ """
2724+ spans = self ._owned_spans ()
2725+ owner = self ._owner (start )
2726+ content = self ._content
2727+ if spans is None or owner is None or content [owner [0 ]] == "#" :
2728+ return None
2729+ body_start , body_end = _python_string_body (content , * owner )
2730+ if start >= body_end :
2731+ return "" , 0 , 0
2732+ word_start = max (start , body_start )
2733+ word_end = max (min (end , body_end ), word_start )
2734+ # Keep the character before the word and its escaping backslashes:
2735+ # they decide whether a quote in the body wraps the command.
2736+ origin = word_start - 1 if word_start > body_start else word_start
2737+ while origin > body_start and content [origin - 1 ] == "\\ " :
2738+ origin -= 1
2739+ # The operand tokenizer reads at most this much after the word.
2740+ limit = word_end + _ROOT_GLOB_COMMAND_CHARS + 1
2741+ pieces = [
2742+ content [origin :word_end ],
2743+ _python_literal_operands (content , owner , word_end , min (body_end , limit )),
2744+ ]
2745+ remaining = limit - body_end
2746+ starts , ends = spans
2747+ index = bisect_right (starts , start ) - 1
2748+
2749+ def next_token (position : int ) -> int :
2750+ # Skip whitespace, line continuations and comment tokens.
2751+ nonlocal index
2752+ while True :
2753+ gap = _PYTHON_TOKEN_GAP_RE .match (content , position )
2754+ position = position if gap is None else gap .end ()
2755+ if (
2756+ index + 1 == len (starts )
2757+ or starts [index + 1 ] != position
2758+ or content [position ] != "#"
2759+ ):
2760+ return position
2761+ index += 1
2762+ position = ends [index ]
2763+
2764+ while remaining > 0 :
2765+ self ._check_runtime ()
2766+ cursor = next_token (ends [index ])
2767+ operator = content [cursor : cursor + 1 ]
2768+ if operator == "+" :
2769+ cursor = next_token (cursor + 1 )
2770+ if index + 1 < len (starts ) and starts [index + 1 ] == cursor :
2771+ index += 1
2772+ literal = (starts [index ], ends [index ])
2773+ literal_start , literal_end = _python_string_body (content , * literal )
2774+ pieces .append (
2775+ _python_literal_operands (content , literal , literal_start , literal_end )
2776+ )
2777+ remaining -= literal_end - literal_start
2778+ continue
2779+ # A parenthesized value, as in ``("$TOOL ") + "-rf /"``, can
2780+ # still be extended after its closing parenthesis.
2781+ while operator == ")" :
2782+ cursor = next_token (cursor + 1 )
2783+ operator = content [cursor : cursor + 1 ]
2784+ if operator in _PYTHON_STRING_EXTENSION_OPERATORS :
2785+ return None
2786+ break
2787+ return "" .join (pieces ), word_start - origin , word_end - origin
2788+
26692789 def word_end (self , start : int ) -> int :
26702790 """Return the end of the host region a shell word at ``start`` can own."""
26712791 spans = self ._owned_spans ()
@@ -4663,13 +4783,19 @@ def _markdown_table_delimiter_columns(
46634783
46644784
46654785def _markdown_shell_text (
4666- content : str , check_runtime : Callable [[], None ], * , complete_context : bool = True
4786+ content : str ,
4787+ check_runtime : Callable [[], None ],
4788+ * ,
4789+ complete_context : bool = True ,
4790+ word_apostrophes : bool = False ,
46674791) -> str :
46684792 """Mask Markdown delimiters while retaining code and exact source offsets.
46694793
46704794 Inline code delimiters are not legacy shell substitutions. Fenced and
46714795 indented code stays literal; longer inline delimiters preserve backticks
46724796 inside their bodies. Pair equal-length runs in linear time.
4797+ ``word_apostrophes`` selects the prose reading, which also masks the
4798+ possessives and contractions of plain prose lines.
46734799 """
46744800 output = list (content )
46754801 prose_apostrophes : set [int ] = set ()
@@ -4897,6 +5023,33 @@ def mask_inline_delimiters() -> None:
48975023 # shell syntax. Fences, indented code, HTML, tables, and
48985024 # pending multiline inline spans never reach this rule.
48995025 prose_apostrophes .add (offset + prefix + prose .start ("apostrophe" ))
5026+ elif (
5027+ word_apostrophes
5028+ and complete_context
5029+ and not runs
5030+ and not heading
5031+ and "'" in leading
5032+ and _MARKDOWN_PROSE_SHELL_SYNTAX_RE .search (leading ) is None
5033+ and (
5034+ line_index == 0
5035+ or (
5036+ lines [line_index - 1 ].endswith ("\n " )
5037+ and not lines [line_index - 1 ].rstrip ("\r \n " ).endswith ("\\ " )
5038+ )
5039+ )
5040+ ):
5041+ # In the prose reading, a prose line owns every
5042+ # possessive and contraction in it. Every quote on the
5043+ # line must be one, the line has no expansion or line
5044+ # continuation, and it starts a new shell line, so no
5045+ # runtime-selected command can take later text as
5046+ # operands through it. The same block exclusions apply.
5047+ apostrophes : list [int ] = []
5048+ for word in _MARKDOWN_WORD_APOSTROPHE_RE .finditer (leading ):
5049+ check_runtime ()
5050+ apostrophes .append (offset + prefix + word .start ("apostrophe" ))
5051+ if len (apostrophes ) == leading .count ("'" ):
5052+ prose_apostrophes .update (apostrophes )
49005053 for match in backtick_runs .finditer (line ):
49015054 check_runtime ()
49025055 runs .append ((offset + match .start (), offset + match .end ()))
@@ -4957,6 +5110,29 @@ def has_bounded_parse_exhaustion(
49575110 complete_context : bool = True ,
49585111) -> bool :
49595112 """Return whether a destructive rm command exceeded the parser's span contract."""
5113+ exhausted = _has_bounded_parse_exhaustion (
5114+ content , check_runtime , file_type = file_type , complete_context = complete_context
5115+ )
5116+ if not exhausted or file_type != "markdown" or not complete_context :
5117+ return exhausted
5118+ # A possessive such as "reference's" in prose is punctuation, yet the
5119+ # lexical reading can leave it open as a shell quote to the end of the
5120+ # file. Retry with the prose reading. Masking an apostrophe can also turn
5121+ # a later closing quote into an opener, so the prose reading may only
5122+ # resolve a limit of the lexical one, never introduce one.
5123+ return _has_bounded_parse_exhaustion (
5124+ content , check_runtime , file_type = file_type , word_apostrophes = True
5125+ )
5126+
5127+
5128+ def _has_bounded_parse_exhaustion (
5129+ content : str ,
5130+ check_runtime : Callable [[], None ],
5131+ * ,
5132+ file_type : str ,
5133+ complete_context : bool = True ,
5134+ word_apostrophes : bool = False ,
5135+ ) -> bool :
49605136 # PowerShell source can invoke another shell with an executable command
49615137 # string. Apply the same bounded checks to those strings; benign PowerShell
49625138 # -replace values have their own narrowly proven expression context.
@@ -4975,7 +5151,12 @@ def has_bounded_parse_exhaustion(
49755151 json_strings = validated_json_string_spans (content , check_runtime )
49765152 structural_quote_closers = {end - 1 for _ , end in json_strings }
49775153 structural_quote_openers = {start for start , _ in json_strings }
4978- content = _markdown_shell_text (content , check_runtime , complete_context = complete_context )
5154+ content = _markdown_shell_text (
5155+ content ,
5156+ check_runtime ,
5157+ complete_context = complete_context ,
5158+ word_apostrophes = word_apostrophes ,
5159+ )
49795160 # Perl ``eval BLOCK`` is recognized from its clause-initial ``eval`` and the
49805161 # adjacent brace alone, so unlike quote ownership it needs no full context.
49815162 if _has_shell_command_word_exhaustion (
0 commit comments