Skip to content

Commit cb458ff

Browse files
authored
Merge pull request #627 from NVIDIA/codex/fix-prose-shell-parser-context
fix(analyzer): avoid false shell parse limits in Markdown
2 parents a50b9c9 + d475b56 commit cb458ff

2 files changed

Lines changed: 391 additions & 2 deletions

File tree

‎src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,12 @@
115115
_SHELL_REDIRECTION_PREFIX_RE = re.compile(r"(?:[0-9]+)?(?:&>>?|<>|>>?|<<-?|>&|<&|[<>])")
116116
_RECURSIVE_OPTION_SOURCE_RE = re.compile(r"-(?:[A-Za-z]*[rR]|-recursive)")
117117
_FORCE_OPTION_SOURCE_RE = re.compile(r"-(?:[A-Za-z]*f|-force)")
118+
_MARKDOWN_CONTRACTION_PROSE_RE = re.compile(
119+
r"(?:The|This|That|These|Those|It|They|We|You|I|A|An)[ \t]+[A-Za-z \t,]*\b"
120+
r"(?:doesn|isn|aren|wasn|weren|don|didn|hasn|haven|hadn|can|couldn|shouldn|wouldn|won|mustn)"
121+
r"(?P<apostrophe>')t\b[A-Za-z \t,]*[.!?]",
122+
re.IGNORECASE,
123+
)
118124
_ROOT_GLOB_DOCUMENTATION_LINE_RE = re.compile(
119125
r"[ \t]*(?:(?:[-*+]|#{1,6})[ \t]+)?"
120126
r"(?:(?:(?:documentation|note|example)[ \t]*:[ \t]*)"
@@ -1932,12 +1938,21 @@ def _has_shell_command_word_exhaustion(
19321938
if parsed.limited:
19331939
return True
19341940
if parsed.dynamic and may_have_destructive_outer_operands:
1935-
tokens, _, exhausted = _bounded_shell_tokens(
1941+
tokens, command_end, exhausted = _bounded_shell_tokens(
19361942
content,
19371943
start,
19381944
parsed.end,
19391945
check_runtime=check_runtime,
19401946
)
1947+
if exhausted and command_end - start < _ROOT_GLOB_COMMAND_CHARS:
1948+
# Earlier prose (for example "row-first") cannot supply this
1949+
# command's operands. Refine only this candidate's exhausted
1950+
# span, including nested commands inside the executable word.
1951+
# Hitting the span bound always remains partial, including when
1952+
# real operands occur beyond that bound. This rescan is bounded
1953+
# by the same constant as the tokenizer.
1954+
check_runtime()
1955+
exhausted = _may_have_destructive_outer_operands(content[start:command_end])
19411956
if (
19421957
exhausted
19431958
or _has_unsupported_brace_expansion(tokens)
@@ -3279,6 +3294,7 @@ def _markdown_shell_text(
32793294
inside their bodies. Pair equal-length runs in linear time.
32803295
"""
32813296
output = list(content)
3297+
prose_apostrophes: set[int] = set()
32823298
runs: list[tuple[int, int]] = []
32833299
list_marker = re.compile(r"(?:[-+*]|[0-9]{1,9}[.)])(?=[ \t])")
32843300
backtick_runs = re.compile(r"`+")
@@ -3489,6 +3505,20 @@ def mask_inline_delimiters() -> None:
34893505
if not cells:
34903506
table_columns = None
34913507
elif not separator and not empty_list_item:
3508+
if (
3509+
complete_context
3510+
and not runs
3511+
and not heading
3512+
and len(leading) <= 512
3513+
and (prose := _MARKDOWN_CONTRACTION_PROSE_RE.fullmatch(leading))
3514+
):
3515+
# A whole plain-language sentence, including a parsed
3516+
# list-item body, owns its contraction;
3517+
# it is not an unclosed shell quote spanning later prose.
3518+
# The restrictive grammar excludes code delimiters and
3519+
# shell syntax. Fences, indented code, HTML, tables, and
3520+
# pending multiline inline spans never reach this rule.
3521+
prose_apostrophes.add(offset + prefix + prose.start("apostrophe"))
34923522
for match in backtick_runs.finditer(line):
34933523
check_runtime()
34943524
runs.append((offset + match.start(), offset + match.end()))
@@ -3505,7 +3535,40 @@ def mask_inline_delimiters() -> None:
35053535
paragraph_list_indent = 0
35063536
offset += len(line)
35073537
mask_inline_delimiters()
3508-
return "".join(output)
3538+
projected = "".join(output)
3539+
if prose_apostrophes:
3540+
# A prose-looking line can still be inside an earlier shell word, such
3541+
# as a multiline bash -c string. Reuse the shell parser to preserve that
3542+
# ownership before masking any apostrophe. Unresolved words retain all
3543+
# remaining bytes; this pass never grants ownership past a parse limit.
3544+
cursor = 0
3545+
next_runtime_check = 0
3546+
last_apostrophe = max(prose_apostrophes)
3547+
parameter_ends: dict[int, _ParameterExpansionEnd] = {}
3548+
substitution_ends: dict[int, int | None] = {}
3549+
backtick_ends: dict[int, int | None] = {}
3550+
while cursor <= last_apostrophe:
3551+
if cursor >= next_runtime_check:
3552+
check_runtime()
3553+
next_runtime_check = cursor + 4096
3554+
if cursor in prose_apostrophes:
3555+
output[cursor] = " "
3556+
elif projected[cursor] in "'\"`$\\":
3557+
parsed = _parse_shell_command_word(
3558+
projected,
3559+
cursor,
3560+
parameter_ends,
3561+
substitution_ends,
3562+
backtick_ends,
3563+
check_runtime=check_runtime,
3564+
)
3565+
if parsed is None or parsed.limited:
3566+
break
3567+
cursor = max(cursor + 1, parsed.end)
3568+
continue
3569+
cursor += 1
3570+
return "".join(output)
3571+
return projected
35093572

35103573

35113574
def has_bounded_parse_exhaustion(

0 commit comments

Comments
 (0)