Skip to content

Commit a2009a6

Browse files
committed
fix(findings): deduplicate cross-analyzer findings before scoring
When multiple analyzers detect the same pattern (e.g. subprocess.run with shell=True) in multiple files of a skill, each occurrence was treated as a completely independent finding. A skill calling subprocess in 5 modules got 5 findings, each contributing full points — even though it represents the same design choice observed in different places. This commit adds a two-pass deduplication step in the report node: 1. Same-file dedup: identical (rule_id, file, matched_text) collapsed to the highest-confidence instance 2. Cross-file dedup: identical (rule_id, matched_text) across different files collapsed to one representative finding Findings without matched_text are never cross-file deduplicated since they lack a reliable identity signal. Output is sorted by severity then file for stable, predictable reports. Fixes NVIDIA#137 Signed-off-by: Mohammed Imran Khan <[email protected]>
1 parent d56ef81 commit a2009a6

3 files changed

Lines changed: 331 additions & 4 deletions

File tree

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Cross-analyzer finding deduplication.
17+
18+
Merges findings that represent the same conceptual issue observed multiple
19+
times — either within the same file or across files with identical patterns.
20+
21+
Deduplication strategy:
22+
1. Same-file dedup: Same rule_id + same file + same matched_text
23+
→ keep highest confidence instance
24+
2. Cross-file consolidation: Same rule_id + same matched_text across files
25+
→ keep highest confidence instance
26+
"""
27+
28+
from __future__ import annotations
29+
30+
from skillspector.logging_config import get_logger
31+
from skillspector.models import Finding
32+
33+
logger = get_logger(__name__)
34+
35+
36+
def _same_file_key(finding: Finding) -> tuple[str, str, str]:
37+
"""Build a deduplication key for same-file matches."""
38+
matched = (finding.matched_text or "").strip()[:100]
39+
return (finding.rule_id, finding.file, matched)
40+
41+
42+
def _cross_file_key(finding: Finding) -> tuple[str, str]:
43+
"""Build a cross-file deduplication key from rule_id and normalized matched_text."""
44+
matched = (finding.matched_text or "").strip()[:100]
45+
return (finding.rule_id, matched)
46+
47+
48+
def deduplicate(findings: list[Finding]) -> list[Finding]:
49+
"""Deduplicate a list of findings, returning a reduced list.
50+
51+
Two-pass deduplication:
52+
1. Same-file: identical (rule_id, file, matched_text) → keep highest confidence
53+
2. Cross-file: identical (rule_id, matched_text) across different files
54+
→ keep highest confidence representative
55+
56+
Findings without matched_text are never cross-file deduplicated (they lack
57+
a reliable identity signal).
58+
"""
59+
if not findings:
60+
return []
61+
62+
original_count = len(findings)
63+
64+
# Pass 1: Same-file deduplication
65+
same_file_best: dict[tuple[str, str, str], Finding] = {}
66+
for f in findings:
67+
key = _same_file_key(f)
68+
existing = same_file_best.get(key)
69+
if existing is None or f.confidence > existing.confidence:
70+
same_file_best[key] = f
71+
72+
after_same_file = list(same_file_best.values())
73+
74+
# Pass 2: Cross-file deduplication (only for findings WITH matched_text)
75+
cross_file_best: dict[tuple[str, str], Finding] = {}
76+
no_text_findings: list[Finding] = []
77+
78+
for f in after_same_file:
79+
matched = (f.matched_text or "").strip()
80+
if not matched:
81+
no_text_findings.append(f)
82+
continue
83+
key = _cross_file_key(f)
84+
existing = cross_file_best.get(key)
85+
if existing is None or f.confidence > existing.confidence:
86+
cross_file_best[key] = f
87+
88+
deduplicated = list(cross_file_best.values()) + no_text_findings
89+
90+
removed = original_count - len(deduplicated)
91+
if removed > 0:
92+
logger.info(
93+
"Deduplication: %d → %d findings (%d duplicates removed)",
94+
original_count,
95+
len(deduplicated),
96+
removed,
97+
)
98+
99+
severity_order = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}
100+
deduplicated.sort(key=lambda f: (severity_order.get(f.severity, 4), f.file, f.start_line))
101+
102+
return deduplicated

‎src/skillspector/nodes/report.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
from skillspector.llm_utils import is_llm_available
3535
from skillspector.logging_config import get_logger
3636
from skillspector.models import Finding
37+
from skillspector.nodes.deduplicate import deduplicate
3738
from skillspector.sarif_models import (
3839
SARIF_SCHEMA_URI,
3940
SarifArtifactLocation,
@@ -376,12 +377,12 @@ def _format_markdown(
376377

377378
def report(state: SkillspectorState) -> dict[str, object]:
378379
"""Generate SARIF, compute risk score, and set report_body from output_format."""
379-
findings = state.get("filtered_findings", state.get("findings", []))
380+
raw_findings = state.get("filtered_findings", state.get("findings", []))
380381
# When use_llm is False, meta_analyzer is skipped; ensure final state has filtered_findings
381382
if "filtered_findings" not in state:
382-
filtered_findings = state.get("findings", [])
383-
else:
384-
filtered_findings = findings
383+
raw_findings = state.get("findings", [])
384+
findings = deduplicate(raw_findings)
385+
filtered_findings = findings
385386
component_metadata = state.get("component_metadata") or []
386387
has_executable_scripts = state.get("has_executable_scripts", False)
387388
manifest = state.get("manifest") or {}

‎tests/nodes/test_deduplicate.py‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Tests for cross-analyzer finding deduplication."""
17+
18+
from __future__ import annotations
19+
20+
from skillspector.models import Finding
21+
from skillspector.nodes.deduplicate import deduplicate
22+
23+
24+
def _finding(
25+
rule_id: str = "TM1",
26+
file: str = "tool.py",
27+
matched_text: str = "subprocess.run(cmd, shell=True)",
28+
confidence: float = 0.8,
29+
severity: str = "HIGH",
30+
start_line: int = 1,
31+
) -> Finding:
32+
return Finding(
33+
rule_id=rule_id,
34+
message=f"Test finding {rule_id}",
35+
severity=severity,
36+
confidence=confidence,
37+
file=file,
38+
start_line=start_line,
39+
matched_text=matched_text,
40+
)
41+
42+
43+
class TestSameFileDedup:
44+
"""Same rule_id + same file + same matched_text → keep highest confidence."""
45+
46+
def test_exact_duplicates_reduced_to_one(self) -> None:
47+
"""Two identical findings in same file → one output."""
48+
findings = [
49+
_finding(file="a.py", start_line=1),
50+
_finding(file="a.py", start_line=5),
51+
]
52+
result = deduplicate(findings)
53+
assert len(result) == 1
54+
55+
def test_keeps_highest_confidence(self) -> None:
56+
"""When duplicates exist, the highest confidence one is kept."""
57+
findings = [
58+
_finding(file="a.py", confidence=0.6),
59+
_finding(file="a.py", confidence=0.9),
60+
_finding(file="a.py", confidence=0.3),
61+
]
62+
result = deduplicate(findings)
63+
assert len(result) == 1
64+
assert result[0].confidence == 0.9
65+
66+
def test_different_rules_same_file_not_deduped(self) -> None:
67+
"""Different rule_ids in same file are independent findings."""
68+
findings = [
69+
_finding(rule_id="TM1", file="a.py"),
70+
_finding(rule_id="TM2", file="a.py"),
71+
]
72+
result = deduplicate(findings)
73+
assert len(result) == 2
74+
75+
def test_different_matched_text_same_file_not_deduped(self) -> None:
76+
"""Same rule but different matched text in same file → separate findings."""
77+
findings = [
78+
_finding(file="a.py", matched_text="subprocess.run(cmd, shell=True)"),
79+
_finding(file="a.py", matched_text="subprocess.Popen(cmd, shell=True)"),
80+
]
81+
result = deduplicate(findings)
82+
assert len(result) == 2
83+
84+
85+
class TestCrossFileDedup:
86+
"""Same rule_id + same matched_text across files → keep best."""
87+
88+
def test_same_pattern_across_files_deduplicated(self) -> None:
89+
"""Same rule + same matched text in different files → one output."""
90+
findings = [
91+
_finding(file="step1.py"),
92+
_finding(file="step2.py"),
93+
_finding(file="step3.py"),
94+
_finding(file="step4.py"),
95+
]
96+
result = deduplicate(findings)
97+
assert len(result) == 1
98+
99+
def test_cross_file_keeps_highest_confidence(self) -> None:
100+
"""Cross-file dedup keeps the highest confidence finding."""
101+
findings = [
102+
_finding(file="a.py", confidence=0.5),
103+
_finding(file="b.py", confidence=0.9),
104+
_finding(file="c.py", confidence=0.7),
105+
]
106+
result = deduplicate(findings)
107+
assert len(result) == 1
108+
assert result[0].confidence == 0.9
109+
assert result[0].file == "b.py"
110+
111+
def test_different_patterns_across_files_not_deduped(self) -> None:
112+
"""Different matched texts are independent even with same rule_id."""
113+
findings = [
114+
_finding(file="a.py", matched_text="curl -k"),
115+
_finding(file="b.py", matched_text="wget --no-check-certificate"),
116+
]
117+
result = deduplicate(findings)
118+
assert len(result) == 2
119+
120+
def test_different_rules_same_pattern_not_deduped(self) -> None:
121+
"""Different rules with same matched text are independent."""
122+
findings = [
123+
_finding(rule_id="TM1", file="a.py", matched_text="curl -k"),
124+
_finding(rule_id="SC1", file="b.py", matched_text="curl -k"),
125+
]
126+
result = deduplicate(findings)
127+
assert len(result) == 2
128+
129+
130+
class TestNoMatchedText:
131+
"""Findings without matched_text are never cross-file deduplicated."""
132+
133+
def test_no_matched_text_kept_independently(self) -> None:
134+
"""Findings with empty/None matched_text are all kept."""
135+
findings = [
136+
_finding(file="a.py", matched_text=""),
137+
_finding(file="b.py", matched_text=""),
138+
]
139+
result = deduplicate(findings)
140+
assert len(result) == 2
141+
142+
def test_none_matched_text_kept(self) -> None:
143+
"""Findings with None matched_text are preserved."""
144+
f1 = Finding(rule_id="TM1", message="Test", file="a.py", start_line=1, matched_text=None)
145+
f2 = Finding(rule_id="TM1", message="Test", file="b.py", start_line=1, matched_text=None)
146+
result = deduplicate([f1, f2])
147+
assert len(result) == 2
148+
149+
150+
class TestEdgeCases:
151+
"""Edge cases and ordering."""
152+
153+
def test_empty_list(self) -> None:
154+
"""Empty input returns empty output."""
155+
assert deduplicate([]) == []
156+
157+
def test_single_finding_unchanged(self) -> None:
158+
"""A single finding passes through unchanged."""
159+
findings = [_finding()]
160+
result = deduplicate(findings)
161+
assert len(result) == 1
162+
assert result[0].rule_id == "TM1"
163+
164+
def test_output_sorted_by_severity_then_file(self) -> None:
165+
"""Output is sorted: CRITICAL > HIGH > MEDIUM > LOW, then by file."""
166+
findings = [
167+
_finding(rule_id="A", severity="LOW", file="z.py", matched_text="low"),
168+
_finding(rule_id="B", severity="CRITICAL", file="a.py", matched_text="crit"),
169+
_finding(rule_id="C", severity="HIGH", file="m.py", matched_text="high"),
170+
_finding(rule_id="D", severity="MEDIUM", file="b.py", matched_text="med"),
171+
]
172+
result = deduplicate(findings)
173+
assert len(result) == 4
174+
assert [r.severity for r in result] == ["CRITICAL", "HIGH", "MEDIUM", "LOW"]
175+
176+
def test_real_world_repetitive_skill(self) -> None:
177+
"""Simulates a skill with subprocess in 5 files — should deduplicate to 1."""
178+
findings = [
179+
_finding(
180+
rule_id="TM1",
181+
file=f"step{i}.py",
182+
matched_text="subprocess.run(cmd, shell=True)",
183+
confidence=0.8,
184+
)
185+
for i in range(5)
186+
]
187+
result = deduplicate(findings)
188+
assert len(result) == 1
189+
190+
def test_mixed_dedup_scenario(self) -> None:
191+
"""Mix of same-file, cross-file, and unique findings."""
192+
findings = [
193+
# Same pattern in 3 files → should become 1
194+
_finding(rule_id="TM1", file="a.py", matched_text="shell=True"),
195+
_finding(rule_id="TM1", file="b.py", matched_text="shell=True"),
196+
_finding(rule_id="TM1", file="c.py", matched_text="shell=True"),
197+
# Different pattern, unique
198+
_finding(rule_id="E1", file="a.py", matched_text="requests.post(url)"),
199+
# Same rule different pattern
200+
_finding(rule_id="TM1", file="d.py", matched_text="--force delete"),
201+
]
202+
result = deduplicate(findings)
203+
# TM1 shell=True (1) + E1 requests.post (1) + TM1 --force (1) = 3
204+
assert len(result) == 3
205+
206+
def test_whitespace_normalization(self) -> None:
207+
"""Leading/trailing whitespace in matched_text is trimmed for key."""
208+
findings = [
209+
_finding(file="a.py", matched_text=" curl -k "),
210+
_finding(file="b.py", matched_text="curl -k"),
211+
]
212+
result = deduplicate(findings)
213+
assert len(result) == 1
214+
215+
def test_long_matched_text_truncated_for_key(self) -> None:
216+
"""Only first 100 chars of matched_text are used for dedup key."""
217+
base = "x" * 100
218+
findings = [
219+
_finding(file="a.py", matched_text=base + "AAAA"),
220+
_finding(file="b.py", matched_text=base + "BBBB"),
221+
]
222+
result = deduplicate(findings)
223+
# First 100 chars are identical → deduplicated
224+
assert len(result) == 1

0 commit comments

Comments
 (0)