Skip to content

Commit 94efd59

Browse files
committed
fix(scoring): apply 1.3x multiplier only to findings from executable files
Previously the risk score multiplied ALL findings by 1.3x if any executable script existed in the skill, punishing documentation and teaching content that mentions attack patterns for educational or deny-list purposes. This change: - Builds a file-executable lookup from component_metadata - Applies 1.3x only to findings from executable files (.py, .sh, .js, etc.) - Leaves non-executable-file findings at base severity weight - No change when component_metadata is unavailable (backward compatible) Added test: doc_findings_no_multiplier verifies markdown findings are not inflated by the executable multiplier. Addresses the scoring concern in NVIDIA#103 Signed-off-by: Perseus Computing <[email protected]>
1 parent ac6b41b commit 94efd59

2 files changed

Lines changed: 69 additions & 11 deletions

File tree

‎src/skillspector/nodes/report.py‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,9 @@ def _severity_to_sarif_level(severity: str) -> Literal["error", "warning", "note
8787

8888

8989
def _compute_risk_score(
90-
findings: list[Finding], has_executable_scripts: bool
90+
findings: list[Finding],
91+
has_executable_scripts: bool,
92+
component_metadata: list[dict[str, object]] | None = None,
9193
) -> tuple[int, str, str]:
9294
"""
9395
Compute risk score (0-100), severity band, and recommendation.
@@ -98,8 +100,16 @@ def _compute_risk_score(
98100
This prevents repeated pattern matches from inflating the score unboundedly.
99101
100102
Base points per severity: CRITICAL=50, HIGH=25, MEDIUM=10, LOW=5.
101-
Multiplier: 1.3x if has_executable_scripts.
103+
1.3x multiplier applied only to findings from executable script files;
104+
findings from documentation files (markdown, text, json, yaml, toml)
105+
are scored at base weight to avoid punishing security documentation.
102106
"""
107+
# Build lookup: file path -> is_executable
108+
file_executable: dict[str, bool] = {}
109+
if component_metadata:
110+
for cm in component_metadata:
111+
file_executable[str(cm.get("path", ""))] = bool(cm.get("executable", False))
112+
103113
rule_occurrence_count: dict[str, int] = {}
104114
score = 0.0
105115

@@ -119,10 +129,13 @@ def _compute_risk_score(
119129
continue
120130

121131
weight = _DIMINISHING_WEIGHTS[count]
122-
score += base_points * weight * confidence
132+
contribution = base_points * weight * confidence
133+
134+
# Apply 1.3x multiplier only to findings from executable files
135+
if has_executable_scripts and file_executable.get(f.file, False):
136+
contribution *= 1.3
123137

124-
if has_executable_scripts:
125-
score *= 1.3
138+
score += contribution
126139

127140
final_score = min(100, max(0, int(score)))
128141

@@ -552,7 +565,7 @@ def report(state: SkillspectorState) -> dict[str, object]:
552565
# additionally de-duplicates so the same issue is not counted twice.
553566
findings_for_scoring = deduplicate(active_findings)
554567
risk_score, risk_severity, risk_recommendation = _compute_risk_score(
555-
findings_for_scoring, has_executable_scripts
568+
findings_for_scoring, has_executable_scripts, component_metadata
556569
)
557570
sarif_report = _build_sarif(active_findings, suppressed)
558571
analysis_completeness = _build_analysis_completeness(
@@ -618,4 +631,4 @@ def report(state: SkillspectorState) -> dict[str, object]:
618631
"filtered_findings": filtered_findings,
619632
"suppressed_findings": suppressed,
620633
}
621-
return out
634+
return out

‎tests/nodes/test_report.py‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -146,8 +146,9 @@ class TestComputeRiskScoreExecutableMultiplier:
146146
"""Tests for the executable scripts multiplier."""
147147

148148
def test_executable_multiplier_applies(self) -> None:
149-
findings = [_finding("R1", "HIGH", confidence=1.0)]
150-
score, _, _ = _compute_risk_score(findings, True)
149+
findings = [_finding("R1", "HIGH", confidence=1.0, file="run.py")]
150+
component_metadata = [{"path": "run.py", "executable": True}]
151+
score, _, _ = _compute_risk_score(findings, True, component_metadata)
151152
# 25 * 1.3 = 32.5 -> 32
152153
assert score == 32
153154

@@ -387,8 +388,8 @@ def test_report_executable_scripts_multiplier(self) -> None:
387388
"""has_executable_scripts applies 1.3x to risk score."""
388389
state: SkillspectorState = {
389390
"filtered_findings": [
390-
_finding("E2", "HIGH", confidence=1.0),
391-
_finding("PE3", "HIGH", confidence=1.0),
391+
_finding("E2", "HIGH", confidence=1.0, file="run.py"),
392+
_finding("PE3", "HIGH", confidence=1.0, file="run.py"),
392393
],
393394
"component_metadata": [
394395
{
@@ -626,3 +627,47 @@ def test_report_no_baseline_unchanged() -> None:
626627
result = report(state)
627628
assert result["risk_score"] == 50
628629
assert result["suppressed_findings"] == []
630+
631+
def test_report_executable_scripts_multiplier() -> None:
632+
"""1.3x multiplier applied only to findings from executable files."""
633+
# 2 HIGH findings in run.py = 2 × 25 × 1.3 = 65 (float-based accumulation)
634+
state: SkillspectorState = {
635+
"filtered_findings": [
636+
_finding("E2", "HIGH", file="run.py"),
637+
_finding("PE3", "HIGH", file="run.py"),
638+
],
639+
"component_metadata": [
640+
{"path": "run.py", "type": "python", "lines": 5, "executable": True, "size_bytes": 200}
641+
],
642+
"has_executable_scripts": True,
643+
"manifest": {},
644+
"skill_path": "/tmp/skill",
645+
"output_format": "json",
646+
}
647+
result = report(state)
648+
assert result["risk_score"] == 65
649+
assert result["risk_severity"] == "HIGH"
650+
assert result["risk_recommendation"] == "DO_NOT_INSTALL"
651+
652+
653+
def test_report_doc_findings_no_multiplier() -> None:
654+
"""Findings from non-executable files (markdown/docs) are not multiplied."""
655+
# 2 HIGH in SKILL.md (non-executable) = 2 × 25 = 50 (no 1.3x)
656+
state: SkillspectorState = {
657+
"filtered_findings": [
658+
_finding("P1", "HIGH", file="SKILL.md"),
659+
_finding("P2", "HIGH", file="SKILL.md"),
660+
],
661+
"component_metadata": [
662+
{"path": "SKILL.md", "type": "markdown", "lines": 10, "executable": False, "size_bytes": 500},
663+
{"path": "run.py", "type": "python", "lines": 5, "executable": True, "size_bytes": 200}
664+
],
665+
"has_executable_scripts": True,
666+
"manifest": {},
667+
"skill_path": "/tmp/skill",
668+
"output_format": "json",
669+
}
670+
result = report(state)
671+
# Without the multiplier: 2 HIGH = 50, not 65
672+
assert result["risk_score"] == 50
673+
assert result["risk_severity"] == "MEDIUM"

0 commit comments

Comments
 (0)