Skip to content

Commit 3a1755c

Browse files
committed
fix(sarif): emit security-severity and scope recursive result URIs
SARIF consumers (CI gates, code scanning dashboards) could not distinguish CRITICAL from HIGH findings, and recursive scans emitted skill-relative URIs with no base, so results from different skills collapsed onto one path. - Emit properties["security-severity"] on every SARIF result using the GitHub code scanning banding (CRITICAL 9.5, HIGH 8.0, MEDIUM 5.5, LOW 3.0); level is unchanged so existing consumers are unaffected. Unknown severities emit no score. - Annotate tool.driver.rules[] entries with security-severity when a rule has a single severity across all findings. - In --recursive merges, set artifactLocation.uriBaseId to SKILLROOT on each result location and run.originalUriBaseIds.SKILLROOT.uri to the skill path, per SARIF 2.1.0 sections 3.4.4 and 3.14.14. Single-skill output is unchanged. Closes NVIDIA#620. Signed-off-by: deepujain <[email protected]>
1 parent c7958a3 commit 3a1755c

6 files changed

Lines changed: 348 additions & 6 deletions

File tree

‎src/skillspector/cli.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2577,6 +2577,11 @@ def _multi_skill_text_summary(
25772577
return f"{risk}\n\n{_multi_skill_text_completeness(completeness)}"
25782578

25792579

2580+
# uriBaseId key used to scope recursive child-run result URIs to their own
2581+
# skill directory (SARIF 2.1.0 section 3.14.14).
2582+
_RECURSIVE_SARIF_URI_BASE_ID = "SKILLROOT"
2583+
2584+
25802585
def _multi_skill_sarif_report(
25812586
processed_skills: list[SkillDirectory],
25822587
results: list[dict[str, object]],
@@ -2606,6 +2611,26 @@ def _multi_skill_sarif_report(
26062611
"path": skill.relative_path,
26072612
}
26082613
run["properties"] = run_properties
2614+
# SARIF 2.1.0 sections 3.4.4 and 3.14.14: scope each child run's
2615+
# skill-relative URIs to its own skill directory so results from
2616+
# different skills stop collapsing onto one repo-root-relative
2617+
# path. Single-skill output is untouched: only recursive merges
2618+
# set uriBaseId.
2619+
relative = skill.relative_path.strip("/")
2620+
if relative:
2621+
for result in run.get("results", []):
2622+
if not isinstance(result, dict):
2623+
continue
2624+
for location in result.get("locations", []):
2625+
physical = (
2626+
location.get("physicalLocation") if isinstance(location, dict) else None
2627+
)
2628+
artifact = (
2629+
physical.get("artifactLocation") if isinstance(physical, dict) else None
2630+
)
2631+
if isinstance(artifact, dict):
2632+
artifact["uriBaseId"] = _RECURSIVE_SARIF_URI_BASE_ID
2633+
run["originalUriBaseIds"] = {_RECURSIVE_SARIF_URI_BASE_ID: {"uri": f"{relative}/"}}
26092634
runs.append(run)
26102635

26112636
invocation_properties: dict[str, object] = {"analysisCompleteness": completeness}

‎src/skillspector/nodes/report.py‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,9 @@ def _build_sarif_properties(
344344
metadata: dict[str, object] = {
345345
"findingId": finding.finding_id,
346346
"severity": finding_dict["severity"],
347+
# GitHub code scanning banding: emitted alongside level so generic
348+
# SARIF consumers can distinguish CRITICAL from HIGH.
349+
"security-severity": _severity_to_security_severity(str(finding.severity)),
347350
"category": finding_dict["category"],
348351
"pattern": finding_dict["pattern"],
349352
"confidence": finding_dict["confidence"],
@@ -396,6 +399,26 @@ def _severity_to_sarif_level(severity: str) -> Literal["error", "warning", "note
396399
}.get(severity.upper(), "note") # type: ignore[return-value]
397400

398401

402+
# SARIF has no critical level, so CRITICAL and HIGH both map to "error" above.
403+
# GitHub code scanning and most SARIF gates band security-severity >= 9.0 as
404+
# critical, 7.0-8.9 as high, 4.0-6.9 as medium, and 0.1-3.9 as low; the values
405+
# below keep each SkillSpector severity inside its band.
406+
_SEVERITY_TO_SECURITY_SEVERITY: dict[str, str] = {
407+
"CRITICAL": "9.5",
408+
"HIGH": "8.0",
409+
"MEDIUM": "5.5",
410+
"LOW": "3.0",
411+
}
412+
413+
414+
def _severity_to_security_severity(severity: str) -> str | None:
415+
"""Map Finding.severity to a SARIF security-severity score string.
416+
417+
Returns None for unknown severities so no misleading score is emitted.
418+
"""
419+
return _SEVERITY_TO_SECURITY_SEVERITY.get(severity.upper())
420+
421+
399422
def _summary_display_value(value: object) -> str | None:
400423
"""Format a summary field for terminal / markdown output."""
401424
if value is None:
@@ -603,10 +626,12 @@ def _build_sarif(
603626
"""Build one SARIF invocation with canonical inspection notifications."""
604627
results: list[SarifResult] = []
605628
seen_rule_ids: dict[str, str] = {}
629+
seen_rule_severities: dict[str, set[str]] = {}
606630

607631
for finding in findings:
608632
if not finding.rule_id or not finding.message:
609633
continue
634+
seen_rule_severities.setdefault(finding.rule_id, set()).add(str(finding.severity).upper())
610635
occurrences = finding.occurrences or [
611636
{
612637
"file": finding.file,
@@ -652,6 +677,7 @@ def _build_sarif(
652677
finding = sf.finding
653678
if not finding.rule_id or not finding.message:
654679
continue
680+
seen_rule_severities.setdefault(finding.rule_id, set()).add(str(finding.severity).upper())
655681
occurrences = finding.occurrences or [
656682
{
657683
"file": finding.file,
@@ -692,13 +718,24 @@ def _build_sarif(
692718
if finding.rule_id not in seen_rule_ids:
693719
seen_rule_ids[finding.rule_id] = finding.message
694720

695-
rules = [
696-
SarifReportingDescriptor(
697-
id=rule_id,
698-
shortDescription=SarifMessage(text=description),
721+
rules = []
722+
for rule_id, description in sorted(seen_rule_ids.items()):
723+
# A rule carries security-severity only when every reported finding
724+
# for it shares one severity; mixed-severity rules stay unannotated
725+
# rather than mislabeling some of their results.
726+
rule_properties: dict[str, object] | None = None
727+
severities = seen_rule_severities.get(rule_id, set())
728+
if len(severities) == 1:
729+
score = _severity_to_security_severity(next(iter(severities)))
730+
if score is not None:
731+
rule_properties = {"security-severity": score}
732+
rules.append(
733+
SarifReportingDescriptor(
734+
id=rule_id,
735+
shortDescription=SarifMessage(text=description),
736+
properties=rule_properties,
737+
)
699738
)
700-
for rule_id, description in sorted(seen_rule_ids.items())
701-
]
702739

703740
completeness = analysis_completeness or {}
704741

‎src/skillspector/sarif_models.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,13 @@ class SarifRegion(BaseModel):
3838
class SarifArtifactLocation(BaseModel):
3939
"""Reference to an artifact (file) in the run."""
4040

41+
model_config = {"populate_by_name": True}
42+
4143
uri: str
44+
# SARIF 2.1.0 section 3.4.4: base URI identifier resolved through the
45+
# run's originalUriBaseIds. Recursive scans scope each child run's
46+
# skill-relative URIs to its own skill directory with this.
47+
uri_base_id: str | None = Field(default=None, alias="uriBaseId")
4248
index: int | None = None
4349
properties: dict[str, object] | None = None
4450

@@ -98,6 +104,9 @@ class SarifReportingDescriptor(BaseModel):
98104
default_configuration: dict[str, object] | None = Field(
99105
default=None, alias="defaultConfiguration"
100106
)
107+
# Rule-level metadata such as security-severity, emitted only when the
108+
# rule has a single severity across every reported finding.
109+
properties: dict[str, object] | None = None
101110

102111

103112
class SarifDriver(BaseModel):
@@ -158,6 +167,11 @@ class SarifRun(BaseModel):
158167
)
159168
artifacts: list[SarifArtifact] | None = None
160169
invocations: list[SarifInvocation] | None = None
170+
# SARIF 2.1.0 section 3.14.14: resolves artifactLocation.uriBaseId
171+
# values used by this run's results.
172+
original_uri_base_ids: dict[str, SarifArtifactLocation] | None = Field(
173+
default=None, alias="originalUriBaseIds"
174+
)
161175

162176

163177
class SarifLog(BaseModel):
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Tests for SARIF security-severity on results and rule descriptors."""
17+
18+
from __future__ import annotations
19+
20+
import pytest
21+
22+
from skillspector.models import Finding
23+
from skillspector.nodes.report import _build_sarif, _severity_to_security_severity
24+
from skillspector.sarif_models import validate_sarif_report
25+
from skillspector.suppression import SuppressedFinding
26+
27+
28+
def _make_finding(rule_id: str = "P5", severity: str = "CRITICAL", **kwargs) -> Finding:
29+
defaults = {
30+
"message": "Malicious instruction",
31+
"confidence": 0.9,
32+
"file": "SKILL.md",
33+
"start_line": 1,
34+
"end_line": 1,
35+
"remediation": "Remove the instruction",
36+
"tags": ["malicious"],
37+
"context": "context",
38+
"matched_text": "match",
39+
"category": "prompt_injection",
40+
"pattern": "P5",
41+
"finding": "snippet",
42+
"explanation": "explain",
43+
"code_snippet": "code",
44+
"intent": None,
45+
}
46+
defaults.update(kwargs)
47+
return Finding(rule_id=rule_id, severity=severity, **defaults)
48+
49+
50+
@pytest.mark.parametrize(
51+
("severity", "expected"),
52+
[
53+
("CRITICAL", "9.5"),
54+
("HIGH", "8.0"),
55+
("MEDIUM", "5.5"),
56+
("LOW", "3.0"),
57+
("critical", "9.5"),
58+
("UNKNOWN", None),
59+
("", None),
60+
],
61+
)
62+
def test_severity_to_security_severity_mapping(severity: str, expected: str | None) -> None:
63+
"""Each known severity maps inside its GitHub code scanning band."""
64+
assert _severity_to_security_severity(severity) == expected
65+
66+
67+
def test_result_carries_security_severity() -> None:
68+
"""A CRITICAL result keeps level error and gains security-severity 9.5."""
69+
sarif = _build_sarif([_make_finding(severity="CRITICAL")])
70+
result = sarif["runs"][0]["results"][0]
71+
assert result["level"] == "error"
72+
assert result["properties"]["security-severity"] == "9.5"
73+
assert result["properties"]["severity"] == "CRITICAL"
74+
75+
76+
def test_high_and_critical_distinguishable() -> None:
77+
"""HIGH and CRITICAL share level error but carry different security-severity."""
78+
sarif = _build_sarif(
79+
[
80+
_make_finding(rule_id="P5", severity="CRITICAL"),
81+
_make_finding(rule_id="E2", severity="HIGH"),
82+
]
83+
)
84+
by_rule = {r["ruleId"]: r for r in sarif["runs"][0]["results"]}
85+
assert by_rule["P5"]["level"] == by_rule["E2"]["level"] == "error"
86+
assert by_rule["P5"]["properties"]["security-severity"] == "9.5"
87+
assert by_rule["E2"]["properties"]["security-severity"] == "8.0"
88+
89+
90+
def test_unknown_severity_omits_security_severity() -> None:
91+
"""Unknown severities emit no security-severity rather than a wrong score."""
92+
sarif = _build_sarif([_make_finding(severity="UNKNOWN")])
93+
result = sarif["runs"][0]["results"][0]
94+
assert "security-severity" not in result["properties"]
95+
96+
97+
def test_rule_descriptor_carries_security_severity_for_single_severity_rule() -> None:
98+
"""A rule with one severity across all findings is annotated."""
99+
sarif = _build_sarif(
100+
[
101+
_make_finding(rule_id="P5", severity="HIGH"),
102+
_make_finding(rule_id="P5", severity="HIGH", file="other.py"),
103+
]
104+
)
105+
rules = {r["id"]: r for r in sarif["runs"][0]["tool"]["driver"]["rules"]}
106+
assert rules["P5"]["properties"]["security-severity"] == "8.0"
107+
108+
109+
def test_rule_descriptor_omits_security_severity_for_mixed_severity_rule() -> None:
110+
"""A rule with mixed severities stays unannotated instead of mislabeling."""
111+
sarif = _build_sarif(
112+
[
113+
_make_finding(rule_id="P5", severity="CRITICAL"),
114+
_make_finding(rule_id="P5", severity="LOW"),
115+
]
116+
)
117+
rules = {r["id"]: r for r in sarif["runs"][0]["tool"]["driver"]["rules"]}
118+
assert "properties" not in rules["P5"]
119+
120+
121+
def test_suppressed_finding_carries_security_severity() -> None:
122+
"""Baseline-suppressed findings keep the annotation for audit consumers."""
123+
finding = _make_finding(severity="MEDIUM")
124+
sarif = _build_sarif([], suppressed=[SuppressedFinding(finding=finding, reason="baseline")])
125+
result = sarif["runs"][0]["results"][0]
126+
assert result["suppressions"][0]["kind"] == "external"
127+
assert result["properties"]["security-severity"] == "5.5"
128+
129+
130+
def test_security_severity_report_validates() -> None:
131+
"""Annotated output still satisfies the SARIF 2.1.0 model."""
132+
sarif = _build_sarif(
133+
[
134+
_make_finding(rule_id="P5", severity="CRITICAL"),
135+
_make_finding(rule_id="E2", severity="HIGH"),
136+
_make_finding(rule_id="M1", severity="MEDIUM"),
137+
_make_finding(rule_id="L1", severity="LOW"),
138+
]
139+
)
140+
validate_sarif_report(sarif)
141+
assert sarif["runs"][0]["results"][0]["properties"]["security-severity"] == "9.5"

‎tests/unit/test_cli.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1993,6 +1993,88 @@ def test_recursive_sarif_preserves_intrinsic_child_state_without_output_limit(
19931993
assert "aggregate safety limit" not in json.dumps(aggregate_notifications)
19941994

19951995

1996+
def _recursive_child_with_results(label: str) -> dict[str, object]:
1997+
"""Build a child recursive result whose SARIF run has two located results."""
1998+
child = _bounded_recursive_result(label)
1999+
sarif = cast(dict[str, object], child["sarif_report"])
2000+
run = cast(list[dict[str, object]], sarif["runs"])[0]
2001+
run["results"] = [
2002+
{
2003+
"ruleId": "P5",
2004+
"message": {"text": "Malicious instruction"},
2005+
"level": "error",
2006+
"locations": [
2007+
{
2008+
"physicalLocation": {
2009+
"artifactLocation": {"uri": "SKILL.md"},
2010+
"region": {"startLine": 1},
2011+
}
2012+
},
2013+
{
2014+
"physicalLocation": {
2015+
"artifactLocation": {"uri": "scripts/helper.py"},
2016+
"region": {"startLine": 3},
2017+
}
2018+
},
2019+
],
2020+
}
2021+
]
2022+
return child
2023+
2024+
2025+
def test_recursive_sarif_scopes_result_uris_to_skill_directory(tmp_path: Path) -> None:
2026+
"""Recursive child results carry uriBaseId and the run maps it to the skill path."""
2027+
skill = SkillDirectory(tmp_path / "malicious_skill", "malicious_skill", "malicious_skill")
2028+
child = _recursive_child_with_results("one")
2029+
completeness = cli._multi_skill_analysis_completeness(
2030+
total_skills=1,
2031+
complete_skills=1,
2032+
partial_skills=0,
2033+
failed_skills=0,
2034+
omitted_skills=0,
2035+
limitations=[],
2036+
)
2037+
2038+
payload = cli._multi_skill_sarif_report([skill], [child], completeness)
2039+
2040+
validate_sarif_report(payload)
2041+
run = payload["runs"][0]
2042+
assert run["properties"]["recursiveSkill"] == {
2043+
"name": "malicious_skill",
2044+
"path": "malicious_skill",
2045+
}
2046+
assert run["originalUriBaseIds"] == {"SKILLROOT": {"uri": "malicious_skill/"}}
2047+
locations = run["results"][0]["locations"]
2048+
assert [loc["physicalLocation"]["artifactLocation"]["uri"] for loc in locations] == [
2049+
"SKILL.md",
2050+
"scripts/helper.py",
2051+
]
2052+
assert {loc["physicalLocation"]["artifactLocation"]["uriBaseId"] for loc in locations} == {
2053+
"SKILLROOT"
2054+
}
2055+
2056+
2057+
def test_recursive_sarif_child_without_results_still_maps_base_id(tmp_path: Path) -> None:
2058+
"""A child run with no results still publishes the skill base URI mapping."""
2059+
skill = SkillDirectory(tmp_path / "one", "one", "one")
2060+
child = _bounded_recursive_result("one")
2061+
completeness = cli._multi_skill_analysis_completeness(
2062+
total_skills=1,
2063+
complete_skills=1,
2064+
partial_skills=0,
2065+
failed_skills=0,
2066+
omitted_skills=0,
2067+
limitations=[],
2068+
)
2069+
2070+
payload = cli._multi_skill_sarif_report([skill], [child], completeness)
2071+
2072+
validate_sarif_report(payload)
2073+
run = payload["runs"][0]
2074+
assert run["originalUriBaseIds"] == {"SKILLROOT": {"uri": "one/"}}
2075+
assert run["results"] == []
2076+
2077+
19962078
def test_recursive_sarif_labels_actual_serialized_output_cap() -> None:
19972079
"""Only a real recursive output bound uses the output-limit reason code."""
19982080
reason = "recursive serialized report character budget 1800 reached"

0 commit comments

Comments
 (0)