Skip to content

Commit 701baea

Browse files
authored
Merge pull request #616 from efegokdemir/codex/issue-304-gemini-finding-category
fix: preserve metadata for LLM findings
2 parents 088e3c3 + d455606 commit 701baea

6 files changed

Lines changed: 117 additions & 5 deletions

File tree

‎src/skillspector/llm_analyzer_base.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -615,14 +615,22 @@ def _normalize_confidence(cls, v: object) -> float:
615615

616616
def to_finding(self, file: str) -> Finding:
617617
"""Convert to a :class:`Finding` for the graph state."""
618+
# Keep the core LLM module independent from analyzer discovery. The
619+
# discovery package imports this module while loading analyzers.
620+
from skillspector.nodes.analyzers.pattern_defaults import get_category, get_pattern_name
621+
622+
message = self.message[:4096]
618623
return Finding(
619624
rule_id=self.rule_id,
620-
message=self.message,
625+
message=message,
621626
severity=self.severity,
622627
confidence=self.confidence,
623628
file=file,
624629
start_line=self.start_line,
625630
end_line=self.end_line,
631+
category=get_category(self.rule_id),
632+
pattern=message or get_pattern_name(self.rule_id),
633+
finding=message,
626634
explanation=self.explanation,
627635
remediation=self.remediation,
628636
)

‎src/skillspector/nodes/analyzers/mcp_tool_poisoning.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1126,16 +1126,21 @@ def _tp4_finding(
11261126
actual = result.actual_behavior_summary[:1024]
11271127
mismatched = [str(item)[:256] for item in result.mismatched_capabilities[:16]]
11281128
mismatched_text = ", ".join(mismatched)[:2048] if mismatched else "unspecified"
1129+
message = (
1130+
f"Description-behavior mismatch: declared purpose is '{declared}' "
1131+
f"but code also performs: {mismatched_text}."
1132+
)[:4096]
1133+
from skillspector.nodes.analyzers.pattern_defaults import get_pattern_name
1134+
11291135
return Finding(
11301136
rule_id="TP4",
1131-
message=(
1132-
f"Description-behavior mismatch: declared purpose is '{declared}' "
1133-
f"but code also performs: {mismatched_text}."
1134-
)[:4096],
1137+
message=message,
11351138
severity="HIGH" if result.confidence >= 0.7 else "MEDIUM",
11361139
confidence=result.confidence,
11371140
file="SKILL.md",
11381141
category=_CATEGORY,
1142+
pattern=get_pattern_name("TP4"),
1143+
finding=message,
11391144
tags=list(_FRAMEWORK_TAGS),
11401145
explanation=(result.explanation[:4096] or f"Declared: {declared}. Actual: {actual}."),
11411146
remediation=(

‎src/skillspector/nodes/analyzers/pattern_defaults.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,18 @@ class PatternCategory(StrEnum):
244244
"DS2": PatternCategory.DESERIALIZATION.value,
245245
"DS3": PatternCategory.DESERIALIZATION.value,
246246
"DS4": PatternCategory.DESERIALIZATION.value,
247+
# Semantic LLM analyzers
248+
"SSD-1": PatternCategory.PROMPT_INJECTION.value,
249+
"SSD-2": PatternCategory.PROMPT_INJECTION.value,
250+
"SSD-3": PatternCategory.DATA_EXFILTRATION.value,
251+
"SSD-4": PatternCategory.PROMPT_INJECTION.value,
252+
"SDI-1": PatternCategory.EXCESSIVE_AGENCY.value,
253+
"SDI-2": PatternCategory.EXCESSIVE_AGENCY.value,
254+
"SDI-3": PatternCategory.EXCESSIVE_AGENCY.value,
255+
"SDI-4": PatternCategory.EXCESSIVE_AGENCY.value,
256+
"SQP-1": PatternCategory.TRIGGER_ABUSE.value,
257+
"SQP-2": PatternCategory.OUTPUT_HANDLING.value,
258+
"SQP-3": PatternCategory.OUTPUT_HANDLING.value,
247259
}
248260

249261
# Rule ID -> pattern display name (for report output)
@@ -335,6 +347,18 @@ class PatternCategory(StrEnum):
335347
"DS2": "Ruby Marshal Deserialization",
336348
"DS3": "Unsafe Ruby YAML Deserialization",
337349
"DS4": "Unsafe JavaScript Deserialization",
350+
# Semantic LLM analyzers
351+
"SSD-1": "Semantic Prompt Injection",
352+
"SSD-2": "Novel Attack Phrasing",
353+
"SSD-3": "Natural-Language Data Exfiltration",
354+
"SSD-4": "Narrative Deception",
355+
"SDI-1": "Description-Behavior Mismatch",
356+
"SDI-2": "Context-Inappropriate Capability",
357+
"SDI-3": "Permission Scope Creep",
358+
"SDI-4": "Intent-Code Divergence",
359+
"SQP-1": "Vague Trigger",
360+
"SQP-2": "Missing User Warning",
361+
"SQP-3": "Natural-Language Policy Violation",
338362
}
339363

340364
# Pattern-specific remediations (how to fix the issue)

‎tests/nodes/test_llm_analyzer_base.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2542,9 +2542,24 @@ def test_to_finding(self) -> None:
25422542
assert finding.start_line == 10
25432543
assert finding.end_line == 12
25442544
assert finding.confidence == 0.95
2545+
assert finding.category == "Security"
2546+
assert finding.pattern == "Hardcoded secret"
2547+
assert finding.finding == "Hardcoded secret"
25452548
assert finding.explanation == "Contains API key"
25462549
assert finding.remediation == "Use env vars"
25472550

2551+
def test_to_finding_populates_report_metadata_for_semantic_rules(self) -> None:
2552+
finding = LLMFinding(
2553+
rule_id="SSD-3",
2554+
message="Natural-language credential exfiltration",
2555+
severity="CRITICAL",
2556+
start_line=3,
2557+
).to_finding("SKILL.md")
2558+
2559+
assert finding.category == "Data Exfiltration"
2560+
assert finding.pattern == "Natural-language credential exfiltration"
2561+
assert finding.finding == "Natural-language credential exfiltration"
2562+
25482563
def test_model_dump(self) -> None:
25492564
f = LLMFinding(
25502565
rule_id="SEC-002",

‎tests/nodes/test_report_sanitizer.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import pytest
2323

24+
from skillspector.llm_analyzer_base import LLMFinding
2425
from skillspector.models import Finding
2526
from skillspector.nodes.report import _clean_text, _sanitize_finding, report
2627
from skillspector.state import SkillspectorState
@@ -57,6 +58,7 @@ def test_sanitize_finding_cleans_text_fields_only() -> None:
5758
assert cleaned.pattern == "pattern leak here"
5859
assert "\x1b" not in (cleaned.remediation or "")
5960
assert "\x07" not in (cleaned.context or "")
61+
assert "\x1b" not in (cleaned.pattern or "") and "\x00" not in (cleaned.pattern or "")
6062
# Non-text fields are unchanged.
6163
assert cleaned.rule_id == "E2"
6264
assert cleaned.start_line == 5
@@ -124,6 +126,32 @@ def test_report_redacts_url_credentials_from_every_finding_field(fmt: str, schem
124126
assert secret not in serialized_findings
125127

126128

129+
@pytest.mark.parametrize("fmt", ["json", "sarif"])
130+
def test_report_sanitizes_llm_message_copied_to_pattern(fmt: str) -> None:
131+
message = "review \x1b[31mhttps://user:[email protected]/?token=secret-token\x1b[0m"
132+
finding = LLMFinding(
133+
rule_id="E1",
134+
message=message,
135+
severity="HIGH",
136+
start_line=1,
137+
).to_finding("SKILL.md")
138+
state: SkillspectorState = {
139+
"filtered_findings": [finding],
140+
"component_metadata": [],
141+
"has_executable_scripts": False,
142+
"manifest": {},
143+
"skill_path": None,
144+
"output_format": fmt,
145+
}
146+
147+
result = report(state)
148+
rendered = result["report_body"]
149+
serialized_findings = json.dumps([item.to_dict() for item in result["filtered_findings"]])
150+
assert "\x1b" not in rendered
151+
assert "secret" not in rendered
152+
assert "secret" not in serialized_findings
153+
154+
127155
def test_nested_evidence_preserves_scalar_types_and_original_finding() -> None:
128156
scalar_values = [None, True, False, 42, 1.25]
129157
finding = _dirty_finding()

‎tests/test_mcp_tool_poisoning.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1086,6 +1086,8 @@ def test_mismatch_detected(self, monkeypatch: pytest.MonkeyPatch):
10861086
tp4 = [f for f in result["findings"] if f.rule_id == "TP4"]
10871087
assert len(tp4) >= 1
10881088
assert tp4[0].severity in {"HIGH", "MEDIUM"}
1089+
assert tp4[0].pattern == "Description-Behavior Mismatch"
1090+
assert tp4[0].finding == tp4[0].message
10891091

10901092
def test_no_mismatch_clean(self, monkeypatch: pytest.MonkeyPatch):
10911093
_mock_tp4_structured_llm(monkeypatch, [{"is_mismatch": False}])
@@ -1095,6 +1097,36 @@ def test_no_mismatch_clean(self, monkeypatch: pytest.MonkeyPatch):
10951097
assert len(tp4) == 0
10961098

10971099

1100+
class TestTP4Finding:
1101+
def test_finding_maps_result_metadata_for_unit_ci(self):
1102+
result = mcp_tool_poisoning._TP4AnalysisResult(
1103+
is_mismatch=True,
1104+
confidence=0.9,
1105+
declared_purpose_summary="Local text transformation",
1106+
actual_behavior_summary="Sends source data to a remote endpoint",
1107+
mismatched_capabilities=["network access"],
1108+
explanation="The declared purpose does not disclose network behavior.",
1109+
)
1110+
batch = mcp_tool_poisoning.Batch(
1111+
file_path="scripts/tool.py",
1112+
content="upload(source)",
1113+
start_line=12,
1114+
end_line=12,
1115+
)
1116+
1117+
finding = mcp_tool_poisoning._tp4_finding(result, batch, "A local text utility")
1118+
1119+
assert finding is not None
1120+
assert finding.rule_id == "TP4"
1121+
assert finding.category == "MCP Tool Poisoning"
1122+
assert finding.pattern == "Description-Behavior Mismatch"
1123+
assert finding.finding == finding.message
1124+
assert "network access" in finding.message
1125+
assert finding.evidence["code_path"] == "scripts/tool.py"
1126+
assert finding.evidence["code_start_line"] == 12
1127+
assert finding.evidence["code_end_line"] == 12
1128+
1129+
10981130
class TestTP4MarkdownFences:
10991131
def test_markdown_only_fenced_python_reaches_tp4(self, monkeypatch: pytest.MonkeyPatch):
11001132
structured = _mock_tp4_structured_llm(

0 commit comments

Comments
 (0)