Skip to content

Commit ee1ac5f

Browse files
Merge branch 'main' into codex/security-text-normalization
2 parents 3d89803 + e5440f6 commit ee1ac5f

4 files changed

Lines changed: 292 additions & 9 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ SkillSpector is part of the [NVIDIA Verified Skills pipeline](https://docs.nvidi
2424
## Features
2525

2626
- **Multi-format input**: Scan Git repos, URLs, zip files, directories, or single files
27-
- **70 vulnerability patterns** across 17 categories: prompt injection, data exfiltration, privilege escalation, supply chain, excessive agency, output handling, system prompt leakage, memory poisoning, tool misuse, rogue agent, anti-refusal, trigger abuse, dangerous code (AST), taint tracking, YARA signatures, MCP least privilege, and MCP tool poisoning
27+
- **71 vulnerability patterns** across 17 categories: prompt injection, data exfiltration, privilege escalation, supply chain, excessive agency, output handling, system prompt leakage, memory poisoning, tool misuse, rogue agent, anti-refusal, trigger abuse, dangerous code (AST), taint tracking, YARA signatures, MCP least privilege, and MCP tool poisoning
2828
- **Two-stage analysis**: Fast static analysis + optional LLM semantic evaluation
2929
- **Live vulnerability lookups**: SC4 queries [OSV.dev](https://osv.dev) for real-time CVE data with automatic offline fallback
3030
- **Multiple output formats**: Terminal, JSON, Markdown, and SARIF reports
@@ -354,7 +354,7 @@ claude mcp add skillspector -- skillspector mcp
354354
355355
## Vulnerability Patterns
356356

357-
SkillSpector detects **70 vulnerability patterns** across 17 categories:
357+
SkillSpector detects **71 vulnerability patterns** across 17 categories:
358358

359359
### Prompt Injection (6 patterns)
360360

@@ -405,14 +405,15 @@ SkillSpector detects **70 vulnerability patterns** across 17 categories:
405405
| SC8 | Shipped Python Bytecode | HIGH | `__pycache__` / `.pyc` present (discovery skips; malicious bytecode bypass) |
406406
| SC9 | Concealed Executable Artifact | HIGH | Executable nested in a document container or hidden/disguised artifact |
407407

408-
### Excessive Agency (4 patterns)
408+
### Excessive Agency (5 patterns)
409409

410410
| ID | Pattern | Severity | Description |
411411
|----|---------|----------|-------------|
412412
| EA1 | Unrestricted Tool Access | HIGH | Unfettered tool access without constraints |
413413
| EA2 | Autonomous Decision Making | HIGH | High-impact decisions without human-in-the-loop |
414414
| EA3 | Scope Creep | MEDIUM | Capabilities extending beyond stated purpose |
415415
| EA4 | Unbounded Resource Access | MEDIUM | No rate limits or quotas on resource consumption |
416+
| EA5 | External Model or Provider Selection | MEDIUM/HIGH | Model/provider pins or coding-CLI shell-outs that can switch billing accounts |
416417

417418
### Output Handling (3 patterns)
418419

‎src/skillspector/nodes/analyzers/pattern_defaults.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ class PatternCategory(StrEnum):
6868
"EA2": "Skill enables autonomous high-impact decisions without human-in-the-loop verification. Critical operations (destructive commands, financial transactions, data deletion) should require explicit user confirmation.",
6969
"EA3": "Skill's behavior or capabilities extend beyond its stated purpose. Scope creep allows an agent to perform actions unrelated to its documented functionality, increasing the attack surface.",
7070
"EA4": "Skill allows unbounded resource consumption (API calls, storage, compute). Without rate limits or quotas, a compromised or misbehaving agent can cause denial-of-service or cost overruns.",
71+
"EA5": "Skill selects an external model or provider that may use a different account or billing plan than the operator expects. Undisclosed model switches can cause unexpected cost or quota consumption.",
7172
# Output Handling (B.1.7)
7273
"OH1": "Model output is used without validation or sanitization. Unvalidated output injected into downstream contexts (SQL, shell, HTML) enables injection attacks and arbitrary code execution.",
7374
"OH2": "Output from one security context is used in another without boundary enforcement. Cross-context output flow can leak sensitive information or escalate privileges across trust boundaries.",
@@ -177,6 +178,7 @@ class PatternCategory(StrEnum):
177178
"EA2": PatternCategory.EXCESSIVE_AGENCY.value,
178179
"EA3": PatternCategory.EXCESSIVE_AGENCY.value,
179180
"EA4": PatternCategory.EXCESSIVE_AGENCY.value,
181+
"EA5": PatternCategory.EXCESSIVE_AGENCY.value,
180182
"OH1": PatternCategory.OUTPUT_HANDLING.value,
181183
"OH2": PatternCategory.OUTPUT_HANDLING.value,
182184
"OH3": PatternCategory.OUTPUT_HANDLING.value,
@@ -264,6 +266,7 @@ class PatternCategory(StrEnum):
264266
"EA2": "Autonomous Decision Making",
265267
"EA3": "Scope Creep",
266268
"EA4": "Unbounded Resource Access",
269+
"EA5": "External Model or Provider Selection",
267270
"OH1": "Unvalidated Output Injection",
268271
"OH2": "Cross-Context Output",
269272
"OH3": "Unbounded Output",
@@ -351,6 +354,7 @@ class PatternCategory(StrEnum):
351354
"EA2": "Add human-in-the-loop confirmation for destructive, irreversible, or high-impact operations. Never auto-execute commands that modify files, send data, or alter system state.",
352355
"EA3": "Limit the skill's scope to its documented purpose. Remove instructions that enable the agent to perform actions outside its stated functionality.",
353356
"EA4": "Set explicit rate limits, timeouts, and resource quotas for API calls, file operations, and compute. Implement circuit breakers for runaway loops.",
357+
"EA5": "Remove the model/provider override or disclose it prominently and require explicit operator approval before invoking an external coding CLI or billed model.",
354358
# Output Handling (B.1.7)
355359
"OH1": "Validate and sanitize all model output before using it in downstream contexts. Use parameterized queries for SQL, shell quoting for commands, and HTML encoding for web output.",
356360
"OH2": "Enforce strict context boundaries. Do not pass output from one security domain into another without explicit validation and redaction of sensitive content.",

‎src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py‎

Lines changed: 207 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,21 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
"""Static patterns: excessive agency (EA1–EA4). Node and analyze() in one module.
16+
"""Static patterns: excessive agency (EA1–EA5). Node and analyze() in one module.
1717
1818
Detects patterns where an agent skill grants unrestricted tool access (EA1),
1919
enables autonomous high-impact decisions without human-in-the-loop (EA2),
2020
exhibits scope creep beyond stated purpose (EA3), or allows unbounded
21-
resource consumption (EA4).
21+
resource consumption (EA4), or selects an external model/provider with billing
22+
implications (EA5).
2223
2324
Framework: LLM06, ASI02.
2425
"""
2526

2627
from __future__ import annotations
2728

2829
import re
30+
import shlex
2931
import sys
3032

3133
from skillspector.logging_config import get_logger
@@ -155,9 +157,210 @@
155157
),
156158
]
157159

160+
# EA5: External Model or Provider Selection
161+
_EA5_FRONTMATTER_KEY = re.compile(
162+
r"^[\"']?(?P<key>model|provider|model_name|model_id)[\"']?[ \t]*:[ \t]*"
163+
r"(?P<value>[^#\s][^#\r\n]*)",
164+
re.IGNORECASE | re.MULTILINE,
165+
)
166+
_EA5_INLINE_CODE = re.compile(r"`(?P<command>[^`\r\n]+)`")
167+
_EA5_IMPERATIVE_PREFIX = re.compile(
168+
r"^(?:run|execute|invoke|call)(?:[ \t]+the)?(?:[ \t]+command)?[ \t]*:?[ \t]+",
169+
re.IGNORECASE,
170+
)
171+
_EA5_INLINE_DIRECTIVE_PREFIX = re.compile(
172+
r"^(?:(?:[-*+]|\d+[.)])[ \t]+)?"
173+
r"(?:run|execute|invoke|call)(?:[ \t]+the)?(?:[ \t]+command)?[ \t]*:?[ \t]*$",
174+
re.IGNORECASE,
175+
)
176+
_EA5_FENCE = re.compile(r"^[ \t]*(?P<marker>`{3,}|~{3,})[ \t]*(?P<language>[\w+-]*)")
177+
_EA5_SHELL_FENCE_LANGUAGES = {"bash", "console", "sh", "shell", "zsh"}
178+
_EA5_MODEL_VALUE = re.compile(
179+
r"^(?:claude|gpt|gemini|deepseek|kimi|glm|minimax|mistral|llama)"
180+
r"(?:$|[-_./:0-9])",
181+
re.IGNORECASE,
182+
)
183+
184+
185+
def _frontmatter_bounds(content: str, file_path: str) -> tuple[int, int] | None:
186+
"""Return the YAML-frontmatter byte offsets for a SKILL.md file."""
187+
if file_path.rsplit("/", 1)[-1].lower() != "skill.md":
188+
return None
189+
opening = re.match(r"\A---[ \t]*\r?\n", content)
190+
if opening is None:
191+
return None
192+
closing = re.search(r"^---[ \t]*$", content[opening.end() :], re.MULTILINE)
193+
if closing is None:
194+
return None
195+
return opening.end(), opening.end() + closing.start()
196+
197+
198+
def _is_model_switch_command(command: str) -> bool:
199+
"""Return whether a shell command selects another coding model/provider."""
200+
try:
201+
tokens = shlex.split(command, comments=False, posix=True)
202+
except ValueError:
203+
tokens = command.split()
204+
if not tokens:
205+
return False
206+
207+
executable = tokens[0].rsplit("/", 1)[-1].lower()
208+
if executable == "claude" and any(
209+
token in {"-p", "--print"} or token.startswith("--print=") for token in tokens[1:]
210+
):
211+
return True
212+
if executable == "codex" and len(tokens) > 1 and tokens[1].lower() == "exec":
213+
return True
214+
215+
if executable.startswith("python") or executable in {"node", "perl", "ruby"}:
216+
return False
217+
218+
for index, token in enumerate(tokens[1:], start=1):
219+
value: str | None = None
220+
if token in {"-m", "--model"} and index + 1 < len(tokens):
221+
value = tokens[index + 1]
222+
elif token.startswith(("-m=", "--model=")):
223+
value = token.split("=", 1)[1]
224+
if value and _EA5_MODEL_VALUE.match(value):
225+
return True
226+
return False
227+
228+
229+
def _command_span(line: str) -> tuple[int, int] | None:
230+
"""Return the model-switch command span for an actionable instruction line."""
231+
# Inline code is handled separately so surrounding prose and punctuation do
232+
# not become part of the command span (or create a duplicate finding).
233+
if "`" in line:
234+
return None
235+
236+
leading = len(line) - len(line.lstrip(" \t"))
237+
candidate = line[leading:]
238+
if candidate.startswith(("$", ">")):
239+
prompt_width = 1 + len(candidate[1:]) - len(candidate[1:].lstrip(" \t"))
240+
leading += prompt_width
241+
candidate = candidate[prompt_width:]
242+
243+
command = candidate.strip().strip("`")
244+
if _is_model_switch_command(command):
245+
start = line.find(command, leading)
246+
return start, start + len(command)
247+
248+
imperative = _EA5_IMPERATIVE_PREFIX.match(candidate)
249+
if imperative is not None:
250+
command = candidate[imperative.end() :].strip().strip("`")
251+
if _is_model_switch_command(command):
252+
start = line.find(command, leading + imperative.end())
253+
return start, start + len(command)
254+
return None
255+
256+
257+
def _inline_command_span(line: str, inline: re.Match[str]) -> tuple[int, int] | None:
258+
"""Return an inline model-switch command only when prose directs its execution."""
259+
if _EA5_INLINE_DIRECTIVE_PREFIX.match(line[: inline.start()]) is None:
260+
return None
261+
command = inline.group("command").strip()
262+
if not _is_model_switch_command(command):
263+
return None
264+
command_offset = (
265+
inline.start("command")
266+
+ len(inline.group("command"))
267+
- len(inline.group("command").lstrip())
268+
)
269+
return command_offset, command_offset + len(command)
270+
271+
272+
def _ea5_findings(content: str, file_path: str) -> list[AnalyzerFinding]:
273+
"""Detect declarative model pins and actionable coding-CLI model switches."""
274+
findings: list[AnalyzerFinding] = []
275+
tag = [PatternCategory.EXCESSIVE_AGENCY.value]
276+
bounds = _frontmatter_bounds(content, file_path)
277+
body_start = 0
278+
if bounds is not None:
279+
start, end = bounds
280+
body_start = end
281+
frontmatter = content[start:end]
282+
for match in _EA5_FRONTMATTER_KEY.finditer(frontmatter):
283+
value = match.group("value").strip().lower()
284+
if value in {'""', "''", "~", "null", "none", "default", "auto", "inherit"}:
285+
continue
286+
absolute_start = start + match.start()
287+
key = match.group("key").lower()
288+
findings.append(
289+
AnalyzerFinding(
290+
rule_id="EA5",
291+
message="External Model or Provider Selection",
292+
severity=Severity.MEDIUM,
293+
location=Location(
294+
file=file_path,
295+
start_line=get_line_number(content, absolute_start),
296+
),
297+
confidence=0.9,
298+
tags=tag,
299+
context=get_context(content, absolute_start),
300+
matched_text=match.group(0)[:200],
301+
evidence={"selection_surface": "frontmatter", "selection_key": key},
302+
)
303+
)
304+
305+
seen: set[tuple[int, int]] = set()
306+
cursor = body_start
307+
fence_marker: str | None = None
308+
fence_language = ""
309+
for line in content[body_start:].splitlines(keepends=True):
310+
line_text = line.rstrip("\r\n")
311+
fence = _EA5_FENCE.match(line_text)
312+
if fence is not None:
313+
marker = fence.group("marker")
314+
if fence_marker is None:
315+
fence_marker = marker[0]
316+
fence_language = fence.group("language").lower()
317+
elif marker[0] == fence_marker:
318+
fence_marker = None
319+
fence_language = ""
320+
cursor += len(line)
321+
continue
322+
323+
candidates: list[tuple[int, int]] = []
324+
if (
325+
fence_marker is None
326+
or fence_language in _EA5_SHELL_FENCE_LANGUAGES
327+
or not fence_language
328+
):
329+
direct = _command_span(line_text)
330+
if direct is not None:
331+
candidates.append(direct)
332+
for inline in _EA5_INLINE_CODE.finditer(line_text):
333+
command_span = _inline_command_span(line_text, inline)
334+
if command_span is not None:
335+
candidates.append(command_span)
336+
337+
for line_start, line_end in candidates:
338+
absolute = (cursor + line_start, cursor + line_end)
339+
if absolute in seen:
340+
continue
341+
seen.add(absolute)
342+
findings.append(
343+
AnalyzerFinding(
344+
rule_id="EA5",
345+
message="External Model or Provider Selection",
346+
severity=Severity.HIGH,
347+
location=Location(
348+
file=file_path,
349+
start_line=get_line_number(content, absolute[0]),
350+
),
351+
confidence=0.9,
352+
tags=tag,
353+
context=get_context(content, absolute[0]),
354+
matched_text=content[absolute[0] : absolute[1]][:200],
355+
evidence={"selection_surface": "command"},
356+
)
357+
)
358+
cursor += len(line)
359+
return findings
360+
158361

159362
def analyze(content: str, file_path: str, file_type: str) -> list[AnalyzerFinding]:
160-
"""Analyze content for excessive agency patterns (EA1–EA4)."""
363+
"""Analyze content for excessive agency patterns (EA1–EA5)."""
161364
findings: list[AnalyzerFinding] = []
162365

163366
def loc(ln: int) -> Location:
@@ -229,6 +432,7 @@ def ctx(start: int) -> str:
229432
matched_text=match.group(0)[:200],
230433
)
231434
)
435+
findings.extend(_ea5_findings(content, file_path))
232436
return findings
233437

234438

0 commit comments

Comments
 (0)