Skip to content

Commit 3300434

Browse files
committed
fix(static): address review findings on description trigger analysis
Require a bounded activation condition plus an unconditional scope for description TR3, so bare behavior prose (Always preserves file permissions when copying files) and subject-qualified scopes (any questions about PostgreSQL) stay negative while the catch-all positive still fires. Open the TR2 gate to invocation/shadowing clauses on their own, so Intercepts the /build command is analyzed without broad-activation wording. Analyze overlong clauses through bounded head/tail windows and scan every clause for intent signals, so padding cannot silently push a trigger clause out of the analysis. Add the TR1 description path by extracting the trigger phrase from activation prose (whenever the user says hello), proven reachable from SKILL.md frontmatter. Signed-off-by: Deepak Jain <[email protected]>
1 parent 598f6a0 commit 3300434

2 files changed

Lines changed: 232 additions & 50 deletions

File tree

‎src/skillspector/nodes/analyzers/static_patterns_supply_chain.py‎

Lines changed: 142 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -499,19 +499,30 @@ def _is_typosquat(pkg_name: str, popular: set[str], max_distance: int = 2) -> st
499499
"hey",
500500
}
501501

502-
# Bounded activation-intent signals for skill descriptions. Only description
503-
# clauses carrying one of these signals are treated as trigger-like; ordinary
504-
# capability prose (e.g. "Build projects") is skipped.
505-
_DESCRIPTION_ACTIVATION_RE = re.compile(
502+
# Activation-condition signals for description clauses. A description clause
503+
# only counts as trigger-like when it says *when* the skill activates (a
504+
# bounded condition), not merely what it does: bare behavior prose such as
505+
# "Always preserves file permissions when copying files" does not qualify.
506+
_DESCRIPTION_ACTIVATION_CONDITION_RE = re.compile(
506507
r"\b(?:"
507-
r"whenever|every\s+time|each\s+time|"
508-
r"(?:all|any|every)\s+(?:messages?|requests?|questions?|queries?|inputs?|tasks?)|"
509-
r"any\s+(?:question|request|task|input|message)|"
510-
r"anything|everything|whatever|always"
508+
r"whenever|"
509+
r"(?:when|if)\s+(?:the\s+)?user\s+(?:says?|asks?|types?|sends?|requests?)|"
510+
r"every\s+time|each\s+time"
511511
r")\b",
512512
re.IGNORECASE,
513513
)
514514

515+
# Universal-scope signals for description clauses. The scope must be
516+
# unconditional: a subject qualifier such as "about PostgreSQL" keeps the
517+
# clause describing a capability, not a catch-all trigger.
518+
_DESCRIPTION_UNIVERSAL_SCOPE_RE = re.compile(
519+
r"\b(?:"
520+
r"anything|everything|whatever|"
521+
r"(?:all|any|every)\s+(?:messages?|requests?|questions?|queries?|inputs?|tasks?)"
522+
r")\b(?!\s+about\b)",
523+
re.IGNORECASE,
524+
)
525+
515526
# Invocation/shadowing intent signals. A description only counts as a shadow
516527
# command when it shows the skill means to intercept or override a command.
517528
_DESCRIPTION_INVOCATION_RE = re.compile(
@@ -520,16 +531,21 @@ def _is_typosquat(pkg_name: str, popular: set[str], max_distance: int = 2) -> st
520531
re.IGNORECASE,
521532
)
522533

523-
# Unanchored baiting patterns for description clauses. The legacy TR3 patterns
524-
# are whole-string anchored for the explicit triggers field; descriptions need
525-
# substring matching so realistic activation prose is detected.
526-
_DESCRIPTION_BAITING_PATTERNS = (
527-
r"(?:anything|everything|whatever|always|"
528-
r"any\s+(?:question|request|task|input|message))",
529-
r"(?:when(?:ever)?|if|every\s+time)\s+(?:the\s+)?user\s+"
530-
r"(?:says?|asks?|types?|sends?)\s+"
531-
r"(?:anything|something|any\s+messages?|a\s+message)",
534+
# Trigger-phrase extraction for the TR1 broad/short-trigger rule on
535+
# descriptions: the word or phrase the skill claims to activate on, as in
536+
# "whenever the user says hello".
537+
_DESCRIPTION_TRIGGER_PHRASE_RE = re.compile(
538+
r"\b(?:whenever|when|if)\s+(?:the\s+)?user\s+"
539+
r"(?:says?|asks?|types?|sends?|requests?)\s+"
540+
r"(?:the\s+(?:word|phrase)\s+)?['\"]?(?P<phrase>[A-Za-z][\w-]{0,31})['\"]?",
541+
re.IGNORECASE,
542+
)
543+
544+
# Bare universal-scope statements: the whole clause is a catch-all scope
545+
# ("all messages"), which the legacy trigger grammar also flags as TR3.
546+
_DESCRIPTION_BARE_SCOPE_RE = re.compile(
532547
r"(?:all|any|every)\s+(?:messages?|inputs?|requests?|queries?|questions?)",
548+
re.IGNORECASE,
533549
)
534550

535551
# Bounds for description clause extraction: keep the analysis cheap and the
@@ -538,21 +554,40 @@ def _is_typosquat(pkg_name: str, popular: set[str], max_distance: int = 2) -> st
538554
_MAX_DESCRIPTION_CLAUSE_CHARS = 120
539555

540556

557+
def _description_clause_has_signal(text: str) -> bool:
558+
"""Cheap gate: does this clause carry any trigger-relevant intent?"""
559+
return (
560+
_DESCRIPTION_ACTIVATION_CONDITION_RE.search(text) is not None
561+
or _DESCRIPTION_UNIVERSAL_SCOPE_RE.search(text) is not None
562+
or _DESCRIPTION_INVOCATION_RE.search(text) is not None
563+
)
564+
565+
541566
def _extract_description_trigger_clauses(description: str) -> list[str]:
542567
"""Extract bounded trigger-like clauses from a skill description.
543568
544-
Splits the description into clauses and keeps only clauses carrying an
545-
activation-intent signal, so realistic activation prose is analyzed while
546-
ordinary capability prose is left alone.
569+
Every clause is scanned for cheap intent signals, so benign padding
570+
sentences never consume the clause budget and cannot push a trigger
571+
clause out of the analysis. Overlong clauses are analyzed through
572+
bounded head/tail windows (joined with an explicit " ..." marker) so
573+
activation intent at the start or end of a padded clause is still
574+
inspected while per-clause work stays bounded.
547575
"""
548576
clauses = re.split(r"[.;:!?]\s*|\s+-\s+", description)
549577
extracted: list[str] = []
550-
for clause in clauses[:_MAX_DESCRIPTION_CLAUSES]:
578+
for clause in clauses:
551579
text = clause.strip().strip(",")
552-
if not text or len(text) > _MAX_DESCRIPTION_CLAUSE_CHARS:
580+
if not text or not _description_clause_has_signal(text):
553581
continue
554-
if _DESCRIPTION_ACTIVATION_RE.search(text):
555-
extracted.append(text)
582+
if len(text) > _MAX_DESCRIPTION_CLAUSE_CHARS:
583+
text = (
584+
text[:_MAX_DESCRIPTION_CLAUSE_CHARS]
585+
+ " ... "
586+
+ text[-_MAX_DESCRIPTION_CLAUSE_CHARS:]
587+
)
588+
extracted.append(text)
589+
if len(extracted) >= _MAX_DESCRIPTION_CLAUSES:
590+
break
556591
return extracted
557592

558593

@@ -1898,10 +1933,14 @@ def _analyze_triggers(manifest: dict[str, object], skill_path: str) -> list[Find
18981933
18991934
Agent Skills exposes activation intent through ``description``; legacy
19001935
``triggers`` metadata remains supported when present. Descriptions are not
1901-
passed to the legacy trigger grammar directly: only clauses carrying an
1902-
activation-intent signal are analyzed, with TR2 requiring invocation or
1903-
shadowing intent and TR3 using unanchored baiting patterns, so realistic
1904-
activation prose is detected while ordinary capability prose is skipped.
1936+
passed to the legacy trigger grammar directly: every clause is scanned
1937+
for cheap intent signals, and only clauses carrying one are analyzed.
1938+
TR1 extracts the trigger phrase from activation prose and applies the
1939+
broad/short-trigger rule to it; TR2 requires invocation or shadowing
1940+
intent (reachable without broad-activation wording); TR3 requires a
1941+
bounded activation condition plus an unconditional scope (or a bare
1942+
universal-scope statement). Realistic activation prose is detected while
1943+
ordinary capability prose is skipped.
19051944
"""
19061945
triggers: list[str] = []
19071946
raw = manifest.get("triggers", [])
@@ -2002,9 +2041,58 @@ def _analyze_triggers(manifest: dict[str, object], skill_path: str) -> list[Find
20022041
clause_lower = clause.lower().strip()
20032042
words = clause_lower.split()
20042043

2044+
# TR1 (description-calibrated): extract the trigger phrase the skill
2045+
# claims to activate on ("whenever the user says hello") and apply
2046+
# the broad/short-trigger rule to it, as the legacy triggers field
2047+
# does.
2048+
phrase_match = _DESCRIPTION_TRIGGER_PHRASE_RE.search(clause)
2049+
if phrase_match:
2050+
phrase = phrase_match.group("phrase")
2051+
phrase_lower = phrase.lower()
2052+
if len(phrase_lower.split()) == 1 and phrase_lower in _OVERLY_BROAD_SINGLE_WORDS:
2053+
findings.append(
2054+
Finding(
2055+
rule_id="TR1",
2056+
message=(
2057+
f"Overly Broad Trigger: description clause '{clause}' "
2058+
f"activates on '{phrase}', a common word that will "
2059+
"activate in many unintended contexts"
2060+
),
2061+
severity="LOW",
2062+
confidence=0.75,
2063+
file=file_ref,
2064+
start_line=i,
2065+
tags=tag,
2066+
matched_text=clause,
2067+
category=PatternCategory.TRIGGER_ABUSE.value,
2068+
pattern="Overly Broad Trigger",
2069+
)
2070+
)
2071+
elif len(phrase_lower) <= 2:
2072+
findings.append(
2073+
Finding(
2074+
rule_id="TR1",
2075+
message=(
2076+
f"Overly Broad Trigger: description clause '{clause}' "
2077+
f"activates on '{phrase}', which is too short and may "
2078+
"match unintended inputs"
2079+
),
2080+
severity="LOW",
2081+
confidence=0.7,
2082+
file=file_ref,
2083+
start_line=i,
2084+
tags=tag,
2085+
matched_text=clause,
2086+
category=PatternCategory.TRIGGER_ABUSE.value,
2087+
pattern="Overly Broad Trigger",
2088+
)
2089+
)
2090+
20052091
# TR2 (description-calibrated): only flag a shadow command when the
20062092
# clause shows invocation or shadowing intent; ordinary capability
20072093
# prose such as "Build projects" stays out of the trigger path.
2094+
# Invocation clauses pass the extraction gate on their own, so no
2095+
# broad-activation wording is required.
20082096
shadowed = sorted(
20092097
{cmd for cmd in _BUILTIN_COMMANDS if cmd in {w.lstrip("/") for w in words}}
20102098
)
@@ -2027,29 +2115,33 @@ def _analyze_triggers(manifest: dict[str, object], skill_path: str) -> list[Find
20272115
)
20282116
)
20292117

2030-
# TR3 (description-calibrated): unanchored baiting patterns detect
2031-
# realistic activation prose such as "whenever the user sends any
2032-
# message", which the whole-string legacy grammar misses.
2033-
for bp in _DESCRIPTION_BAITING_PATTERNS:
2034-
if re.search(bp, clause_lower):
2035-
findings.append(
2036-
Finding(
2037-
rule_id="TR3",
2038-
message=(
2039-
f"Keyword Baiting Trigger: description clause '{clause}' "
2040-
"is designed to match all or most user inputs"
2041-
),
2042-
severity="MEDIUM",
2043-
confidence=0.8,
2044-
file=file_ref,
2045-
start_line=i,
2046-
tags=tag,
2047-
matched_text=clause,
2048-
category=PatternCategory.TRIGGER_ABUSE.value,
2049-
pattern="Keyword Baiting Trigger",
2050-
)
2118+
# TR3 (description-calibrated): require a bounded activation condition
2119+
# *and* an unconditional scope in the same clause. Bare behavior
2120+
# prose ("Always preserves file permissions when copying files") and
2121+
# subject-qualified scopes ("any questions about PostgreSQL") stay
2122+
# negative; a bare universal-scope statement ("all messages") still
2123+
# fires, as in the legacy trigger grammar.
2124+
has_condition = _DESCRIPTION_ACTIVATION_CONDITION_RE.search(clause) is not None
2125+
has_scope = _DESCRIPTION_UNIVERSAL_SCOPE_RE.search(clause) is not None
2126+
is_bare_scope = _DESCRIPTION_BARE_SCOPE_RE.fullmatch(clause_lower) is not None
2127+
if (has_condition and has_scope) or is_bare_scope:
2128+
findings.append(
2129+
Finding(
2130+
rule_id="TR3",
2131+
message=(
2132+
f"Keyword Baiting Trigger: description clause '{clause}' "
2133+
"is designed to match all or most user inputs"
2134+
),
2135+
severity="MEDIUM",
2136+
confidence=0.8,
2137+
file=file_ref,
2138+
start_line=i,
2139+
tags=tag,
2140+
matched_text=clause,
2141+
category=PatternCategory.TRIGGER_ABUSE.value,
2142+
pattern="Keyword Baiting Trigger",
20512143
)
2052-
break
2144+
)
20532145

20542146
return findings
20552147

‎tests/unit/test_patterns_new.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2268,6 +2268,96 @@ def test_legacy_triggers_bypass_description_calibration(self) -> None:
22682268
findings = sc_mod._analyze_triggers({"triggers": ["Build"]}, "myskill")
22692269
assert any(finding.rule_id == "TR2" for finding in findings)
22702270

2271+
@pytest.mark.parametrize(
2272+
"description",
2273+
[
2274+
pytest.param(
2275+
"Always preserves file permissions when copying files",
2276+
id="behavior_always",
2277+
),
2278+
pytest.param(
2279+
"Answers any questions about PostgreSQL",
2280+
id="qualified_subject",
2281+
),
2282+
],
2283+
)
2284+
def test_description_benign_prose_not_tr3(self, description: str) -> None:
2285+
"""rng1995 #541 P1: behavior prose needs a bounded condition + scope."""
2286+
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
2287+
assert findings == []
2288+
2289+
def test_description_catch_all_positive_still_tr3(self) -> None:
2290+
"""rng1995 #541 P1: bounded condition + scope still fires TR3."""
2291+
findings = sc_mod._analyze_triggers(
2292+
{"description": "Use this skill whenever the user sends any message"},
2293+
"myskill",
2294+
)
2295+
assert any(finding.rule_id == "TR3" for finding in findings)
2296+
2297+
def test_description_shadow_command_without_activation_wording(self) -> None:
2298+
"""rng1995 #541 P1: invocation intent alone opens the TR2 path."""
2299+
findings = sc_mod._analyze_triggers(
2300+
{"description": "Intercepts the /build command"},
2301+
"myskill",
2302+
)
2303+
assert "TR2" in {finding.rule_id for finding in findings}
2304+
2305+
@pytest.mark.parametrize("target_len", [120, 121])
2306+
def test_description_long_clause_window_still_analyzed(self, target_len: int) -> None:
2307+
"""rng1995 #541 P1: 120/121-char clauses keep start-of-clause intent."""
2308+
base = "Use this skill whenever the user sends any message"
2309+
description = f"{base} {'x' * (target_len - len(base) - 1)}"
2310+
assert len(description) == target_len
2311+
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
2312+
assert any(finding.rule_id == "TR3" for finding in findings)
2313+
2314+
@pytest.mark.parametrize("benign_count", [7, 8])
2315+
def test_description_trigger_clause_after_benign_padding(self, benign_count: int) -> None:
2316+
"""rng1995 #541 P1: benign sentences cannot push a trigger clause out."""
2317+
benign = ". ".join(
2318+
f"Benign capability sentence number {n}" for n in range(1, benign_count + 1)
2319+
)
2320+
description = f"{benign}. Use this skill whenever the user sends any message"
2321+
findings = sc_mod._analyze_triggers({"description": description}, "myskill")
2322+
assert any(finding.rule_id == "TR3" for finding in findings)
2323+
2324+
def test_description_broad_trigger_word_reaches_tr1(self) -> None:
2325+
"""rng1995 #541 P2: TR1 fires on the trigger phrase in a description."""
2326+
findings = sc_mod._analyze_triggers(
2327+
{"description": "Use this skill whenever the user says hello"},
2328+
"myskill",
2329+
)
2330+
tr1 = [finding for finding in findings if finding.rule_id == "TR1"]
2331+
assert len(tr1) == 1
2332+
assert "hello" in tr1[0].message
2333+
2334+
def test_description_tr1_from_skill_md_frontmatter(self, tmp_path) -> None:
2335+
"""rng1995 #541 P2: TR1 is reachable end to end from SKILL.md frontmatter."""
2336+
from skillspector.nodes.build_context import _parse_manifest
2337+
2338+
skill_dir = tmp_path / "hello-skill"
2339+
skill_dir.mkdir()
2340+
(skill_dir / "SKILL.md").write_text(
2341+
"---\n"
2342+
"name: hello-skill\n"
2343+
"description: Use this skill whenever the user says hello\n"
2344+
"---\n"
2345+
"# hello-skill\n",
2346+
encoding="utf-8",
2347+
)
2348+
manifest = _parse_manifest(skill_dir)
2349+
assert manifest["description"] == "Use this skill whenever the user says hello"
2350+
findings = sc_mod._analyze_triggers(manifest, "hello-skill")
2351+
assert any(finding.rule_id == "TR1" for finding in findings)
2352+
2353+
def test_description_capability_prose_reaches_no_trigger_rules(self) -> None:
2354+
"""rng1995 #541 P2: normal capability prose stays negative on TR1."""
2355+
findings = sc_mod._analyze_triggers(
2356+
{"description": "Helps developers review pull requests efficiently"},
2357+
"myskill",
2358+
)
2359+
assert findings == []
2360+
22712361

22722362
# ── Supply Chain Helpers ───────────────────────────────────────────────
22732363

0 commit comments

Comments
 (0)