Skip to content

Commit 61e9821

Browse files
committed
fix(static_yara): keep rule-set scope out of path-keyed accounting
The rule-load event for dropped YARA rules is labelled with the path `yara_rules`. Finalization groups reference outcomes and per-component coverage by path, so a benign, fully read file of that name linked from SKILL.md was charged with the rule set's partial outcome: a false HIGH AE1, risk score 25 and 50% coverage. Renaming the file made it vanish. Every relative path is also a legal file name, so no label can be made collision-free. Give these rows their own LedgerRecordType.RULE_SET and exclude them by type, not by name: - _reference_coverage_findings() ignores rule-set rows when deciding whether a referenced artifact was incompletely inspected. - finalize_ledger() does not fold rule-set targets into per-component coverage. - The public exception row carries scope="rule_set", which is part of the merge key so it never merges with a real file's row, and SARIF gives it no physical location. The scan stays a nonfatal partial scan, and --fail-on-incomplete still exits 1, because a rule really was dropped. Signed-off-by: Souptik Chakraborty <[email protected]>
1 parent 6abb623 commit 61e9821

6 files changed

Lines changed: 237 additions & 7 deletions

File tree

‎src/skillspector/inspection_ledger.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,15 @@ class LedgerRecordType(StrEnum):
3636
WORK_ITEM = "work_item"
3737
SYSTEM = "system"
3838
SCOPE_BOUNDARY = "scope_boundary"
39+
# An analyzer's own configuration (e.g. its YARA rule set), not a skill
40+
# artifact. Its ``path`` is only a report-safe label, and every relative
41+
# path is also a legal file name, so path-keyed accounting must exclude
42+
# these records by type -- no choice of label can be collision-free.
43+
RULE_SET = "rule_set"
44+
45+
46+
RULE_SET_SCOPE: Final = "rule_set"
47+
"""Public ``scope`` of exception rows describing a rule set rather than a file."""
3948

4049

4150
class LedgerReason(StrEnum):
@@ -261,6 +270,7 @@ class InspectionLedgerException(TypedDict):
261270
error_class: NotRequired[str]
262271
analyzers: NotRequired[list[str]]
263272
fatal: NotRequired[bool]
273+
scope: NotRequired[str]
264274

265275

266276
class AnalysisCompleteness(TypedDict):
@@ -553,6 +563,7 @@ def _exception(
553563
error_class: str | None = None,
554564
analyzers: Iterable[str] = (),
555565
fatal: bool,
566+
scope: str | None = None,
556567
) -> InspectionLedgerException:
557568
"""Build the public, safe projection of one exceptional ledger fact."""
558569
exception: InspectionLedgerException = {
@@ -570,9 +581,16 @@ def _exception(
570581
exception["analyzers"] = analyzer_ids
571582
if error_class:
572583
exception["error_class"] = error_class
584+
if scope:
585+
exception["scope"] = scope
573586
return exception
574587

575588

589+
def _is_rule_set_record(event: Mapping[str, object]) -> bool:
590+
"""Return whether a ledger row describes a rule set rather than an artifact."""
591+
return event.get("record_type") == LedgerRecordType.RULE_SET
592+
593+
576594
def _exception_from_event(
577595
event: InspectionLedgerEvent, *, fatal: bool
578596
) -> InspectionLedgerException:
@@ -593,6 +611,7 @@ def _exception_from_event(
593611
error_class=event.get("error_class"),
594612
analyzers=[str(event.get("analyzer_id", ""))],
595613
fatal=fatal,
614+
scope=RULE_SET_SCOPE if _is_rule_set_record(event) else None,
596615
)
597616

598617

@@ -611,6 +630,9 @@ def _merge_exception_projection(
611630
exception["start_line"],
612631
exception["end_line"],
613632
exception.get("error_class"),
633+
# A rule-set row and a real file's row can share a path label;
634+
# they must never merge into one public row.
635+
exception.get("scope"),
614636
)
615637
existing = grouped.get(key)
616638
if existing is None:
@@ -863,9 +885,17 @@ def accounting_error(path: object = None) -> None:
863885
ledger_exceptions = _merge_exception_projection(exceptional_rows)
864886
scope_exclusions = _merge_exception_projection(scope_rows)
865887

888+
# Rule-set rows describe an analyzer's configuration, not an artifact. Their
889+
# path is only a label, so folding them into per-component coverage would
890+
# charge the rule set's incompleteness to any real file with that name.
891+
rule_set_work_ids = {
892+
str(event.get("work_id", "")) for event in events if _is_rule_set_record(event)
893+
}
866894
per_component: dict[str, list[LedgerOutcome]] = {component: [] for component in components}
867895
if primary_targets:
868896
for _analyzer_id, target, matches in primary_targets:
897+
if str(target.get("work_id", "")) in rule_set_work_ids:
898+
continue
869899
path = _safe_path(target.get("path"), components)
870900
outcomes = per_component.setdefault(path, [])
871901
if len(matches) == 1:

‎src/skillspector/nodes/analyzers/static_yara.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1253,17 +1253,23 @@ def _rule_limit_response(
12531253
# both planned targets resolve to two matching events, and
12541254
# reconciliation raises a fatal ``unaccounted_work`` instead of
12551255
# the nonfatal partial scan this is meant to record. Falling
1256-
# back to ``system:static`` makes the identity disjoint from
1256+
# back to ``rule_set:static`` makes the identity disjoint from
12571257
# every analyzer work item by construction, so no choice of
12581258
# filename can collide -- renaming the synthetic path alone
12591259
# would only move the collision to the next unlucky name.
12601260
outcome=LedgerOutcome.PARTIAL,
1261-
record_type=LedgerRecordType.SYSTEM,
1261+
# RULE_SET, not SYSTEM: SYSTEM rows describe real artifacts
1262+
# and feed path-keyed accounting (per-component coverage and
1263+
# AE1 reference findings). This row describes the rule set,
1264+
# so finalization excludes it from that accounting by type,
1265+
# and a benign file named ``yara_rules`` cannot be charged
1266+
# with this partial outcome.
1267+
record_type=LedgerRecordType.RULE_SET,
12621268
phase="static",
1263-
# Not a scanned skill file: a synthetic scope for the rule
1264-
# set itself. Ledger paths must be relative POSIX paths, and
1265-
# the real rules directory (builtin or --yara-rules-dir) is
1266-
# absolute, so it cannot be used here.
1269+
# Not a scanned skill file: a report-safe label for the rule
1270+
# set. Ledger paths must be relative POSIX paths, and the real
1271+
# rules directory (builtin or --yara-rules-dir) is absolute, so
1272+
# it cannot be used here.
12671273
path="yara_rules/",
12681274
reason=LedgerReason.READ_ERROR,
12691275
observed_artifacts=rules_skipped,

‎src/skillspector/nodes/finalize_inspection_ledger.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ def _reference_coverage_findings(
3535
for event in state.get("inspection_ledger") or []:
3636
if not isinstance(event, Mapping):
3737
continue
38+
if event.get("record_type") == LedgerRecordType.RULE_SET:
39+
# A rule set's path is a label, not an artifact: a real file that
40+
# happens to share it was not inspected any less completely.
41+
continue
3842
outcome = str(event.get("outcome", ""))
3943
if outcome in {"partial", "failed", "out_of_scope"}:
4044
exceptional_outcomes.setdefault(str(event.get("path", "")), set()).add(outcome)

‎src/skillspector/nodes/report.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -723,8 +723,11 @@ def notification_from_exception(
723723
path = str(exception.get("path", ""))
724724
start_line = exception.get("start_line")
725725
end_line = exception.get("end_line")
726+
scope = exception.get("scope")
726727
locations = None
727-
if path:
728+
# A rule-set row's path is a label, not an artifact; giving it a
729+
# physical location would attribute it to any real file of that name.
730+
if path and not scope:
728731
region = (
729732
SarifRegion(
730733
startLine=int(start_line),
@@ -748,6 +751,8 @@ def notification_from_exception(
748751
}
749752
if exception.get("fatal") is not None:
750753
properties["fatal"] = bool(exception["fatal"])
754+
if scope:
755+
properties["scope"] = str(scope)
751756
analyzers = exception.get("analyzers")
752757
if isinstance(analyzers, list):
753758
properties["analyzers"] = list(analyzers)

‎tests/nodes/analyzers/test_static_yara.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
from unittest.mock import MagicMock
3131

3232
import pytest
33+
from typer.testing import CliRunner
3334

35+
from skillspector.cli import app
3436
from skillspector.inspection_ledger import LedgerReason
3537
from skillspector.nodes.analyzers import static_yara
3638
from skillspector.nodes.analyzers.static_runner import MAX_FILE_CHARS
@@ -1693,6 +1695,92 @@ def test_rule_load_event_does_not_collide_with_a_component_of_the_same_name(
16931695
for event in events
16941696
)
16951697

1698+
@pytest.mark.parametrize("referenced_name", ["yara_rules", "normal.txt"])
1699+
def test_referenced_file_named_like_the_rule_set_is_not_charged_with_its_dropped_rule(
1700+
self, tmp_path, referenced_name
1701+
):
1702+
"""A real file sharing the rule-set label must be scored like any other file.
1703+
1704+
The rule-load event is labelled ``yara_rules``. Finalization groups
1705+
reference outcomes and per-component coverage by path, so a benign,
1706+
fully read file of that name, linked from ``SKILL.md``, was charged with
1707+
the rule set's partial outcome: a false HIGH AE1, risk score 25 and 50%
1708+
coverage, all of which vanished when only the filename changed. The scan
1709+
must stay partial (a rule really was dropped), but nothing file-specific
1710+
may be inferred from the label. Driven through the real CLI so the
1711+
finalizer and report generation are both exercised.
1712+
"""
1713+
skill = tmp_path / "skill"
1714+
skill.mkdir()
1715+
(skill / "SKILL.md").write_text(
1716+
"---\nname: demo\ndescription: A harmless demo skill.\n---\n\n"
1717+
f"# Demo\n\nSee [the notes]({referenced_name}) for details.\n",
1718+
encoding="utf-8",
1719+
)
1720+
(skill / referenced_name).write_text("Plain harmless notes.\n", encoding="utf-8")
1721+
rules_dir = tmp_path / "rules"
1722+
rules_dir.mkdir()
1723+
(rules_dir / "valid.yar").write_text("rule never_fires { condition: false }\n")
1724+
(rules_dir / "broken.yar").write_text('rule broken { strings: $a = "x" condition: $a\n')
1725+
1726+
def scan(output_format: str, *extra: str) -> tuple[int, dict]:
1727+
out = tmp_path / f"report-{output_format}-{len(extra)}.json"
1728+
result = CliRunner().invoke(
1729+
app,
1730+
[
1731+
"scan",
1732+
str(skill),
1733+
"--no-llm",
1734+
"--yara-rules-dir",
1735+
str(rules_dir),
1736+
"--format",
1737+
output_format,
1738+
"--output",
1739+
str(out),
1740+
*extra,
1741+
],
1742+
)
1743+
return result.exit_code, json.loads(out.read_text(encoding="utf-8"))
1744+
1745+
exit_code, report = scan("json")
1746+
assert exit_code == 0
1747+
assert [issue["id"] for issue in report["issues"]] == [], (
1748+
"a rule-set failure must not invent a finding against a file of the same name"
1749+
)
1750+
assert report["risk_assessment"]["score"] == 0
1751+
completeness = report["analysis_completeness"]
1752+
assert completeness["coverage_percent"] == 100.0
1753+
assert completeness["partially_inspected_files"] == 0
1754+
assert completeness["entirely_uninspected_files"] == 0
1755+
1756+
# The dropped rule itself is still reported, as a nonfatal partial scan
1757+
# explicitly scoped to the rule set rather than to an artifact.
1758+
assert completeness["is_complete"] is False
1759+
assert completeness["execution_successful"] is True
1760+
rule_set_rows = [
1761+
row for row in completeness["ledger_exceptions"] if row.get("scope") == "rule_set"
1762+
]
1763+
assert len(rule_set_rows) == 1
1764+
assert rule_set_rows[0]["reason_code"] == LedgerReason.READ_ERROR
1765+
assert rule_set_rows[0]["fatal"] is False
1766+
assert all(
1767+
row.get("scope") == "rule_set"
1768+
for row in completeness["ledger_exceptions"]
1769+
if row["reason_code"] == LedgerReason.READ_ERROR
1770+
)
1771+
1772+
strict_exit, _ = scan("json", "--fail-on-incomplete")
1773+
assert strict_exit == 1
1774+
1775+
# SARIF must not point the rule-set notification at an artifact either.
1776+
_, sarif = scan("sarif")
1777+
notifications = sarif["runs"][0]["invocations"][0]["toolExecutionNotifications"]
1778+
rule_set_notifications = [
1779+
item for item in notifications if item.get("properties", {}).get("scope") == "rule_set"
1780+
]
1781+
assert len(rule_set_notifications) == 1
1782+
assert not rule_set_notifications[0].get("locations")
1783+
16961784
@pytest.mark.parametrize(
16971785
("filename", "content", "expected_fragment"),
16981786
[

‎tests/nodes/test_finalize_inspection_ledger.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -754,6 +754,103 @@ def test_resolved_reference_ae1_disposition_matrix_is_llm_independent(
754754
assert result["effective_finding_ids"] == []
755755

756756

757+
def test_rule_set_row_is_excluded_from_path_keyed_accounting_by_type() -> None:
758+
"""A rule-set label equal to a real file's path must not touch that file.
759+
760+
``components`` holds a fully read file whose path equals the rule-set label,
761+
and SKILL.md links to it. The rule-set row must keep the scan partial without
762+
lowering that file's coverage, synthesizing AE1, or merging with the file's
763+
own public exception row.
764+
"""
765+
file_event = ledger_event(
766+
analyzer_id="static_runner",
767+
outcome=LedgerOutcome.COMPLETED,
768+
phase="static",
769+
path="yara_rules",
770+
)
771+
rule_set_event = ledger_event(
772+
outcome=LedgerOutcome.PARTIAL,
773+
record_type=LedgerRecordType.RULE_SET,
774+
phase="static",
775+
path="yara_rules/",
776+
reason=LedgerReason.READ_ERROR,
777+
observed_artifacts=1,
778+
limit_artifacts=0,
779+
)
780+
# A real, same-path, same-reason row from the file itself must stay separate.
781+
real_file_row = ledger_event(
782+
outcome=LedgerOutcome.PARTIAL,
783+
record_type=LedgerRecordType.SYSTEM,
784+
phase="static",
785+
path="yara_rules",
786+
reason=LedgerReason.READ_ERROR,
787+
)
788+
assert rule_set_event["path"] == file_event["path"] == "yara_rules"
789+
state: SkillspectorState = {
790+
"components": ["SKILL.md", "yara_rules"],
791+
"findings": [],
792+
"effective_finding_ids": [],
793+
"artifact_references": [
794+
{
795+
"source_path": "SKILL.md",
796+
"line": 3,
797+
"column": 5,
798+
"evidence": "See [the notes](yara_rules).",
799+
"target_path": "yara_rules",
800+
"status": "resolved",
801+
"disposition": "complete",
802+
}
803+
],
804+
"inspection_ledger": [
805+
ledger_event(
806+
analyzer_id="static_runner",
807+
outcome=LedgerOutcome.COMPLETED,
808+
phase="static",
809+
path="SKILL.md",
810+
),
811+
file_event,
812+
rule_set_event,
813+
],
814+
"analyzer_status_events": [
815+
analyzer_status_event(
816+
analyzer_id="static_runner",
817+
status="completed",
818+
planned_work=[
819+
_target(
820+
inspection_work_id("static_runner", "SKILL.md", None, None), "SKILL.md"
821+
),
822+
_target(file_event["work_id"], "yara_rules"),
823+
],
824+
),
825+
analyzer_status_event(
826+
analyzer_id="static_yara",
827+
status="degraded",
828+
planned_work=[_target(rule_set_event["work_id"], "yara_rules")],
829+
),
830+
],
831+
}
832+
833+
result = finalize_inspection_ledger(state)
834+
835+
assert [finding.rule_id for finding in result["findings"]] == []
836+
completeness = result["analysis_completeness"]
837+
assert completeness["coverage_percent"] == 100.0
838+
assert completeness["fully_inspected_files"] == 2
839+
assert completeness["partially_inspected_files"] == 0
840+
assert completeness["is_complete"] is False
841+
assert completeness["execution_successful"] is True
842+
assert [row.get("scope") for row in completeness["ledger_exceptions"]] == ["rule_set"]
843+
844+
# Control: the same label on a SYSTEM row is real artifact evidence and must
845+
# still drive both AE1 and coverage, and must not merge with the rule-set row.
846+
control_state = dict(state)
847+
control_state["inspection_ledger"] = [*state["inspection_ledger"], real_file_row]
848+
control = finalize_inspection_ledger(control_state)
849+
assert [finding.rule_id for finding in control["findings"]] == ["AE1"]
850+
rows = control["analysis_completeness"]["ledger_exceptions"]
851+
assert sorted(str(row.get("scope")) for row in rows) == ["None", "rule_set"]
852+
853+
757854
@pytest.mark.parametrize(
758855
("disposition", "reason", "expected_ae7"),
759856
[

0 commit comments

Comments
 (0)