|
30 | 30 | from unittest.mock import MagicMock |
31 | 31 |
|
32 | 32 | import pytest |
| 33 | +from typer.testing import CliRunner |
33 | 34 |
|
| 35 | +from skillspector.cli import app |
34 | 36 | from skillspector.inspection_ledger import LedgerReason |
35 | 37 | from skillspector.nodes.analyzers import static_yara |
36 | 38 | from skillspector.nodes.analyzers.static_runner import MAX_FILE_CHARS |
@@ -1693,6 +1695,92 @@ def test_rule_load_event_does_not_collide_with_a_component_of_the_same_name( |
1693 | 1695 | for event in events |
1694 | 1696 | ) |
1695 | 1697 |
|
| 1698 | + @pytest.mark.parametrize("referenced_name", ["yara_rules", "normal.txt"]) |
| 1699 | + def test_referenced_file_named_like_the_rule_set_is_not_charged_with_its_dropped_rule( |
| 1700 | + self, tmp_path, referenced_name |
| 1701 | + ): |
| 1702 | + """A real file sharing the rule-set label must be scored like any other file. |
| 1703 | +
|
| 1704 | + The rule-load event is labelled ``yara_rules``. Finalization groups |
| 1705 | + reference outcomes and per-component coverage by path, so a benign, |
| 1706 | + fully read file of that name, linked from ``SKILL.md``, was charged with |
| 1707 | + the rule set's partial outcome: a false HIGH AE1, risk score 25 and 50% |
| 1708 | + coverage, all of which vanished when only the filename changed. The scan |
| 1709 | + must stay partial (a rule really was dropped), but nothing file-specific |
| 1710 | + may be inferred from the label. Driven through the real CLI so the |
| 1711 | + finalizer and report generation are both exercised. |
| 1712 | + """ |
| 1713 | + skill = tmp_path / "skill" |
| 1714 | + skill.mkdir() |
| 1715 | + (skill / "SKILL.md").write_text( |
| 1716 | + "---\nname: demo\ndescription: A harmless demo skill.\n---\n\n" |
| 1717 | + f"# Demo\n\nSee [the notes]({referenced_name}) for details.\n", |
| 1718 | + encoding="utf-8", |
| 1719 | + ) |
| 1720 | + (skill / referenced_name).write_text("Plain harmless notes.\n", encoding="utf-8") |
| 1721 | + rules_dir = tmp_path / "rules" |
| 1722 | + rules_dir.mkdir() |
| 1723 | + (rules_dir / "valid.yar").write_text("rule never_fires { condition: false }\n") |
| 1724 | + (rules_dir / "broken.yar").write_text('rule broken { strings: $a = "x" condition: $a\n') |
| 1725 | + |
| 1726 | + def scan(output_format: str, *extra: str) -> tuple[int, dict]: |
| 1727 | + out = tmp_path / f"report-{output_format}-{len(extra)}.json" |
| 1728 | + result = CliRunner().invoke( |
| 1729 | + app, |
| 1730 | + [ |
| 1731 | + "scan", |
| 1732 | + str(skill), |
| 1733 | + "--no-llm", |
| 1734 | + "--yara-rules-dir", |
| 1735 | + str(rules_dir), |
| 1736 | + "--format", |
| 1737 | + output_format, |
| 1738 | + "--output", |
| 1739 | + str(out), |
| 1740 | + *extra, |
| 1741 | + ], |
| 1742 | + ) |
| 1743 | + return result.exit_code, json.loads(out.read_text(encoding="utf-8")) |
| 1744 | + |
| 1745 | + exit_code, report = scan("json") |
| 1746 | + assert exit_code == 0 |
| 1747 | + assert [issue["id"] for issue in report["issues"]] == [], ( |
| 1748 | + "a rule-set failure must not invent a finding against a file of the same name" |
| 1749 | + ) |
| 1750 | + assert report["risk_assessment"]["score"] == 0 |
| 1751 | + completeness = report["analysis_completeness"] |
| 1752 | + assert completeness["coverage_percent"] == 100.0 |
| 1753 | + assert completeness["partially_inspected_files"] == 0 |
| 1754 | + assert completeness["entirely_uninspected_files"] == 0 |
| 1755 | + |
| 1756 | + # The dropped rule itself is still reported, as a nonfatal partial scan |
| 1757 | + # explicitly scoped to the rule set rather than to an artifact. |
| 1758 | + assert completeness["is_complete"] is False |
| 1759 | + assert completeness["execution_successful"] is True |
| 1760 | + rule_set_rows = [ |
| 1761 | + row for row in completeness["ledger_exceptions"] if row.get("scope") == "rule_set" |
| 1762 | + ] |
| 1763 | + assert len(rule_set_rows) == 1 |
| 1764 | + assert rule_set_rows[0]["reason_code"] == LedgerReason.READ_ERROR |
| 1765 | + assert rule_set_rows[0]["fatal"] is False |
| 1766 | + assert all( |
| 1767 | + row.get("scope") == "rule_set" |
| 1768 | + for row in completeness["ledger_exceptions"] |
| 1769 | + if row["reason_code"] == LedgerReason.READ_ERROR |
| 1770 | + ) |
| 1771 | + |
| 1772 | + strict_exit, _ = scan("json", "--fail-on-incomplete") |
| 1773 | + assert strict_exit == 1 |
| 1774 | + |
| 1775 | + # SARIF must not point the rule-set notification at an artifact either. |
| 1776 | + _, sarif = scan("sarif") |
| 1777 | + notifications = sarif["runs"][0]["invocations"][0]["toolExecutionNotifications"] |
| 1778 | + rule_set_notifications = [ |
| 1779 | + item for item in notifications if item.get("properties", {}).get("scope") == "rule_set" |
| 1780 | + ] |
| 1781 | + assert len(rule_set_notifications) == 1 |
| 1782 | + assert not rule_set_notifications[0].get("locations") |
| 1783 | + |
1696 | 1784 | @pytest.mark.parametrize( |
1697 | 1785 | ("filename", "content", "expected_fragment"), |
1698 | 1786 | [ |
|
0 commit comments