Skip to content

Commit ec6633e

Browse files
authored
Merge pull request #661 from NVIDIA/naren/fix-ae1-missing-reference-source
fix(ae1): keep reference-resolution caveats from flagging SKILL.md self-references
2 parents 8831219 + 9db69b2 commit ec6633e

3 files changed

Lines changed: 230 additions & 0 deletions

File tree

‎src/skillspector/nodes/finalize_inspection_ledger.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
}
4848
_MAX_PASSIVE_PNG_CHUNKS = 4_096
4949
_MAX_REFERENCE_REASONS = 16
50+
_REFERENCE_CAVEAT_REASONS = (LedgerReason.REFERENCE_MISSING, LedgerReason.REFERENCE_UNRESOLVED)
5051
_REFERENCE_LIMIT_FIELDS = frozenset(
5152
f"{prefix}_{unit}"
5253
for prefix in ("observed", "limit")
@@ -531,6 +532,18 @@ def _reference_coverage_findings(
531532
else []
532533
)
533534
for event in ledger_events:
535+
# Reference resolution records a mention of a path that the bundle does
536+
# not carry, or that matches more than one bundled file, as a partial
537+
# event on the file that contains the mention. Neither leaves bytes of
538+
# that file uninspected. The completeness ledger still reports them;
539+
# they must not turn a reference to that file (such as `SKILL.md`)
540+
# into AE1.
541+
if (
542+
event.get("outcome") == LedgerOutcome.PARTIAL
543+
and event.get("phase") == "reference_resolution"
544+
and str(event.get("reason_code")) in _REFERENCE_CAVEAT_REASONS
545+
):
546+
continue
534547
events_by_path.setdefault(str(event.get("path", "")), []).append(event)
535548
status_shape_complete, invalid_status_paths = _status_paths_with_incomplete_evidence(
536549
state.get("analyzer_status_events"), ledger_events

‎tests/nodes/test_finalize_inspection_ledger.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import skillspector.nodes.report as report_module
1818
import skillspector.state as state_module
1919
from skillspector.inspection_ledger import (
20+
InspectionLedgerEvent,
2021
LedgerOutcome,
2122
LedgerReason,
2223
LedgerRecordType,
@@ -1907,6 +1908,139 @@ def test_unresolved_reference_does_not_synthesize_ae1(status: str) -> None:
19071908
assert result["effective_finding_ids"] == []
19081909

19091910

1911+
def _reference_caveat(
1912+
reason: LedgerReason,
1913+
*,
1914+
outcome: LedgerOutcome = LedgerOutcome.PARTIAL,
1915+
phase: str = "reference_resolution",
1916+
) -> InspectionLedgerEvent:
1917+
"""Build the event reference resolution records on SKILL.md for an unresolved mention."""
1918+
return ledger_event(
1919+
outcome=outcome,
1920+
record_type=LedgerRecordType.SYSTEM,
1921+
phase=phase,
1922+
path="SKILL.md",
1923+
start_line=3,
1924+
end_line=3,
1925+
reason=reason,
1926+
)
1927+
1928+
1929+
def _self_reference_state(
1930+
*events: InspectionLedgerEvent, status: str = "missing"
1931+
) -> SkillspectorState:
1932+
"""SKILL.md mentions an unresolved path and then references itself."""
1933+
return {
1934+
"artifact_inventory": [{"path": "SKILL.md", "disposition": "analyzed"}],
1935+
"artifact_references": [
1936+
{
1937+
"source_path": "SKILL.md",
1938+
"line": 3,
1939+
"evidence": "templates/config.yaml",
1940+
"target_path": None,
1941+
"status": status,
1942+
},
1943+
{
1944+
"source_path": "SKILL.md",
1945+
"line": 5,
1946+
"evidence": "Keep `SKILL.md` under 500 lines.",
1947+
"target_path": "SKILL.md",
1948+
"status": "resolved",
1949+
"disposition": "analyzed",
1950+
},
1951+
],
1952+
"inspection_ledger": list(events),
1953+
}
1954+
1955+
1956+
_REFERENCE_CAVEAT_CASES = [
1957+
pytest.param("missing", LedgerReason.REFERENCE_MISSING, id="missing"),
1958+
pytest.param("ambiguous", LedgerReason.REFERENCE_UNRESOLVED, id="ambiguous"),
1959+
]
1960+
1961+
1962+
@pytest.mark.parametrize(("status", "reason"), _REFERENCE_CAVEAT_CASES)
1963+
def test_reference_caveat_does_not_make_its_source_an_ae1_target(
1964+
status: str, reason: LedgerReason
1965+
) -> None:
1966+
state = _self_reference_state(_reference_caveat(reason), status=status)
1967+
1968+
assert finalizer_module._reference_coverage_findings(state) == []
1969+
1970+
1971+
@pytest.mark.parametrize(("status", "reason"), _REFERENCE_CAVEAT_CASES)
1972+
def test_self_reference_ae1_still_reports_source_content_limitations(
1973+
status: str, reason: LedgerReason
1974+
) -> None:
1975+
findings = finalizer_module._reference_coverage_findings(
1976+
_self_reference_state(
1977+
_reference_caveat(reason),
1978+
ledger_event(
1979+
outcome=LedgerOutcome.PARTIAL,
1980+
phase="static",
1981+
analyzer_id="static_patterns_tool_misuse",
1982+
path="SKILL.md",
1983+
reason=LedgerReason.STATIC_PARSE_LIMIT,
1984+
start_line=4,
1985+
end_line=4,
1986+
),
1987+
status=status,
1988+
)
1989+
)
1990+
1991+
assert [(finding.rule_id, finding.start_line) for finding in findings] == [("AE1", 5)]
1992+
reasons = findings[0].to_dict()["evidence"]["reasons"]
1993+
assert [row["reason_code"] for row in reasons] == ["static_parse_limit"]
1994+
1995+
1996+
@pytest.mark.parametrize(
1997+
("outcome", "phase", "reason"),
1998+
[
1999+
pytest.param(
2000+
LedgerOutcome.FAILED,
2001+
"reference_resolution",
2002+
LedgerReason.REFERENCE_MISSING,
2003+
id="failed-missing",
2004+
),
2005+
pytest.param(
2006+
LedgerOutcome.FAILED,
2007+
"reference_resolution",
2008+
LedgerReason.REFERENCE_UNRESOLVED,
2009+
id="failed-unresolved",
2010+
),
2011+
pytest.param(
2012+
LedgerOutcome.PARTIAL,
2013+
"static",
2014+
LedgerReason.REFERENCE_MISSING,
2015+
id="other-phase-missing",
2016+
),
2017+
pytest.param(
2018+
LedgerOutcome.PARTIAL,
2019+
"cache",
2020+
LedgerReason.REFERENCE_UNRESOLVED,
2021+
id="other-phase-unresolved",
2022+
),
2023+
pytest.param(
2024+
LedgerOutcome.PARTIAL,
2025+
"reference_resolution",
2026+
LedgerReason.REFERENCE_EXTRACTION_LIMIT,
2027+
id="extraction-limit",
2028+
),
2029+
],
2030+
)
2031+
def test_other_reference_event_shapes_still_yield_self_reference_ae1(
2032+
outcome: LedgerOutcome, phase: str, reason: LedgerReason
2033+
) -> None:
2034+
findings = finalizer_module._reference_coverage_findings(
2035+
_self_reference_state(_reference_caveat(reason, outcome=outcome, phase=phase))
2036+
)
2037+
2038+
assert [(finding.rule_id, finding.start_line) for finding in findings] == [("AE1", 5)]
2039+
evidence = findings[0].to_dict()["evidence"]
2040+
assert evidence["target_disposition"] == outcome.value
2041+
assert [row["reason_code"] for row in evidence["reasons"]] == [reason.value]
2042+
2043+
19102044
def test_guard_analyzer_node_converts_unexpected_exception_to_fatal_facts() -> None:
19112045
def broken_node(_state: SkillspectorState) -> AnalyzerNodeResponse:
19122046
raise RuntimeError("provider detail must remain private")

‎tests/nodes/test_security_remediation.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1732,6 +1732,89 @@ def test_unresolved_primary_reference_blocks_complete_verdict(tmp_path: Path, ca
17321732
assert result["risk_recommendation"] != "SAFE"
17331733

17341734

1735+
@pytest.mark.parametrize(
1736+
("primary", "mentions", "status", "reason"),
1737+
[
1738+
pytest.param(
1739+
"SKILL.md",
1740+
"Other skills keep templates under `templates/config.yaml`.\n"
1741+
"Keep `SKILL.md` under 500 lines.\n",
1742+
"missing",
1743+
"reference_missing",
1744+
id="backticked-self-reference",
1745+
),
1746+
pytest.param(
1747+
"SKILL.md",
1748+
"Other skills keep templates under `templates/config.yaml`.\n"
1749+
"Keep [this file](SKILL.md) under 500 lines.\n",
1750+
"missing",
1751+
"reference_missing",
1752+
id="markdown-link-self-reference",
1753+
),
1754+
pytest.param(
1755+
"SKILL.md",
1756+
"Other skills keep templates in [config](templates/config.yaml).\n"
1757+
"Keep `SKILL.md` under 500 lines.\n",
1758+
"missing",
1759+
"reference_missing",
1760+
id="markdown-link-missing-target",
1761+
),
1762+
pytest.param(
1763+
"skill.md",
1764+
"Other skills keep templates under `templates/config.yaml`.\n"
1765+
"Keep `skill.md` under 500 lines.\n",
1766+
"missing",
1767+
"reference_missing",
1768+
id="lowercase-primary",
1769+
),
1770+
pytest.param(
1771+
"SKILL.md",
1772+
"Edit `config.yaml` before running.\nKeep `SKILL.md` under 500 lines.\n",
1773+
"ambiguous",
1774+
"reference_unresolved",
1775+
id="ambiguous-reference",
1776+
),
1777+
],
1778+
)
1779+
def test_reference_caveat_does_not_turn_self_reference_into_ae1(
1780+
tmp_path: Path, primary: str, mentions: str, status: str, reason: str
1781+
) -> None:
1782+
references = tmp_path / "references"
1783+
references.mkdir()
1784+
(references / "guide.md").write_text("# Guide\n\nPlain guide text.\n", encoding="utf-8")
1785+
if status == "ambiguous":
1786+
for subdirectory in ("a", "b"):
1787+
(tmp_path / subdirectory).mkdir()
1788+
(tmp_path / subdirectory / "config.yaml").write_text("mode: local\n", encoding="utf-8")
1789+
(tmp_path / primary).write_text(
1790+
f"# Skill\n\nRead [the guide](references/guide.md).\n{mentions}",
1791+
encoding="utf-8",
1792+
)
1793+
1794+
result = graph.invoke(
1795+
{
1796+
"input_path": str(tmp_path),
1797+
"output_format": "json",
1798+
"use_llm": False,
1799+
}
1800+
)
1801+
1802+
statuses = Counter(
1803+
(reference["status"], reference["target_path"])
1804+
for reference in result["artifact_references"]
1805+
)
1806+
assert statuses[("resolved", primary)] == 1
1807+
assert statuses[(status, None)] == 1
1808+
assert not any(finding.rule_id == "AE1" for finding in result["filtered_findings"])
1809+
# The unresolved mention is still reported as a completeness caveat.
1810+
assert [
1811+
(row["path"], row["reason_code"])
1812+
for row in result["analysis_completeness"]["ledger_exceptions"]
1813+
] == [(primary, reason)]
1814+
assert result["analysis_completeness"]["is_complete"] is False
1815+
assert result["risk_recommendation"] == "CAUTION"
1816+
1817+
17351818
@pytest.mark.asyncio
17361819
async def test_unresolved_reference_caveat_does_not_block_mcp_install(tmp_path: Path) -> None:
17371820
"""A reference caveat hides no bytes, so it must not fail safe_to_install.

0 commit comments

Comments
 (0)