|
17 | 17 | import skillspector.nodes.report as report_module |
18 | 18 | import skillspector.state as state_module |
19 | 19 | from skillspector.inspection_ledger import ( |
| 20 | + InspectionLedgerEvent, |
20 | 21 | LedgerOutcome, |
21 | 22 | LedgerReason, |
22 | 23 | LedgerRecordType, |
@@ -1907,6 +1908,139 @@ def test_unresolved_reference_does_not_synthesize_ae1(status: str) -> None: |
1907 | 1908 | assert result["effective_finding_ids"] == [] |
1908 | 1909 |
|
1909 | 1910 |
|
| 1911 | +def _reference_caveat( |
| 1912 | + reason: LedgerReason, |
| 1913 | + *, |
| 1914 | + outcome: LedgerOutcome = LedgerOutcome.PARTIAL, |
| 1915 | + phase: str = "reference_resolution", |
| 1916 | +) -> InspectionLedgerEvent: |
| 1917 | + """Build the event reference resolution records on SKILL.md for an unresolved mention.""" |
| 1918 | + return ledger_event( |
| 1919 | + outcome=outcome, |
| 1920 | + record_type=LedgerRecordType.SYSTEM, |
| 1921 | + phase=phase, |
| 1922 | + path="SKILL.md", |
| 1923 | + start_line=3, |
| 1924 | + end_line=3, |
| 1925 | + reason=reason, |
| 1926 | + ) |
| 1927 | + |
| 1928 | + |
| 1929 | +def _self_reference_state( |
| 1930 | + *events: InspectionLedgerEvent, status: str = "missing" |
| 1931 | +) -> SkillspectorState: |
| 1932 | + """SKILL.md mentions an unresolved path and then references itself.""" |
| 1933 | + return { |
| 1934 | + "artifact_inventory": [{"path": "SKILL.md", "disposition": "analyzed"}], |
| 1935 | + "artifact_references": [ |
| 1936 | + { |
| 1937 | + "source_path": "SKILL.md", |
| 1938 | + "line": 3, |
| 1939 | + "evidence": "templates/config.yaml", |
| 1940 | + "target_path": None, |
| 1941 | + "status": status, |
| 1942 | + }, |
| 1943 | + { |
| 1944 | + "source_path": "SKILL.md", |
| 1945 | + "line": 5, |
| 1946 | + "evidence": "Keep `SKILL.md` under 500 lines.", |
| 1947 | + "target_path": "SKILL.md", |
| 1948 | + "status": "resolved", |
| 1949 | + "disposition": "analyzed", |
| 1950 | + }, |
| 1951 | + ], |
| 1952 | + "inspection_ledger": list(events), |
| 1953 | + } |
| 1954 | + |
| 1955 | + |
| 1956 | +_REFERENCE_CAVEAT_CASES = [ |
| 1957 | + pytest.param("missing", LedgerReason.REFERENCE_MISSING, id="missing"), |
| 1958 | + pytest.param("ambiguous", LedgerReason.REFERENCE_UNRESOLVED, id="ambiguous"), |
| 1959 | +] |
| 1960 | + |
| 1961 | + |
| 1962 | +@pytest.mark.parametrize(("status", "reason"), _REFERENCE_CAVEAT_CASES) |
| 1963 | +def test_reference_caveat_does_not_make_its_source_an_ae1_target( |
| 1964 | + status: str, reason: LedgerReason |
| 1965 | +) -> None: |
| 1966 | + state = _self_reference_state(_reference_caveat(reason), status=status) |
| 1967 | + |
| 1968 | + assert finalizer_module._reference_coverage_findings(state) == [] |
| 1969 | + |
| 1970 | + |
| 1971 | +@pytest.mark.parametrize(("status", "reason"), _REFERENCE_CAVEAT_CASES) |
| 1972 | +def test_self_reference_ae1_still_reports_source_content_limitations( |
| 1973 | + status: str, reason: LedgerReason |
| 1974 | +) -> None: |
| 1975 | + findings = finalizer_module._reference_coverage_findings( |
| 1976 | + _self_reference_state( |
| 1977 | + _reference_caveat(reason), |
| 1978 | + ledger_event( |
| 1979 | + outcome=LedgerOutcome.PARTIAL, |
| 1980 | + phase="static", |
| 1981 | + analyzer_id="static_patterns_tool_misuse", |
| 1982 | + path="SKILL.md", |
| 1983 | + reason=LedgerReason.STATIC_PARSE_LIMIT, |
| 1984 | + start_line=4, |
| 1985 | + end_line=4, |
| 1986 | + ), |
| 1987 | + status=status, |
| 1988 | + ) |
| 1989 | + ) |
| 1990 | + |
| 1991 | + assert [(finding.rule_id, finding.start_line) for finding in findings] == [("AE1", 5)] |
| 1992 | + reasons = findings[0].to_dict()["evidence"]["reasons"] |
| 1993 | + assert [row["reason_code"] for row in reasons] == ["static_parse_limit"] |
| 1994 | + |
| 1995 | + |
| 1996 | +@pytest.mark.parametrize( |
| 1997 | + ("outcome", "phase", "reason"), |
| 1998 | + [ |
| 1999 | + pytest.param( |
| 2000 | + LedgerOutcome.FAILED, |
| 2001 | + "reference_resolution", |
| 2002 | + LedgerReason.REFERENCE_MISSING, |
| 2003 | + id="failed-missing", |
| 2004 | + ), |
| 2005 | + pytest.param( |
| 2006 | + LedgerOutcome.FAILED, |
| 2007 | + "reference_resolution", |
| 2008 | + LedgerReason.REFERENCE_UNRESOLVED, |
| 2009 | + id="failed-unresolved", |
| 2010 | + ), |
| 2011 | + pytest.param( |
| 2012 | + LedgerOutcome.PARTIAL, |
| 2013 | + "static", |
| 2014 | + LedgerReason.REFERENCE_MISSING, |
| 2015 | + id="other-phase-missing", |
| 2016 | + ), |
| 2017 | + pytest.param( |
| 2018 | + LedgerOutcome.PARTIAL, |
| 2019 | + "cache", |
| 2020 | + LedgerReason.REFERENCE_UNRESOLVED, |
| 2021 | + id="other-phase-unresolved", |
| 2022 | + ), |
| 2023 | + pytest.param( |
| 2024 | + LedgerOutcome.PARTIAL, |
| 2025 | + "reference_resolution", |
| 2026 | + LedgerReason.REFERENCE_EXTRACTION_LIMIT, |
| 2027 | + id="extraction-limit", |
| 2028 | + ), |
| 2029 | + ], |
| 2030 | +) |
| 2031 | +def test_other_reference_event_shapes_still_yield_self_reference_ae1( |
| 2032 | + outcome: LedgerOutcome, phase: str, reason: LedgerReason |
| 2033 | +) -> None: |
| 2034 | + findings = finalizer_module._reference_coverage_findings( |
| 2035 | + _self_reference_state(_reference_caveat(reason, outcome=outcome, phase=phase)) |
| 2036 | + ) |
| 2037 | + |
| 2038 | + assert [(finding.rule_id, finding.start_line) for finding in findings] == [("AE1", 5)] |
| 2039 | + evidence = findings[0].to_dict()["evidence"] |
| 2040 | + assert evidence["target_disposition"] == outcome.value |
| 2041 | + assert [row["reason_code"] for row in evidence["reasons"]] == [reason.value] |
| 2042 | + |
| 2043 | + |
1910 | 2044 | def test_guard_analyzer_node_converts_unexpected_exception_to_fatal_facts() -> None: |
1911 | 2045 | def broken_node(_state: SkillspectorState) -> AnalyzerNodeResponse: |
1912 | 2046 | raise RuntimeError("provider detail must remain private") |
|
0 commit comments