Skip to content

Commit d51fa57

Browse files
committed
fix(cli): keep exit code 2 for a bad --baseline in recursive scans
Threading --baseline into the recursive path (previous commit) exposed an error-contract break: _scan_multi_skill() was dispatched before scan()'s try/except, and _scan_state() called load_baseline() outside the per-skill try. A missing or malformed baseline therefore escaped as an uncaught traceback and exited 1, the "risk found" code, while the single-skill path printed a clean error and exited 2. That is worse than the bug it replaced: on the base commit the same command exited 0 because --baseline was silently dropped. In CI an operator typo in a baseline path became indistinguishable from a DO_NOT_INSTALL verdict. Load the baseline once in scan(), inside the existing try, and move the recursive dispatch inside that try as well. _scan_state() and _scan_multi_skill() now take an already-loaded Baseline rather than a Path, so the file is also no longer re-read and re-parsed once per sub-skill. Restores the contract documented in docs/SUPPRESSION.md ("A missing or malformed baseline file exits with code 2") and README's exit-code table. Adds three regression tests: recursive missing baseline exits 2, recursive malformed baseline exits 2, and the terminal summary's Findings column excludes suppressed findings. The last one was previously unpinned; reverting that line to the pre-suppression form passed the entire suite. Signed-off-by: Werner Kasselman <[email protected]>
1 parent e9ecd0b commit d51fa57

2 files changed

Lines changed: 169 additions & 34 deletions

File tree

‎src/skillspector/cli.py‎

Lines changed: 56 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,12 @@
3838
from skillspector.graph import graph
3939
from skillspector.logging_config import get_logger, set_level
4040
from skillspector.multi_skill import MultiSkillDetectionResult, detect_skills
41-
from skillspector.suppression import build_baseline_dict, dump_baseline, load_baseline
41+
from skillspector.suppression import (
42+
Baseline,
43+
build_baseline_dict,
44+
dump_baseline,
45+
load_baseline,
46+
)
4247

4348
logger = get_logger(__name__)
4449

@@ -122,10 +127,16 @@ def _scan_state(
122127
format: FormatChoice,
123128
no_llm: bool,
124129
yara_rules_dir: str | None = None,
125-
baseline: Path | None = None,
130+
baseline: Baseline | None = None,
126131
show_suppressed: bool = False,
127132
) -> dict[str, object]:
128-
"""Build initial graph state from scan CLI args."""
133+
"""Build initial graph state from scan CLI args.
134+
135+
*baseline* is an already-loaded :class:`Baseline`, not a path. Loading is done
136+
once in ``scan()`` inside its ``try`` so a missing or malformed baseline is
137+
reported as a clean error with exit code 2 on every path (single and
138+
recursive), and so the file is not re-read and re-parsed per sub-skill.
139+
"""
129140
state: dict[str, object] = {
130141
"input_path": input_path,
131142
"output_format": format.value,
@@ -134,8 +145,7 @@ def _scan_state(
134145
if yara_rules_dir is not None:
135146
state["yara_rules_dir"] = yara_rules_dir
136147
if baseline is not None:
137-
# Loading may raise FileNotFoundError/ValueError, mapped to exit code 2 by scan().
138-
state["baseline"] = load_baseline(baseline)
148+
state["baseline"] = baseline
139149
state["show_suppressed"] = show_suppressed
140150
return state
141151

@@ -288,42 +298,50 @@ def scan(
288298
set_level("DEBUG")
289299

290300
resolved_path = Path(input_path).resolve()
291-
if recursive and resolved_path.is_dir():
292-
detection = detect_skills(resolved_path)
293-
if detection.is_multi_skill:
294-
_scan_multi_skill(
295-
detection,
296-
format,
297-
output,
298-
no_llm,
299-
yara_rules_dir,
300-
verbose,
301-
baseline=baseline,
302-
show_suppressed=show_suppressed,
303-
)
304-
return
305-
if not detection.has_root_skill and len(detection.skills) == 0:
306-
console.print(
307-
"[yellow]Warning:[/yellow] --recursive specified but no sub-skills "
308-
"detected. Scanning as single skill."
309-
)
310-
elif resolved_path.is_dir():
311-
detection = detect_skills(resolved_path)
312-
if detection.is_multi_skill:
313-
console.print(
314-
f"[yellow]Warning:[/yellow] Found {len(detection.skills)} skills in "
315-
f"this directory. Use --recursive to scan each independently."
316-
)
317301

318302
result = None
319303
try:
304+
# Load once, here, inside the try: load_baseline() raises
305+
# FileNotFoundError/ValueError, which the handlers below map to exit code 2.
306+
# The recursive dispatch is inside this try for the same reason -- when it sat
307+
# outside, a bad --baseline escaped as an uncaught traceback and exited 1 (the
308+
# "risk found" code), contradicting docs/SUPPRESSION.md and the single-skill path.
309+
loaded_baseline = load_baseline(baseline) if baseline is not None else None
310+
311+
if recursive and resolved_path.is_dir():
312+
detection = detect_skills(resolved_path)
313+
if detection.is_multi_skill:
314+
_scan_multi_skill(
315+
detection,
316+
format,
317+
output,
318+
no_llm,
319+
yara_rules_dir,
320+
verbose,
321+
baseline=loaded_baseline,
322+
show_suppressed=show_suppressed,
323+
)
324+
return
325+
if not detection.has_root_skill and len(detection.skills) == 0:
326+
console.print(
327+
"[yellow]Warning:[/yellow] --recursive specified but no sub-skills "
328+
"detected. Scanning as single skill."
329+
)
330+
elif resolved_path.is_dir():
331+
detection = detect_skills(resolved_path)
332+
if detection.is_multi_skill:
333+
console.print(
334+
f"[yellow]Warning:[/yellow] Found {len(detection.skills)} skills in "
335+
f"this directory. Use --recursive to scan each independently."
336+
)
337+
320338
yara_dir = str(yara_rules_dir.resolve()) if yara_rules_dir else None
321339
state = _scan_state(
322340
input_path,
323341
format,
324342
no_llm,
325343
yara_rules_dir=yara_dir,
326-
baseline=baseline,
344+
baseline=loaded_baseline,
327345
show_suppressed=show_suppressed,
328346
)
329347
if verbose:
@@ -406,10 +424,14 @@ def _scan_multi_skill(
406424
no_llm: bool,
407425
yara_rules_dir: Path | None,
408426
verbose: bool,
409-
baseline: Path | None = None,
427+
baseline: Baseline | None = None,
410428
show_suppressed: bool = False,
411429
) -> None:
412-
"""Scan each detected sub-skill independently and produce a combined report."""
430+
"""Scan each detected sub-skill independently and produce a combined report.
431+
432+
*baseline* is already loaded by ``scan()``; it is reused across sub-skills rather
433+
than re-read and re-parsed once per skill.
434+
"""
413435
skills = detection.skills
414436
console.print(f"[bold]Multi-skill directory detected:[/bold] {len(skills)} skills found\n")
415437

‎tests/unit/test_cli.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -659,3 +659,116 @@ def test_result_finding_count_branches() -> None:
659659
# Nothing usable -> 0. Malformed (suppressed longer than filtered) floors at 0.
660660
assert _result_finding_count({}) == 0
661661
assert _result_finding_count({"filtered_findings": [1], "suppressed_findings": [1, 2, 3]}) == 0
662+
663+
664+
def test_cli_recursive_missing_baseline_exits_2(tmp_path: Path) -> None:
665+
"""Recursive counterpart of test_cli_scan_missing_baseline_exits_2.
666+
667+
Regression for the error-contract break found reviewing PR #205: once --baseline
668+
was threaded into the recursive path, load_baseline() ran outside scan()'s
669+
try/except, so a bad path escaped as an uncaught FileNotFoundError and exited 1,
670+
the "risk found" code. docs/SUPPRESSION.md and README's exit-code table both
671+
require 2 for a missing or malformed baseline.
672+
"""
673+
collection = tmp_path / "collection"
674+
for name in ("alpha", "beta"):
675+
sub = collection / name
676+
sub.mkdir(parents=True)
677+
(sub / "SKILL.md").write_text(f"---\nname: {name}\n---\n# {name}\n", encoding="utf-8")
678+
679+
result = runner.invoke(
680+
app,
681+
[
682+
"scan",
683+
str(collection),
684+
"--recursive",
685+
"--no-llm",
686+
"--baseline",
687+
str(tmp_path / "missing.yaml"),
688+
],
689+
)
690+
691+
assert result.exit_code == 2
692+
assert "baseline" in result.output.lower()
693+
assert "Traceback" not in result.output
694+
695+
696+
def test_cli_recursive_malformed_baseline_exits_2(tmp_path: Path) -> None:
697+
"""A malformed (unparseable) baseline exits 2 recursively, same as single-skill."""
698+
collection = tmp_path / "collection"
699+
sub = collection / "alpha"
700+
sub.mkdir(parents=True)
701+
(sub / "SKILL.md").write_text("---\nname: alpha\n---\n# alpha\n", encoding="utf-8")
702+
(collection / "beta").mkdir()
703+
(collection / "beta" / "SKILL.md").write_text(
704+
"---\nname: beta\n---\n# beta\n", encoding="utf-8"
705+
)
706+
707+
bad = tmp_path / "bad.yaml"
708+
bad.write_text("not: [valid\n yaml: :::\n", encoding="utf-8")
709+
710+
result = runner.invoke(
711+
app,
712+
["scan", str(collection), "--recursive", "--no-llm", "--baseline", str(bad)],
713+
)
714+
715+
assert result.exit_code == 2
716+
assert "baseline" in result.output.lower()
717+
assert "Traceback" not in result.output
718+
719+
720+
def test_cli_recursive_terminal_summary_finding_count_excludes_suppressed(
721+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
722+
) -> None:
723+
"""The terminal summary's Findings column must also exclude suppressed findings.
724+
725+
The combined-JSON count is pinned by
726+
test_cli_recursive_json_finding_count_excludes_suppressed, but the terminal
727+
summary at cli.py's _scan_multi_skill uses the same helper on a separate line and
728+
was previously unpinned: reverting it to the pre-suppression form passed the whole
729+
suite. This pins the user-visible column.
730+
"""
731+
import skillspector.cli as cli_mod
732+
733+
collection = tmp_path / "collection"
734+
for name in ("alpha", "beta"):
735+
sub = collection / name
736+
sub.mkdir(parents=True)
737+
(sub / "SKILL.md").write_text(f"---\nname: {name}\n---\n# {name}\n", encoding="utf-8")
738+
739+
# alpha fully suppressed (active 0); beta suppresses 2 of 3 (active 1).
740+
per_skill = {
741+
"alpha": {
742+
"findings": ["raw1", "raw2", "raw3"],
743+
"filtered_findings": ["raw1", "raw2", "raw3"],
744+
"suppressed_findings": ["raw1", "raw2", "raw3"],
745+
},
746+
"beta": {
747+
"findings": ["raw1", "raw2", "raw3"],
748+
"filtered_findings": ["raw1", "raw2", "raw3"],
749+
"suppressed_findings": ["raw1", "raw2"],
750+
},
751+
}
752+
753+
def fake_invoke(state: dict[str, object], config: object = None) -> dict[str, object]:
754+
name = Path(str(state["input_path"])).name
755+
return {
756+
"risk_score": 0,
757+
"risk_severity": "LOW",
758+
"report_body": "{}",
759+
**per_skill[name],
760+
}
761+
762+
monkeypatch.setattr(cli_mod.graph, "invoke", fake_invoke)
763+
764+
result = runner.invoke(app, ["scan", str(collection), "--recursive", "--no-llm"])
765+
766+
assert result.exit_code == 0
767+
# Summary rows are "<name> <score> <severity> <finding_count>"; assert the count
768+
# column, not a bare substring, so a pre-suppression 3 cannot pass.
769+
rows = {
770+
parts[0]: parts[-1]
771+
for line in result.output.splitlines()
772+
if len(parts := line.split()) == 4 and parts[0] in ("alpha", "beta")
773+
}
774+
assert rows == {"alpha": "0", "beta": "1"}

0 commit comments

Comments
 (0)