Skip to content

Commit e9ecd0b

Browse files
fix(cli): count post-suppression findings in recursive multi-skill report
The recursive summary table and combined JSON report computed finding_count from filtered_findings, but the report node returns filtered_findings as the full pre-partition set (kept plus baseline-suppressed) and lists the suppressed subset separately under suppressed_findings; it never reduces filtered_findings to active-only. Counting len(filtered_findings) therefore reported pre-suppression totals, so a baselined sub-skill showed score 0 but a non-zero finding_count, disagreeing with the per-skill report body and risk score. Add _result_finding_count(), which returns len(filtered_findings) - len(suppressed_findings) when filtered_findings is a list (partition_findings guarantees kept + suppressed == filtered_findings, so this is the active count), and falls back to the raw findings length only when filtered_findings is absent or not a list, without subtracting there since raw findings are not the population that produced suppressed_findings. Use it at both the summary table and the combined JSON report. Add a regression test covering a fully suppressed sub-skill (active 0) and a partially suppressed one (active 1), plus a unit test exercising each branch of the helper. Refs NVIDIA#201 Signed-off-by: Werner Kasselman <[email protected]>
1 parent a99a32d commit e9ecd0b

2 files changed

Lines changed: 128 additions & 8 deletions

File tree

‎src/skillspector/cli.py‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,30 @@ def _build_trace_config(input_path: str, format: FormatChoice, no_llm: bool) ->
375375
}
376376

377377

378+
def _result_finding_count(result: dict[str, object]) -> int:
379+
"""Count active (post-suppression) findings for the multi-skill summary.
380+
381+
The report node returns ``filtered_findings`` as the full LLM-filtered set
382+
(kept plus baseline-suppressed) and the suppressed subset separately under
383+
``suppressed_findings`` (see ``nodes/report.py``). ``partition_findings``
384+
guarantees ``kept + suppressed == filtered_findings``, so the active count is
385+
``len(filtered_findings) - len(suppressed_findings)`` and is never negative for
386+
real report output; the ``max(..., 0)`` is only a display-safety floor against
387+
a malformed result. Subtracting is done solely on this branch: the raw
388+
``findings`` fallback (reached only when ``filtered_findings`` is absent or not
389+
a list, e.g. a malformed or error result, never real report output) must not
390+
subtract, since raw findings are not the population that produced
391+
``suppressed_findings``.
392+
"""
393+
filtered = result.get("filtered_findings")
394+
if isinstance(filtered, list):
395+
suppressed = result.get("suppressed_findings")
396+
suppressed_count = len(suppressed) if isinstance(suppressed, list) else 0
397+
return max(len(filtered) - suppressed_count, 0)
398+
findings = result.get("findings")
399+
return len(findings) if isinstance(findings, list) else 0
400+
401+
378402
def _scan_multi_skill(
379403
detection: MultiSkillDetectionResult,
380404
format: FormatChoice,
@@ -429,8 +453,7 @@ def _scan_multi_skill(
429453
continue
430454
score = result.get("risk_score", 0)
431455
severity = result.get("risk_severity", "LOW")
432-
filtered = result.get("filtered_findings") or result.get("findings")
433-
finding_count = len(filtered) if isinstance(filtered, list) else 0
456+
finding_count = _result_finding_count(result)
434457
console.print(f" {skill.name:<30} {score:<8} {severity:<12} {finding_count:<10}")
435458

436459
console.print("")
@@ -452,18 +475,14 @@ def _scan_multi_skill(
452475
"path": skill.relative_path,
453476
"risk_score": result.get("risk_score", 0),
454477
"risk_severity": result.get("risk_severity", "LOW"),
455-
"finding_count": len(
456-
result.get("filtered_findings") or result.get("findings") or []
457-
),
478+
"finding_count": _result_finding_count(result),
458479
}
459480
entry.update(payload)
460481
entry["name"] = skill.name
461482
entry["path"] = skill.relative_path
462483
entry["risk_score"] = result.get("risk_score", 0)
463484
entry["risk_severity"] = result.get("risk_severity", "LOW")
464-
entry["finding_count"] = len(
465-
result.get("filtered_findings") or result.get("findings") or []
466-
)
485+
entry["finding_count"] = _result_finding_count(result)
467486
combined["skills"].append(entry)
468487
Path(output).write_text(json.dumps(combined, indent=2), encoding="utf-8")
469488
console.print(f"[green]Combined report saved to:[/green] {output}")

‎tests/unit/test_cli.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,3 +558,104 @@ def fake_invoke(state: dict[str, object], config: object = None) -> dict[str, ob
558558
for state in captured:
559559
assert "baseline" in state # the baseline was loaded and threaded in
560560
assert state.get("show_suppressed") is True
561+
562+
563+
def test_cli_recursive_json_finding_count_excludes_suppressed(
564+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
565+
) -> None:
566+
"""The combined JSON report must count active (post-suppression) findings.
567+
568+
Regression for the over-count: the report node returns ``filtered_findings``
569+
as the full pre-partition set (kept plus baseline-suppressed) and lists the
570+
suppressed subset separately under ``suppressed_findings`` (see
571+
``nodes/report.py``); it never reduces ``filtered_findings`` to active-only.
572+
So ``len(filtered_findings)`` reports pre-suppression totals and the count must
573+
subtract ``suppressed_findings``. This test mirrors that real return shape (not
574+
an empty ``filtered_findings``, which suppression never produces): one skill
575+
fully suppressed (active 0) and one partially suppressed (active 1).
576+
"""
577+
import skillspector.cli as cli_mod
578+
579+
collection = tmp_path / "collection"
580+
for name in ("alpha", "beta"):
581+
sub = collection / name
582+
sub.mkdir(parents=True)
583+
(sub / "SKILL.md").write_text(f"---\nname: {name}\n---\n# {name}\n", encoding="utf-8")
584+
585+
# Return states shaped exactly like nodes/report.py: filtered_findings holds
586+
# every finding (kept + suppressed); suppressed_findings holds the suppressed
587+
# subset. alpha is fully suppressed (active 0); beta suppresses 2 of 3 (active 1).
588+
per_skill = {
589+
"alpha": {
590+
"findings": ["raw1", "raw2", "raw3"],
591+
"filtered_findings": ["raw1", "raw2", "raw3"],
592+
"suppressed_findings": ["raw1", "raw2", "raw3"],
593+
},
594+
"beta": {
595+
"findings": ["raw1", "raw2", "raw3"],
596+
"filtered_findings": ["raw1", "raw2", "raw3"],
597+
"suppressed_findings": ["raw1", "raw2"],
598+
},
599+
}
600+
601+
def fake_invoke(state: dict[str, object], config: object = None) -> dict[str, object]:
602+
# The scanned sub-skill path ends with the skill directory name.
603+
name = Path(str(state["input_path"])).name
604+
return {
605+
"risk_score": 0,
606+
"risk_severity": "LOW",
607+
"report_body": "{}",
608+
**per_skill[name],
609+
}
610+
611+
monkeypatch.setattr(cli_mod.graph, "invoke", fake_invoke)
612+
613+
out_file = tmp_path / "combined.json"
614+
scan = runner.invoke(
615+
app,
616+
[
617+
"scan",
618+
str(collection),
619+
"--recursive",
620+
"--no-llm",
621+
"--format",
622+
"json",
623+
"--output",
624+
str(out_file),
625+
],
626+
)
627+
628+
assert scan.exit_code == 0
629+
data = json.loads(out_file.read_text())
630+
by_name = {s["name"]: s["finding_count"] for s in data["skills"]}
631+
assert by_name == {"alpha": 0, "beta": 1}
632+
633+
634+
def test_result_finding_count_branches() -> None:
635+
"""Unit-cover every branch of _result_finding_count directly.
636+
637+
Mirrors the report-node contract: filtered_findings is the full pre-partition
638+
set, suppressed_findings its suppressed subset. The raw-findings fallback (only
639+
when filtered_findings is absent or not a list) must NOT subtract suppressed,
640+
since raw findings are not the population that produced suppressed_findings.
641+
"""
642+
from skillspector.cli import _result_finding_count
643+
644+
# Real report shape: active = len(filtered) - len(suppressed).
645+
assert (
646+
_result_finding_count({"filtered_findings": [1, 2, 3], "suppressed_findings": [1, 2]}) == 1
647+
)
648+
assert (
649+
_result_finding_count({"filtered_findings": [1, 2, 3], "suppressed_findings": [1, 2, 3]})
650+
== 0
651+
)
652+
# No baseline: suppressed_findings absent -> full filtered count.
653+
assert _result_finding_count({"filtered_findings": [1, 2, 3]}) == 3
654+
# Fallback: filtered_findings absent -> raw findings length, WITHOUT subtracting
655+
# suppressed (raw findings are not the suppressed population).
656+
assert _result_finding_count({"findings": [1, 2, 3], "suppressed_findings": [1, 2]}) == 3
657+
# Non-list filtered_findings also takes the fallback branch.
658+
assert _result_finding_count({"filtered_findings": None, "findings": [1]}) == 1
659+
# Nothing usable -> 0. Malformed (suppressed longer than filtered) floors at 0.
660+
assert _result_finding_count({}) == 0
661+
assert _result_finding_count({"filtered_findings": [1], "suppressed_findings": [1, 2, 3]}) == 0

0 commit comments

Comments
 (0)