Skip to content

Commit b93da63

Browse files
fix(cli): count post-suppression findings in recursive multi-skill report
The recursive summary table and combined JSON report computed finding_count from filtered_findings, but the report node returns filtered_findings as the full pre-partition set (kept plus baseline-suppressed) and lists the suppressed subset separately under suppressed_findings; it never reduces filtered_findings to active-only. Counting len(filtered_findings) therefore reported pre-suppression totals, so a baselined sub-skill showed score 0 but a non-zero finding_count, disagreeing with the per-skill report body and risk score. Add _result_finding_count(), which returns len(filtered_findings) - len(suppressed_findings) when filtered_findings is a list (partition_findings guarantees kept + suppressed == filtered_findings, so this is the active count), and falls back to the raw findings length only when filtered_findings is absent or not a list, without subtracting there since raw findings are not the population that produced suppressed_findings. Use it at both the summary table and the combined JSON report. Add a regression test covering a fully suppressed sub-skill (active 0) and a partially suppressed one (active 1), plus a unit test exercising each branch of the helper. Refs #201 Signed-off-by: Werner Kasselman <[email protected]>
1 parent 065d8fd commit b93da63

2 files changed

Lines changed: 127 additions & 5 deletions

File tree

‎src/skillspector/cli.py‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,30 @@ def _build_trace_config(input_path: str, format: FormatChoice, no_llm: bool) ->
361361
}
362362

363363

364+
def _result_finding_count(result: dict[str, object]) -> int:
365+
"""Count active (post-suppression) findings for the multi-skill summary.
366+
367+
The report node returns ``filtered_findings`` as the full LLM-filtered set
368+
(kept plus baseline-suppressed) and the suppressed subset separately under
369+
``suppressed_findings`` (see ``nodes/report.py``). ``partition_findings``
370+
guarantees ``kept + suppressed == filtered_findings``, so the active count is
371+
``len(filtered_findings) - len(suppressed_findings)`` and is never negative for
372+
real report output; the ``max(..., 0)`` is only a display-safety floor against
373+
a malformed result. Subtracting is done solely on this branch: the raw
374+
``findings`` fallback (reached only when ``filtered_findings`` is absent or not
375+
a list, e.g. a malformed or error result, never real report output) must not
376+
subtract, since raw findings are not the population that produced
377+
``suppressed_findings``.
378+
"""
379+
filtered = result.get("filtered_findings")
380+
if isinstance(filtered, list):
381+
suppressed = result.get("suppressed_findings")
382+
suppressed_count = len(suppressed) if isinstance(suppressed, list) else 0
383+
return max(len(filtered) - suppressed_count, 0)
384+
findings = result.get("findings")
385+
return len(findings) if isinstance(findings, list) else 0
386+
387+
364388
def _scan_multi_skill(
365389
detection: MultiSkillDetectionResult,
366390
format: FormatChoice,
@@ -415,8 +439,7 @@ def _scan_multi_skill(
415439
continue
416440
score = result.get("risk_score", 0)
417441
severity = result.get("risk_severity", "LOW")
418-
filtered = result.get("filtered_findings") or result.get("findings")
419-
finding_count = len(filtered) if isinstance(filtered, list) else 0
442+
finding_count = _result_finding_count(result)
420443
console.print(f" {skill.name:<30} {score:<8} {severity:<12} {finding_count:<10}")
421444

422445
console.print("")
@@ -438,9 +461,7 @@ def _scan_multi_skill(
438461
"path": skill.relative_path,
439462
"risk_score": result.get("risk_score", 0),
440463
"risk_severity": result.get("risk_severity", "LOW"),
441-
"finding_count": len(
442-
result.get("filtered_findings") or result.get("findings") or []
443-
),
464+
"finding_count": _result_finding_count(result),
444465
}
445466
)
446467
Path(output).write_text(json.dumps(combined, indent=2), encoding="utf-8")

‎tests/unit/test_cli.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,3 +253,104 @@ def fake_invoke(state: dict[str, object], config: object = None) -> dict[str, ob
253253
for state in captured:
254254
assert "baseline" in state # the baseline was loaded and threaded in
255255
assert state.get("show_suppressed") is True
256+
257+
258+
def test_cli_recursive_json_finding_count_excludes_suppressed(
259+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
260+
) -> None:
261+
"""The combined JSON report must count active (post-suppression) findings.
262+
263+
Regression for the over-count: the report node returns ``filtered_findings``
264+
as the full pre-partition set (kept plus baseline-suppressed) and lists the
265+
suppressed subset separately under ``suppressed_findings`` (see
266+
``nodes/report.py``); it never reduces ``filtered_findings`` to active-only.
267+
So ``len(filtered_findings)`` reports pre-suppression totals and the count must
268+
subtract ``suppressed_findings``. This test mirrors that real return shape (not
269+
an empty ``filtered_findings``, which suppression never produces): one skill
270+
fully suppressed (active 0) and one partially suppressed (active 1).
271+
"""
272+
import skillspector.cli as cli_mod
273+
274+
collection = tmp_path / "collection"
275+
for name in ("alpha", "beta"):
276+
sub = collection / name
277+
sub.mkdir(parents=True)
278+
(sub / "SKILL.md").write_text(f"---\nname: {name}\n---\n# {name}\n", encoding="utf-8")
279+
280+
# Return states shaped exactly like nodes/report.py: filtered_findings holds
281+
# every finding (kept + suppressed); suppressed_findings holds the suppressed
282+
# subset. alpha is fully suppressed (active 0); beta suppresses 2 of 3 (active 1).
283+
per_skill = {
284+
"alpha": {
285+
"findings": ["raw1", "raw2", "raw3"],
286+
"filtered_findings": ["raw1", "raw2", "raw3"],
287+
"suppressed_findings": ["raw1", "raw2", "raw3"],
288+
},
289+
"beta": {
290+
"findings": ["raw1", "raw2", "raw3"],
291+
"filtered_findings": ["raw1", "raw2", "raw3"],
292+
"suppressed_findings": ["raw1", "raw2"],
293+
},
294+
}
295+
296+
def fake_invoke(state: dict[str, object], config: object = None) -> dict[str, object]:
297+
# The scanned sub-skill path ends with the skill directory name.
298+
name = Path(str(state["input_path"])).name
299+
return {
300+
"risk_score": 0,
301+
"risk_severity": "LOW",
302+
"report_body": "{}",
303+
**per_skill[name],
304+
}
305+
306+
monkeypatch.setattr(cli_mod.graph, "invoke", fake_invoke)
307+
308+
out_file = tmp_path / "combined.json"
309+
scan = runner.invoke(
310+
app,
311+
[
312+
"scan",
313+
str(collection),
314+
"--recursive",
315+
"--no-llm",
316+
"--format",
317+
"json",
318+
"--output",
319+
str(out_file),
320+
],
321+
)
322+
323+
assert scan.exit_code == 0
324+
data = json.loads(out_file.read_text())
325+
by_name = {s["name"]: s["finding_count"] for s in data["skills"]}
326+
assert by_name == {"alpha": 0, "beta": 1}
327+
328+
329+
def test_result_finding_count_branches() -> None:
330+
"""Unit-cover every branch of _result_finding_count directly.
331+
332+
Mirrors the report-node contract: filtered_findings is the full pre-partition
333+
set, suppressed_findings its suppressed subset. The raw-findings fallback (only
334+
when filtered_findings is absent or not a list) must NOT subtract suppressed,
335+
since raw findings are not the population that produced suppressed_findings.
336+
"""
337+
from skillspector.cli import _result_finding_count
338+
339+
# Real report shape: active = len(filtered) - len(suppressed).
340+
assert (
341+
_result_finding_count({"filtered_findings": [1, 2, 3], "suppressed_findings": [1, 2]}) == 1
342+
)
343+
assert (
344+
_result_finding_count({"filtered_findings": [1, 2, 3], "suppressed_findings": [1, 2, 3]})
345+
== 0
346+
)
347+
# No baseline: suppressed_findings absent -> full filtered count.
348+
assert _result_finding_count({"filtered_findings": [1, 2, 3]}) == 3
349+
# Fallback: filtered_findings absent -> raw findings length, WITHOUT subtracting
350+
# suppressed (raw findings are not the suppressed population).
351+
assert _result_finding_count({"findings": [1, 2, 3], "suppressed_findings": [1, 2]}) == 3
352+
# Non-list filtered_findings also takes the fallback branch.
353+
assert _result_finding_count({"filtered_findings": None, "findings": [1]}) == 1
354+
# Nothing usable -> 0. Malformed (suppressed longer than filtered) floors at 0.
355+
assert _result_finding_count({}) == 0
356+
assert _result_finding_count({"filtered_findings": [1], "suppressed_findings": [1, 2, 3]}) == 0

0 commit comments

Comments
 (0)