Skip to content

Commit 0eeae6f

Browse files
committed
fix(llm): isolate batch failures in Stage 2 and keep unanalysed findings
One exception anywhere in the arun_batches fan-out aborted the whole Stage 2 pass: asyncio.gather without return_exceptions cancelled the remaining batches, the meta-analyzer's blanket except caught the propagated error, and every file silently fell back to static-only results while the CLI still exited 0 (#9). arun_batches now isolates failures per batch: a transient error (timeout, 429, oversized-chunk 400) is logged and costs only its own batch. ValueError and NotImplementedError still propagate, since they signal misconfiguration rather than infra trouble. With partial results possible, apply_filter could no longer treat a missing confirmation as a rejection: a finding whose batch never returned would be silently dropped — a false negative manufactured by an infrastructure error (#11). The meta-analyzer now partitions findings by whether a returned batch actually carried them: analysed findings go through the normal confirm-or-drop filter, unanalysed ones are kept via the existing fallback path, and a WARNING logs how many findings were kept unfiltered so the gap is visible. Net effect: an infra failure can only ever cost enrichment on a file, never the finding itself, and one bad call no longer turns off the semantic filter for the whole scan. Fixes #9, fixes #11 Signed-off-by: nyxst4ck <[email protected]>
1 parent 1a7bf02 commit 0eeae6f

4 files changed

Lines changed: 224 additions & 2 deletions

File tree

‎src/skillspector/llm_analyzer_base.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,14 @@ async def arun_batches(
374374
*max_concurrency* LLM requests in parallel. Both cross-file and
375375
cross-chunk batches are parallelized in a single gather call.
376376
377+
Failures are isolated per batch: a transient error (timeout, 429,
378+
oversized-chunk 400, ...) costs only its own batch, which is logged
379+
and omitted from the result, so one bad call cannot cancel the rest
380+
of the fan-out. Callers can detect partial results by comparing the
381+
returned batches against the submitted ones. ``ValueError`` and
382+
``NotImplementedError`` signal misconfiguration rather than infra
383+
trouble and keep propagating.
384+
377385
The return type mirrors :meth:`run_batches`.
378386
"""
379387
sem = asyncio.Semaphore(max_concurrency)
@@ -394,7 +402,16 @@ async def _process(batch: Batch) -> tuple[Batch, list]:
394402
logger.debug("LLM response for %s", batch.file_label)
395403
return (batch, self.parse_response(response, batch))
396404

397-
return list(await asyncio.gather(*[_process(b) for b in batches]))
405+
results = await asyncio.gather(*[_process(b) for b in batches], return_exceptions=True)
406+
successful: list[tuple[Batch, list]] = []
407+
for batch, result in zip(batches, results, strict=True):
408+
if isinstance(result, (ValueError, NotImplementedError)):
409+
raise result
410+
if isinstance(result, BaseException):
411+
logger.warning("LLM batch failed for %s: %s", batch.file_label, result)
412+
continue
413+
successful.append(result)
414+
return successful
398415

399416
# -- Convenience --------------------------------------------------------
400417

‎src/skillspector/nodes/meta_analyzer.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,28 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse:
392392
)
393393

394394
batch_results = asyncio.run(analyzer.arun_batches(batches, metadata_text=metadata_text))
395-
filtered = analyzer.apply_filter(findings, batch_results)
395+
396+
if len(batch_results) < len(batches):
397+
# Some batches never returned. A finding the LLM never saw has no
398+
# verdict — keep it via the fallback path instead of letting
399+
# apply_filter treat the missing confirmation as a rejection.
400+
analysed_ids = {id(f) for batch, _ in batch_results for f in batch.findings}
401+
analysed = [f for f in findings if id(f) in analysed_ids]
402+
unanalysed = [f for f in findings if id(f) not in analysed_ids]
403+
else:
404+
analysed, unanalysed = findings, []
405+
406+
filtered = analyzer.apply_filter(analysed, batch_results)
407+
if unanalysed:
408+
logger.warning(
409+
"Meta-analyzer: %d/%d batches failed; keeping %d findings in %d "
410+
"files unfiltered (no LLM verdict)",
411+
len(batches) - len(batch_results),
412+
len(batches),
413+
len(unanalysed),
414+
len({f.file for f in unanalysed}),
415+
)
416+
filtered.extend(_fallback_filtered(unanalysed))
396417

397418
logger.debug(
398419
"LLM filtering done: %d findings -> %d after filter",

‎tests/nodes/test_llm_analyzer_base.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,42 @@ async def _delayed_ainvoke(prompt: str) -> LLMAnalysisResult:
484484

485485
assert seen_files == {f"file_{i}.py" for i in range(num_batches)}
486486

487+
@patch(MOCK_PATCH_TARGET, _mock_get_chat_model)
488+
async def test_failed_batch_does_not_abort_the_others(self) -> None:
489+
"""A transient failure costs only its own batch, not the whole fan-out."""
490+
491+
async def _flaky_ainvoke(prompt: str) -> LLMAnalysisResult:
492+
if "b.py" in prompt:
493+
raise RuntimeError("429 Too Many Requests")
494+
return LLMAnalysisResult(findings=[])
495+
496+
analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL)
497+
analyzer._structured_llm.ainvoke = _flaky_ainvoke
498+
499+
batches = [
500+
Batch(file_path="a.py", content="code a"),
501+
Batch(file_path="b.py", content="code b"),
502+
Batch(file_path="c.py", content="code c"),
503+
]
504+
results = await analyzer.arun_batches(batches)
505+
assert {batch.file_path for batch, _ in results} == {"a.py", "c.py"}
506+
507+
@patch(MOCK_PATCH_TARGET, _mock_get_chat_model)
508+
async def test_all_batches_failed_returns_empty(self) -> None:
509+
analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL)
510+
analyzer._structured_llm.ainvoke = AsyncMock(side_effect=RuntimeError("boom"))
511+
batches = [Batch(file_path="a.py", content="code")]
512+
assert await analyzer.arun_batches(batches) == []
513+
514+
@patch(MOCK_PATCH_TARGET, _mock_get_chat_model)
515+
async def test_value_error_still_propagates(self) -> None:
516+
"""ValueError signals misconfiguration, not infra trouble — never swallowed."""
517+
analyzer = LLMAnalyzerBase(base_prompt="test", model=self.MODEL)
518+
analyzer._structured_llm.ainvoke = AsyncMock(side_effect=ValueError("no API key"))
519+
batches = [Batch(file_path="a.py", content="code")]
520+
with pytest.raises(ValueError, match="no API key"):
521+
await analyzer.arun_batches(batches)
522+
487523

488524
# ---------------------------------------------------------------------------
489525
# _format_findings_for_prompt (per-file, no truncation)

‎tests/nodes/test_meta_analyzer.py‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
"""Tests for the meta_analyzer node: batch-failure resilience of the filter.
17+
18+
Covers the keep/drop semantics when Stage 2 batches fail (issues #9/#11):
19+
a finding the LLM rejected is dropped, a finding the LLM never saw is kept
20+
via the fallback path, and one failed batch must not disable filtering for
21+
the batches that succeeded.
22+
"""
23+
24+
from __future__ import annotations
25+
26+
from unittest.mock import AsyncMock, patch
27+
28+
from skillspector.llm_analyzer_base import Batch
29+
from skillspector.models import Finding
30+
from skillspector.nodes.meta_analyzer import LLMMetaAnalyzer, meta_analyzer
31+
32+
MOCK_PATCH_TARGET = "skillspector.llm_analyzer_base.get_chat_model"
33+
34+
35+
def _mock_get_chat_model(*_args, **_kwargs):
36+
from unittest.mock import MagicMock
37+
38+
mock_llm = MagicMock()
39+
mock_llm.with_structured_output.return_value = MagicMock()
40+
return mock_llm
41+
42+
43+
def _confirm(pattern_id: str, file: str, start_line: int) -> dict[str, object]:
44+
"""LLM item confirming a finding, as parse_response would emit it."""
45+
return {
46+
"pattern_id": pattern_id,
47+
"is_vulnerability": True,
48+
"confidence": 0.9,
49+
"explanation": "confirmed by llm",
50+
"remediation": "fix it",
51+
"_file": file,
52+
"start_line": start_line,
53+
"end_line": None,
54+
}
55+
56+
57+
@patch(MOCK_PATCH_TARGET, _mock_get_chat_model)
58+
class TestMetaAnalyzerPartialBatchFailure:
59+
def _state(self, findings: list[Finding]) -> dict[str, object]:
60+
return {
61+
"findings": findings,
62+
"use_llm": True,
63+
"file_cache": {"a.py": "code a", "b.py": "code b"},
64+
"manifest": {},
65+
"model_config": {},
66+
}
67+
68+
def test_unanalysed_findings_survive_a_failed_batch(self) -> None:
69+
"""Findings whose batch failed are kept (no verdict != rejection)."""
70+
f_confirmed = Finding(rule_id="R1", message="m", file="a.py", start_line=1)
71+
f_rejected = Finding(rule_id="R2", message="m", file="a.py", start_line=5)
72+
f_unseen = Finding(rule_id="R1", message="m", file="b.py", start_line=3)
73+
74+
batch_a = Batch(file_path="a.py", content="code a", findings=[f_confirmed, f_rejected])
75+
batch_b = Batch(file_path="b.py", content="code b", findings=[f_unseen])
76+
77+
# batch_b never returned (timeout/429): only batch_a's verdicts exist,
78+
# and the LLM confirmed R1 but stayed silent on R2 (= rejection).
79+
partial_results = [(batch_a, [_confirm("R1", "a.py", 1)])]
80+
81+
with (
82+
patch.object(LLMMetaAnalyzer, "get_batches", return_value=[batch_a, batch_b]),
83+
patch.object(
84+
LLMMetaAnalyzer,
85+
"arun_batches",
86+
new_callable=AsyncMock,
87+
return_value=partial_results,
88+
),
89+
):
90+
result = meta_analyzer(self._state([f_confirmed, f_rejected, f_unseen]))
91+
92+
filtered = result["filtered_findings"]
93+
kept = {(f.file, f.rule_id) for f in filtered}
94+
95+
# the real filter still applies to the batch that came back
96+
assert ("a.py", "R1") in kept
97+
assert ("a.py", "R2") not in kept
98+
# the finding the LLM never saw must NOT be silently dropped
99+
assert ("b.py", "R1") in kept
100+
101+
confirmed = next(f for f in filtered if f.file == "a.py")
102+
assert confirmed.explanation == "confirmed by llm"
103+
104+
def test_all_batches_failed_keeps_everything_via_fallback(self) -> None:
105+
f1 = Finding(rule_id="R1", message="m", file="a.py", start_line=1)
106+
f2 = Finding(rule_id="R2", message="m", file="b.py", start_line=2)
107+
batch_a = Batch(file_path="a.py", content="code a", findings=[f1])
108+
batch_b = Batch(file_path="b.py", content="code b", findings=[f2])
109+
110+
with (
111+
patch.object(LLMMetaAnalyzer, "get_batches", return_value=[batch_a, batch_b]),
112+
patch.object(
113+
LLMMetaAnalyzer,
114+
"arun_batches",
115+
new_callable=AsyncMock,
116+
return_value=[],
117+
),
118+
):
119+
result = meta_analyzer(self._state([f1, f2]))
120+
121+
kept = {(f.file, f.rule_id) for f in result["filtered_findings"]}
122+
assert kept == {("a.py", "R1"), ("b.py", "R2")}
123+
124+
def test_no_failures_keeps_strict_confirm_or_drop(self) -> None:
125+
"""When every batch returns, unconfirmed findings are dropped as before."""
126+
f_confirmed = Finding(rule_id="R1", message="m", file="a.py", start_line=1)
127+
f_rejected = Finding(rule_id="R2", message="m", file="b.py", start_line=2)
128+
batch_a = Batch(file_path="a.py", content="code a", findings=[f_confirmed])
129+
batch_b = Batch(file_path="b.py", content="code b", findings=[f_rejected])
130+
131+
full_results = [
132+
(batch_a, [_confirm("R1", "a.py", 1)]),
133+
(batch_b, []),
134+
]
135+
136+
with (
137+
patch.object(LLMMetaAnalyzer, "get_batches", return_value=[batch_a, batch_b]),
138+
patch.object(
139+
LLMMetaAnalyzer,
140+
"arun_batches",
141+
new_callable=AsyncMock,
142+
return_value=full_results,
143+
),
144+
):
145+
result = meta_analyzer(self._state([f_confirmed, f_rejected]))
146+
147+
kept = {(f.file, f.rule_id) for f in result["filtered_findings"]}
148+
assert kept == {("a.py", "R1")}

0 commit comments

Comments
 (0)