Skip to content

Commit 5dfbf1b

Browse files
Merge branch 'main' into yashraj/cache-shell-flag-scope-analysis
2 parents 58a8545 + 7ee4371 commit 5dfbf1b

11 files changed

Lines changed: 441 additions & 175 deletions

File tree

‎contrib/batch_scan/api_pool.py‎

Lines changed: 57 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -475,106 +475,95 @@ async def ainvoke_with_usage(self, prompt: str, collector: object) -> object:
475475

476476
# -- Internal -------------------------------------------------------------
477477

478+
@staticmethod
479+
def _remaining(deadline: float) -> float:
480+
from skillspector.llm_analyzer_base import LLMRuntimeLimitError
481+
482+
remaining = deadline - time.monotonic()
483+
if remaining <= 0:
484+
raise LLMRuntimeLimitError("pooled request runtime limit reached")
485+
return remaining
486+
478487
def _invoke_with_retry(
479-
self,
480-
prompt: str,
481-
*,
482-
callbacks: list[object] | None = None,
488+
self, prompt: str, *, callbacks: list[object] | None = None,
483489
) -> object:
484-
"""Sync retry loop — acquire slot, call LLM, release, retry on 429."""
485-
last_exception: Exception | None = None
486-
490+
"""Share one deadline across slot waiting, requests, and key retries."""
491+
deadline = time.monotonic() + self._timeout
487492
for attempt in range(self._max_retries + 1):
488-
key = self._pool.acquire()
489-
llm = self._build_llm(key)
490493
try:
491-
if callbacks is None:
492-
result = llm.invoke(prompt)
493-
else:
494-
result = llm.invoke(prompt, config={"callbacks": callbacks})
495-
self._pool.release(key, success=True)
496-
if attempt > 0:
497-
self._pool.record_retry_success()
498-
return result
494+
key = self._pool.acquire(timeout=self._remaining(deadline))
495+
except Exception:
496+
self._remaining(deadline)
497+
raise
498+
rate_limited = False
499+
try:
500+
llm = self._build_llm(key, timeout=self._remaining(deadline))
501+
result = llm.invoke(prompt) if callbacks is None else llm.invoke(
502+
prompt, config={"callbacks": callbacks}
503+
)
499504
except Exception as exc:
500-
if self._is_rate_limit(exc) and attempt < self._max_retries:
501-
self._pool.release(key, success=False)
502-
logger.debug(
503-
"PooledChatModel: rate-limited, retrying "
504-
"(attempt %d/%d)",
505-
attempt + 1,
506-
self._max_retries,
507-
)
505+
rate_limited = self._is_rate_limit(exc)
506+
if rate_limited and attempt < self._max_retries:
508507
continue
509-
self._pool.release(key, success=True)
510-
last_exception = exc
511508
raise
512-
513-
raise RuntimeError(
514-
f"PooledChatModel: exhausted {self._max_retries} retries "
515-
"due to rate-limit errors"
516-
) from last_exception
509+
finally:
510+
self._pool.release(key, success=not rate_limited)
511+
if attempt > 0:
512+
self._pool.record_retry_success()
513+
return result
514+
raise RuntimeError("PooledChatModel exhausted key retries")
517515

518516
async def _ainvoke_with_retry(
519-
self,
520-
prompt: str,
521-
*,
522-
callbacks: list[object] | None = None,
517+
self, prompt: str, *, callbacks: list[object] | None = None,
523518
) -> object:
524-
"""Async retry loop — non-blocking acquire first, block only if full."""
519+
"""Bound asynchronous slot waiting without leaving an acquiring thread."""
525520
import asyncio
526-
last_exception: Exception | None = None
527521

522+
deadline = time.monotonic() + self._timeout
528523
for attempt in range(self._max_retries + 1):
524+
self._remaining(deadline)
529525
key = self._pool.try_acquire()
530-
if key is None:
531-
key = await asyncio.to_thread(self._pool.acquire)
532-
llm = self._build_llm(key)
526+
while key is None:
527+
await asyncio.sleep(min(0.05, self._remaining(deadline)))
528+
self._remaining(deadline)
529+
key = self._pool.try_acquire()
530+
rate_limited = False
533531
try:
534-
if callbacks is None:
535-
result = await llm.ainvoke(prompt)
536-
else:
537-
result = await llm.ainvoke(prompt, config={"callbacks": callbacks})
538-
self._pool.release(key, success=True)
539-
if attempt > 0:
540-
self._pool.record_retry_success()
541-
return result
532+
llm = self._build_llm(key, timeout=self._remaining(deadline))
533+
result = await llm.ainvoke(prompt) if callbacks is None else await llm.ainvoke(
534+
prompt, config={"callbacks": callbacks}
535+
)
542536
except Exception as exc:
543-
if self._is_rate_limit(exc) and attempt < self._max_retries:
544-
self._pool.release(key, success=False)
545-
logger.debug(
546-
"PooledChatModel: rate-limited, retrying "
547-
"(attempt %d/%d)",
548-
attempt + 1,
549-
self._max_retries,
550-
)
537+
rate_limited = self._is_rate_limit(exc)
538+
if rate_limited and attempt < self._max_retries:
551539
continue
552-
self._pool.release(key, success=True)
553-
last_exception = exc
554540
raise
555-
556-
raise RuntimeError(
557-
f"PooledChatModel: exhausted {self._max_retries} retries "
558-
"due to rate-limit errors"
559-
) from last_exception
560-
561-
def _build_llm(self, key: ApiKey):
541+
finally:
542+
self._pool.release(key, success=not rate_limited)
543+
if attempt > 0:
544+
self._pool.record_retry_success()
545+
return result
546+
raise RuntimeError("PooledChatModel exhausted key retries")
547+
548+
def _build_llm(self, key: ApiKey, *, timeout: float | None = None):
562549
"""Build a fresh :class:`~langchain_openai.ChatOpenAI` for *key*."""
563550
from langchain_openai import ChatOpenAI
564551
from pydantic import SecretStr
565552

553+
timeout = self._timeout if timeout is None else timeout
566554
try:
567555
import httpx
568-
_timeout = httpx.Timeout(self._timeout, connect=8.0)
556+
_timeout = httpx.Timeout(timeout, connect=min(8.0, timeout))
569557
except ImportError:
570-
_timeout = self._timeout
558+
_timeout = timeout
571559

572560
return ChatOpenAI(
573561
model=key.model,
574562
base_url=key.base_url,
575563
api_key=SecretStr(key.key),
576564
max_completion_tokens=self._max_tokens,
577565
timeout=_timeout,
566+
max_retries=0,
578567
)
579568

580569
@staticmethod

‎contrib/batch_scan/docs/DESIGN.md‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -284,22 +284,22 @@ raw LLM string → _strip_markdown_fences() → json.loads() → model_validate(
284284
The two-step parse (stdlib `json.loads` then Pydantic `model_validate`) exists
285285
because:
286286

287-
1. `json.loads` is fast, deterministic, and raises clear `JSONDecodeError` on
288-
malformed output — we catch this and return `[]` (empty findings).
289-
2. `model_validate` enforces the schema: required fields, literal enums,
290-
confidence range, string length. Schema violations are caught and returned
291-
as `[]` with a warning log.
292-
293-
**Error propagation:** If the LLM returns invalid JSON or schema-mismatched
294-
output, the analyzer returns `[]` (no findings for that file). The scan
295-
continues — a single malformed LLM response never blocks the pipeline.
296-
The warning is logged at `WARNING` level so operators can monitor parse-failure
297-
rates without sifting through debug logs.
298-
299-
Patch 3 adds a `_sanitize_meta_finding()` pass after validation to handle
300-
known LLM quirks: `null` string fields → `""`, unrecognized enum values
301-
(e.g., `"none"`) → `"low"`. These are applied post-validation because they
302-
represent recoverable soft errors, not hard schema violations.
287+
1. `json.loads` parses the response. Invalid JSON raises the core structured-response
288+
validation error rather than producing an empty findings list.
289+
2. `model_validate` checks the required fields and value types. Schema failures
290+
use the same retry and reporting path.
291+
292+
**Error propagation:** Invalid JSON and schema failures are retried up to four
293+
attempts. If they persist, the ledger records `skipped` with reason
294+
`llm_structured_response_invalid`; the analyzer is degraded and the skill counts
295+
as incomplete. The scan continues and keeps findings from completed work.
296+
Core logs `LLM structured response validation failed for ... retrying` and,
297+
on exhaustion, `... after 4 attempts`, without logging the raw response.
298+
299+
Patch 3 sanitizes known soft quirks before validation: null explanation and
300+
remediation become empty strings, impact labels are case-folded, and unknown
301+
impact labels use `low`. Invalid findings still fail validation. Optional prose
302+
in `overall_assessment` is ignored so valid finding verdicts are retained.
303303

304304
## Gap-Fill Rule Selection Criteria
305305

‎contrib/batch_scan/docs/README.md‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,10 +341,11 @@ fi
341341
classified as English and lose gap-fill coverage.
342342
3. **No SARIF output.** Upstream supports it; this contrib adds terminal/JSON/Markdown.
343343
4. **Gap-fill quality not benchmarked for non-English.** No ground-truth comparison exists.
344-
5. **`parse_response` JSON recovery is best-effort.** When the LLM returns
345-
malformed JSON, the analyzer returns empty findings (no crash). This is a
346-
graceful-degradation choice: a single malformed response won't block the
347-
pipeline, but the user won't know which findings were lost.
344+
5. **Malformed provider responses reduce coverage.** Compatibility-mode discovery
345+
and meta responses are retried up to four attempts, then recorded as
346+
`llm_structured_response_invalid` and counted as incomplete. Valid findings
347+
from completed batches are kept. Gap-fill parsing still drops malformed
348+
responses until the separate gap-fill failure fix is included.
348349

349350
See `DESIGN.md` for architecture details and `docs/archive/FUTURE_WORK.md` for suggested directions.
350351

0 commit comments

Comments
 (0)