Skip to content

Commit a4e317b

Browse files
committed
fix(scan): port transitive scanning to current main contracts
Signed-off-by: Rod Boev <[email protected]>
1 parent 5680c2c commit a4e317b

22 files changed

Lines changed: 3263 additions & 108 deletions

‎src/skillspector/cli.py‎

Lines changed: 645 additions & 28 deletions
Large diffs are not rendered by default.

‎src/skillspector/input_handler.py‎

Lines changed: 193 additions & 29 deletions
Large diffs are not rendered by default.

‎src/skillspector/llm_analyzer_base.py‎

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import os
3232
import time
3333
from collections import defaultdict
34+
from collections.abc import Callable
3435
from dataclasses import dataclass, field
3536
from typing import Any, Literal, cast
3637

@@ -464,11 +465,20 @@ class LLMAnalyzerBase:
464465

465466
response_schema: type | None = LLMAnalysisResult
466467

467-
def __init__(self, base_prompt: str, model: str, *, node: str = "llm_analyzer"):
468+
def __init__(
469+
self,
470+
base_prompt: str,
471+
model: str,
472+
*,
473+
node: str = "llm_analyzer",
474+
timeout: float | None | Callable[[], float | None] = None,
475+
):
468476
self.base_prompt = base_prompt
469477
self.model = model
478+
self._timeout = timeout
479+
self._dynamic_timeout = callable(timeout)
470480
self._input_budget = get_max_input_tokens(model)
471-
self._llm = get_chat_model(model=model)
481+
self._llm = get_chat_model(model=model, timeout=self._remaining_timeout())
472482
self._uses_native_connection_retries = _uses_native_connection_retries(self._llm)
473483
self._structured_llm = (
474484
self._llm.with_structured_output(self.response_schema) if self.response_schema else None
@@ -480,6 +490,20 @@ def __init__(self, base_prompt: str, model: str, *, node: str = "llm_analyzer"):
480490
chat_model=self._llm,
481491
)
482492

493+
def _remaining_timeout(self) -> float | None:
494+
if callable(self._timeout):
495+
return self._timeout()
496+
return self._timeout
497+
498+
def _model_for_call(self) -> tuple[object, object | None]:
499+
if not self._dynamic_timeout:
500+
return self._llm, self._structured_llm
501+
llm = get_chat_model(model=self.model, timeout=self._remaining_timeout())
502+
structured = (
503+
llm.with_structured_output(self.response_schema) if self.response_schema else None
504+
)
505+
return llm, structured
506+
483507
@property
484508
def inference_usage(self) -> list[dict[str, object]]:
485509
"""Provider-reported usage captured for this analyzer instance."""
@@ -585,15 +609,14 @@ def _invoke_batch(self, batch: Batch, prompt: str) -> tuple[Batch, list]:
585609
estimate_tokens(prompt),
586610
len(batch.findings),
587611
)
588-
if self._structured_llm:
612+
llm, structured_llm = self._model_for_call()
613+
if structured_llm:
589614
try:
590-
response = _invoke_with_usage(self._structured_llm, prompt, self._usage_collector)
615+
response = _invoke_with_usage(structured_llm, prompt, self._usage_collector)
591616
except (StructuredOutputParseError, ValidationError) as exc:
592617
raise _StructuredResponseValidationError from exc
593618
else:
594-
response = _raw_response_text(
595-
_invoke_with_usage(self._llm, prompt, self._usage_collector)
596-
)
619+
response = _raw_response_text(_invoke_with_usage(llm, prompt, self._usage_collector))
597620
logger.debug("LLM response for %s", batch.file_label)
598621
return batch, self.parse_response(response, batch)
599622

@@ -649,16 +672,15 @@ async def _ainvoke_batch(self, batch: Batch, prompt: str) -> tuple[Batch, list]:
649672
estimate_tokens(prompt),
650673
len(batch.findings),
651674
)
652-
if self._structured_llm:
675+
llm, structured_llm = self._model_for_call()
676+
if structured_llm:
653677
try:
654-
response = await _ainvoke_with_usage(
655-
self._structured_llm, prompt, self._usage_collector
656-
)
678+
response = await _ainvoke_with_usage(structured_llm, prompt, self._usage_collector)
657679
except (StructuredOutputParseError, ValidationError) as exc:
658680
raise _StructuredResponseValidationError from exc
659681
else:
660682
response = _raw_response_text(
661-
await _ainvoke_with_usage(self._llm, prompt, self._usage_collector)
683+
await _ainvoke_with_usage(llm, prompt, self._usage_collector)
662684
)
663685
logger.debug("LLM response for %s", batch.file_label)
664686
return batch, self.parse_response(response, batch)

‎src/skillspector/llm_utils.py‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,11 +222,19 @@ class _StructuredAgentCLIModel:
222222
``complete()``, then parses and validates the response into *schema*.
223223
"""
224224

225-
def __init__(self, provider: object, model: str, max_output_tokens: int, schema: type) -> None:
225+
def __init__(
226+
self,
227+
provider: object,
228+
model: str,
229+
max_output_tokens: int,
230+
schema: type,
231+
timeout: float | None = None,
232+
) -> None:
226233
self._provider = provider
227234
self._model = model
228235
self._max_output_tokens = max_output_tokens
229236
self._schema = schema
237+
self._timeout = timeout
230238

231239
def _augment(self, prompt: str) -> str:
232240
schema_json = json.dumps(self._schema.model_json_schema(), indent=2)
@@ -243,6 +251,7 @@ def _complete(self, prompt: str) -> str:
243251
self._augment(prompt),
244252
model=self._model,
245253
max_output_tokens=self._max_output_tokens,
254+
timeout=self._timeout,
246255
)
247256

248257
def invoke(self, prompt: str) -> object:
@@ -282,10 +291,13 @@ class AgentCLIChatModel:
282291
message rather than a confusing ``AttributeError``.
283292
"""
284293

285-
def __init__(self, provider: object, model: str, max_output_tokens: int) -> None:
294+
def __init__(
295+
self, provider: object, model: str, max_output_tokens: int, timeout: float | None = None
296+
) -> None:
286297
self._provider = provider
287298
self._model = model
288299
self._max_output_tokens = max_output_tokens
300+
self._timeout = timeout
289301

290302
def batch(self, *args: object, **kwargs: object) -> NoReturn:
291303
raise NotImplementedError(
@@ -304,6 +316,7 @@ def invoke(self, prompt: str) -> _AgentCLIMessage:
304316
prompt,
305317
model=self._model,
306318
max_output_tokens=self._max_output_tokens,
319+
timeout=self._timeout,
307320
)
308321
return _AgentCLIMessage(text)
309322

@@ -312,11 +325,13 @@ async def ainvoke(self, prompt: str) -> _AgentCLIMessage:
312325

313326
def with_structured_output(self, schema: type) -> _StructuredAgentCLIModel:
314327
return _StructuredAgentCLIModel(
315-
self._provider, self._model, self._max_output_tokens, schema
328+
self._provider, self._model, self._max_output_tokens, schema, self._timeout
316329
)
317330

318331

319-
def get_chat_model(model: str | None = None) -> BaseChatModel | AgentCLIChatModel:
332+
def get_chat_model(
333+
model: str | None = None, *, timeout: float | None = None
334+
) -> BaseChatModel | AgentCLIChatModel:
320335
"""Return a chat model for the active provider.
321336
322337
For CLI providers (``claude_cli``, ``codex_cli``, ``gemini_cli``) this
@@ -339,6 +354,7 @@ def get_chat_model(model: str | None = None) -> BaseChatModel | AgentCLIChatMode
339354
provider,
340355
resolved_model,
341356
get_max_output_tokens(resolved_model),
357+
timeout,
342358
)
343359
register_chat_model_provider(chat_model, provider)
344360
return chat_model
@@ -347,7 +363,7 @@ def get_chat_model(model: str | None = None) -> BaseChatModel | AgentCLIChatMode
347363
chat_model, effective_provider = create_chat_model_with_provider(
348364
model=model,
349365
max_tokens=get_max_output_tokens(model),
350-
timeout=120,
366+
timeout=timeout if timeout is not None else 120,
351367
)
352368
register_chat_model_provider(chat_model, effective_provider)
353369
return chat_model
@@ -409,6 +425,7 @@ def chat_completion(
409425
usage_collector: InferenceUsageCollector | None = None,
410426
node: str = "chat_completion",
411427
request_kind: str = "chat_completion",
428+
timeout: float | None = None,
412429
) -> str:
413430
"""Request a single chat completion and return the assistant content.
414431
@@ -419,7 +436,7 @@ def chat_completion(
419436
which normalise content blocks to a single string) and falls back to
420437
``.content`` for the CLI adapter's ``_AgentCLIMessage``.
421438
"""
422-
chat_model = get_chat_model(model=model)
439+
chat_model = get_chat_model(model=model, timeout=timeout)
423440
active_provider = get_active_provider()
424441
resolved_model = str(
425442
model

‎src/skillspector/models.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,12 @@ class Finding:
8989
tags: list[str] = field(default_factory=list)
9090
context: str | None = None
9191
matched_text: str | None = None
92+
transitive_depth: int = 0
93+
source_url: str | None = None
9294

9395
def to_dict(self) -> dict[str, object]:
9496
"""Return a JSON-serializable dict representation (full finding shape)."""
95-
return {
97+
data: dict[str, object] = {
9698
"id": self.rule_id,
9799
"finding_id": self.finding_id,
98100
"category": self.category,
@@ -113,6 +115,11 @@ def to_dict(self) -> dict[str, object]:
113115
# finding the LLM filter did not confirm but which is preserved anyway).
114116
"tags": list(self.tags),
115117
}
118+
if self.transitive_depth:
119+
data["transitive_depth"] = self.transitive_depth
120+
if self.source_url:
121+
data["source_url"] = self.source_url
122+
return data
116123

117124
def __str__(self) -> str:
118125
return f"{self.rule_id}: {self.message} ({self.file}:{self.start_line})"

‎src/skillspector/nodes/build_context.py‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,13 @@
4747
)
4848
from skillspector.logging_config import get_logger
4949
from skillspector.python_ast import prewarm_python_ast_cache
50-
from skillspector.state import SkillspectorState
50+
from skillspector.state import (
51+
SkillspectorState,
52+
transitive_note_truncation,
53+
transitive_remaining_bytes,
54+
transitive_remaining_seconds,
55+
transitive_traversal_state,
56+
)
5157

5258
logger = get_logger(__name__)
5359

@@ -360,13 +366,22 @@ def _build_component_metadata(
360366

361367

362368
def _read_file_cache(
363-
skill_dir: Path, components: list[str]
369+
skill_dir: Path,
370+
components: list[str],
371+
state: SkillspectorState | None = None,
364372
) -> tuple[dict[str, str], list[InspectionLedgerEvent]]:
365373
"""Build readable file content and terminal events for cache failures."""
366374
file_cache: dict[str, str] = {}
367375
ledger_events: list[InspectionLedgerEvent] = []
368376
skill_root = skill_dir.resolve(strict=False)
377+
traversal = transitive_traversal_state(state) if state is not None else None
378+
remaining_bytes = transitive_remaining_bytes(state) if state is not None else None
369379
for path in components:
380+
remaining_seconds = transitive_remaining_seconds(state) if state is not None else None
381+
if remaining_seconds is not None and remaining_seconds <= 0:
382+
if state is not None:
383+
transitive_note_truncation(state, f"time budget exhausted before reading {path}")
384+
break
370385
full = skill_dir / path
371386
if _is_symlink(full) or _resolves_outside(full, skill_root):
372387
ledger_events.append(
@@ -405,6 +420,10 @@ def _read_file_cache(
405420
)
406421
)
407422
continue
423+
if remaining_bytes is not None and file_stat.st_size > remaining_bytes:
424+
if state is not None:
425+
transitive_note_truncation(state, f"byte budget exhausted before reading {path}")
426+
break
408427
if not S_ISREG(file_stat.st_mode):
409428
ledger_events.append(
410429
ledger_event(
@@ -418,7 +437,16 @@ def _read_file_cache(
418437
continue
419438
try:
420439
content = _read_text_no_follow(full)
440+
content_bytes = len(content.encode("utf-8"))
441+
if remaining_bytes is not None and content_bytes > remaining_bytes:
442+
if state is not None:
443+
transitive_note_truncation(state, f"byte budget exhausted while reading {path}")
444+
break
421445
file_cache[path] = content
446+
record_bytes = getattr(traversal, "record_bytes", None)
447+
if callable(record_bytes):
448+
record_bytes(content_bytes)
449+
remaining_bytes = transitive_remaining_bytes(state) if state is not None else None
422450
except FileNotFoundError as exc:
423451
ledger_events.append(
424452
ledger_event(
@@ -570,7 +598,7 @@ def build_context(state: SkillspectorState) -> dict[str, object]:
570598
)
571599
for path in sorted(selected_baselines)
572600
]
573-
file_cache, cache_events = _read_file_cache(skill_dir, components)
601+
file_cache, cache_events = _read_file_cache(skill_dir, components, state)
574602
python_ast_cache_key = prewarm_python_ast_cache(components, file_cache)
575603
manifest = _parse_manifest(skill_dir)
576604
metadata_components = [

‎src/skillspector/nodes/deduplicate.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,16 +33,16 @@
3333
logger = get_logger(__name__)
3434

3535

36-
def _same_file_key(finding: Finding) -> tuple[str, str, str]:
36+
def _same_file_key(finding: Finding) -> tuple[str, str, str, str]:
3737
"""Build a deduplication key for same-file matches."""
3838
matched = (finding.matched_text or "").strip()[:100]
39-
return (finding.rule_id, finding.file, matched)
39+
return (finding.rule_id, finding.file, matched, finding.source_url or "")
4040

4141

42-
def _cross_file_key(finding: Finding) -> tuple[str, str]:
42+
def _cross_file_key(finding: Finding) -> tuple[str, str, str]:
4343
"""Build a cross-file deduplication key from rule_id and normalized matched_text."""
4444
matched = (finding.matched_text or "").strip()[:100]
45-
return (finding.rule_id, matched)
45+
return (finding.rule_id, matched, finding.source_url or "")
4646

4747

4848
def deduplicate(findings: list[Finding]) -> list[Finding]:
@@ -62,7 +62,7 @@ def deduplicate(findings: list[Finding]) -> list[Finding]:
6262
original_count = len(findings)
6363

6464
# Pass 1: Same-file deduplication
65-
same_file_best: dict[tuple[str, str, str], Finding] = {}
65+
same_file_best: dict[tuple[str, str, str, str], Finding] = {}
6666
for f in findings:
6767
key = _same_file_key(f)
6868
existing = same_file_best.get(key)
@@ -72,7 +72,7 @@ def deduplicate(findings: list[Finding]) -> list[Finding]:
7272
after_same_file = list(same_file_best.values())
7373

7474
# Pass 2: Cross-file deduplication (only for findings WITH matched_text)
75-
cross_file_best: dict[tuple[str, str], Finding] = {}
75+
cross_file_best: dict[tuple[str, str, str], Finding] = {}
7676
no_text_findings: list[Finding] = []
7777

7878
for f in after_same_file:

0 commit comments

Comments
 (0)