Skip to content

Commit 9919c15

Browse files
Run TP4 batches concurrently
_check_tp4 executed its LLM batches through the serial run_batches_detailed loop, while the semantic analyzers and the meta-review fan out through arun_batches. On skills with many code chunks TP4 became the long tail of the scan and could exhaust the workflow deadline on its own. Use run_async(analyzer.arun_batches_detailed(batches)). The shared limiter still honours SKILLSPECTOR_MAX_LLM_CONCURRENCY (1 keeps the old serial behaviour), failures are recorded per batch as before, and successes keep the batch order, so the TP4_MAX_FINDINGS cap and the ledger projection are unchanged. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: elliottwaves-20 <[email protected]>
1 parent 4a55062 commit 9919c15

3 files changed

Lines changed: 61 additions & 10 deletions

File tree

‎src/skillspector/nodes/analyzers/mcp_tool_poisoning.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
append_output_language_instruction,
4646
estimate_tokens,
4747
)
48+
from skillspector.llm_utils import run_async
4849
from skillspector.model_info import get_max_input_tokens
4950
from skillspector.models import Finding, compute_match_fingerprint
5051
from skillspector.nodes.analyzers.static_runner import MAX_FINDINGS_PER_ANALYZER
@@ -1482,7 +1483,9 @@ def iter_candidates() -> Iterator[_TP4Candidate]:
14821483
)
14831484
analyzer = _TP4Analyzer(model, timeout=timeout)
14841485
attempted = True
1485-
batch_outcome = analyzer.run_batches_detailed(batches)
1486+
# Fan out like the semantic analyzers do. The shared limiter still honours
1487+
# SKILLSPECTOR_MAX_LLM_CONCURRENCY, and results keep the batch order.
1488+
batch_outcome = run_async(analyzer.arun_batches_detailed(batches))
14861489
result.inference_usage = cast(list[InferenceUsageRecord], analyzer.inference_usage)
14871490
seen_finding_ids: set[str] = set()
14881491
unexpected_response = False

‎tests/integration/test_graph.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ def inference_usage(self) -> list[object]:
290290
def __init__(self, _model: str, **_kwargs: object) -> None:
291291
pass
292292

293-
def run_batches_detailed(self, _batches: object) -> object:
293+
async def arun_batches_detailed(self, _batches: object) -> object:
294294
raise RuntimeError("simulated LLM transport failure")
295295

296296
# Semantic analyzers and meta_analyzer fail while constructing their shared

‎tests/test_mcp_tool_poisoning.py‎

Lines changed: 56 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,19 @@
1717

1818
from __future__ import annotations
1919

20+
import asyncio
2021
import base64
2122
import json
2223
import re
2324
from pathlib import Path
24-
from unittest.mock import MagicMock
25+
from unittest.mock import AsyncMock, MagicMock
2526

2627
import pytest
2728
import yaml
2829
from pydantic import BaseModel
2930

3031
from skillspector.inspection_ledger import LedgerOutcome, LedgerReason
32+
from skillspector.llm_analyzer_base import LLMAnalyzerBase
3133
from skillspector.llm_utils import AgentCLIChatModel
3234
from skillspector.nodes.analyzers import mcp_tool_poisoning
3335
from skillspector.nodes.deduplicate import deduplicate
@@ -214,6 +216,9 @@ def invoke_with_usage(self, prompt: str, collector: object) -> object:
214216
return self.response_schema.model_validate(response)
215217
return response
216218

219+
async def ainvoke_with_usage(self, prompt: str, collector: object) -> object:
220+
return self.invoke_with_usage(prompt, collector)
221+
217222

218223
class _FakeChatModel:
219224
def __init__(self, structured_llm: _FakeStructuredLLM) -> None:
@@ -1641,7 +1646,7 @@ def test_llm_call_failure_returns_empty(self, monkeypatch: pytest.MonkeyPatch):
16411646

16421647
def test_persistently_malformed_response_returns_empty(self, monkeypatch: pytest.MonkeyPatch):
16431648
state = _make_state("mcp_mismatched_skill", use_llm=True)
1644-
monkeypatch.setattr("skillspector.llm_analyzer_base.time.sleep", lambda _delay: None)
1649+
monkeypatch.setattr(LLMAnalyzerBase, "_asleep_before_retry", AsyncMock())
16451650
structured_llm = _mock_tp4_structured_llm(
16461651
monkeypatch,
16471652
[{}, {}, {}, {}],
@@ -1659,8 +1664,8 @@ def test_persistently_malformed_response_returns_empty(self, monkeypatch: pytest
16591664

16601665
def test_malformed_response_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16611666
state = _make_state("mcp_mismatched_skill", use_llm=True)
1662-
sleep = MagicMock()
1663-
monkeypatch.setattr("skillspector.llm_analyzer_base.time.sleep", sleep)
1667+
sleep = AsyncMock()
1668+
monkeypatch.setattr(LLMAnalyzerBase, "_asleep_before_retry", sleep)
16641669
structured_llm = _mock_tp4_structured_llm(
16651670
monkeypatch,
16661671
[{}, {"is_mismatch": False}],
@@ -1675,8 +1680,8 @@ def test_malformed_response_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16751680

16761681
def test_cli_parse_error_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16771682
state = _make_state("mcp_mismatched_skill", use_llm=True)
1678-
sleep = MagicMock()
1679-
monkeypatch.setattr("skillspector.llm_analyzer_base.time.sleep", sleep)
1683+
sleep = AsyncMock()
1684+
monkeypatch.setattr(LLMAnalyzerBase, "_asleep_before_retry", sleep)
16801685
provider = MagicMock()
16811686
provider.complete.side_effect = ["not JSON", '{"is_mismatch": false}']
16821687
monkeypatch.setattr(
@@ -1692,8 +1697,8 @@ def test_cli_parse_error_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16921697

16931698
def test_out_of_range_confidence_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16941699
state = _make_state("mcp_mismatched_skill", use_llm=True)
1695-
sleep = MagicMock()
1696-
monkeypatch.setattr("skillspector.llm_analyzer_base.time.sleep", sleep)
1700+
sleep = AsyncMock()
1701+
monkeypatch.setattr(LLMAnalyzerBase, "_asleep_before_retry", sleep)
16971702
structured_llm = _mock_tp4_structured_llm(
16981703
monkeypatch,
16991704
[{"is_mismatch": True, "confidence": 1.7}, {"is_mismatch": False}],
@@ -1707,6 +1712,49 @@ def test_out_of_range_confidence_is_retried(self, monkeypatch: pytest.MonkeyPatc
17071712
assert result["llm_call_log"] == [{"node": "mcp_tool_poisoning", "ok": True, "error": None}]
17081713

17091714

1715+
class TestTP4Concurrency:
1716+
def test_batches_run_concurrently_within_the_shared_limit(
1717+
self, monkeypatch: pytest.MonkeyPatch
1718+
) -> None:
1719+
"""TP4 fans out its batches like the semantic analyzers instead of one by one."""
1720+
state = {
1721+
"manifest": {"name": "concurrency", "description": "Runs local calculations."},
1722+
"file_cache": {
1723+
"tool.py": "".join(f"value_{index} = {index}\n" for index in range(300))
1724+
},
1725+
"component_metadata": [{"path": "tool.py", "type": "python"}],
1726+
"use_llm": True,
1727+
"model_config": {"default": "test-model"},
1728+
}
1729+
monkeypatch.setenv("SKILLSPECTOR_MAX_LLM_CONCURRENCY", "2")
1730+
monkeypatch.setattr(mcp_tool_poisoning, "TP4_MAX_BATCH_INPUT_TOKENS", 256)
1731+
monkeypatch.setattr(mcp_tool_poisoning, "TP4_MAX_BATCHES", 4)
1732+
monkeypatch.setattr(mcp_tool_poisoning, "TP4_MIN_CODE_TOKENS", 1)
1733+
monkeypatch.setattr(mcp_tool_poisoning, "get_max_input_tokens", lambda _model: 2048)
1734+
structured = _mock_tp4_structured_llm(
1735+
monkeypatch, [{"is_mismatch": False} for _ in range(4)]
1736+
)
1737+
in_flight = 0
1738+
peak = 0
1739+
1740+
async def slow_ainvoke(llm: _FakeStructuredLLM, prompt: str, collector: object) -> object:
1741+
nonlocal in_flight, peak
1742+
in_flight += 1
1743+
peak = max(peak, in_flight)
1744+
await asyncio.sleep(0.05)
1745+
in_flight -= 1
1746+
return llm.invoke_with_usage(prompt, collector)
1747+
1748+
monkeypatch.setattr(_FakeStructuredLLM, "ainvoke_with_usage", slow_ainvoke)
1749+
1750+
result = node(state)
1751+
1752+
assert structured.calls == 4
1753+
assert peak == 2
1754+
semantic = [event for event in result["inspection_ledger"] if event["phase"] == "semantic"]
1755+
assert sum(event["outcome"] is LedgerOutcome.COMPLETED for event in semantic) == 4
1756+
1757+
17101758
class TestTP4Telemetry:
17111759
"""TP4 records llm_call_log so the report's degradation detector counts it
17121760
consistently with the semantic analyzers and the meta-analyzer."""

0 commit comments

Comments
 (0)