1717
1818from __future__ import annotations
1919
20+ import asyncio
2021import base64
2122import json
2223import re
2324from pathlib import Path
24- from unittest .mock import MagicMock
25+ from unittest .mock import AsyncMock , MagicMock
2526
2627import pytest
2728import yaml
2829from pydantic import BaseModel
2930
3031from skillspector .inspection_ledger import LedgerOutcome , LedgerReason
32+ from skillspector .llm_analyzer_base import LLMAnalyzerBase
3133from skillspector .llm_utils import AgentCLIChatModel
3234from skillspector .nodes .analyzers import mcp_tool_poisoning
3335from skillspector .nodes .deduplicate import deduplicate
@@ -214,6 +216,9 @@ def invoke_with_usage(self, prompt: str, collector: object) -> object:
214216 return self .response_schema .model_validate (response )
215217 return response
216218
219+ async def ainvoke_with_usage (self , prompt : str , collector : object ) -> object :
220+ return self .invoke_with_usage (prompt , collector )
221+
217222
218223class _FakeChatModel :
219224 def __init__ (self , structured_llm : _FakeStructuredLLM ) -> None :
@@ -1641,7 +1646,7 @@ def test_llm_call_failure_returns_empty(self, monkeypatch: pytest.MonkeyPatch):
16411646
16421647 def test_persistently_malformed_response_returns_empty (self , monkeypatch : pytest .MonkeyPatch ):
16431648 state = _make_state ("mcp_mismatched_skill" , use_llm = True )
1644- monkeypatch .setattr ("skillspector.llm_analyzer_base.time.sleep " , lambda _delay : None )
1649+ monkeypatch .setattr (LLMAnalyzerBase , "_asleep_before_retry " , AsyncMock () )
16451650 structured_llm = _mock_tp4_structured_llm (
16461651 monkeypatch ,
16471652 [{}, {}, {}, {}],
@@ -1659,8 +1664,8 @@ def test_persistently_malformed_response_returns_empty(self, monkeypatch: pytest
16591664
16601665 def test_malformed_response_is_retried (self , monkeypatch : pytest .MonkeyPatch ):
16611666 state = _make_state ("mcp_mismatched_skill" , use_llm = True )
1662- sleep = MagicMock ()
1663- monkeypatch .setattr ("skillspector.llm_analyzer_base.time.sleep " , sleep )
1667+ sleep = AsyncMock ()
1668+ monkeypatch .setattr (LLMAnalyzerBase , "_asleep_before_retry " , sleep )
16641669 structured_llm = _mock_tp4_structured_llm (
16651670 monkeypatch ,
16661671 [{}, {"is_mismatch" : False }],
@@ -1675,8 +1680,8 @@ def test_malformed_response_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16751680
16761681 def test_cli_parse_error_is_retried (self , monkeypatch : pytest .MonkeyPatch ):
16771682 state = _make_state ("mcp_mismatched_skill" , use_llm = True )
1678- sleep = MagicMock ()
1679- monkeypatch .setattr ("skillspector.llm_analyzer_base.time.sleep " , sleep )
1683+ sleep = AsyncMock ()
1684+ monkeypatch .setattr (LLMAnalyzerBase , "_asleep_before_retry " , sleep )
16801685 provider = MagicMock ()
16811686 provider .complete .side_effect = ["not JSON" , '{"is_mismatch": false}' ]
16821687 monkeypatch .setattr (
@@ -1692,8 +1697,8 @@ def test_cli_parse_error_is_retried(self, monkeypatch: pytest.MonkeyPatch):
16921697
16931698 def test_out_of_range_confidence_is_retried (self , monkeypatch : pytest .MonkeyPatch ):
16941699 state = _make_state ("mcp_mismatched_skill" , use_llm = True )
1695- sleep = MagicMock ()
1696- monkeypatch .setattr ("skillspector.llm_analyzer_base.time.sleep " , sleep )
1700+ sleep = AsyncMock ()
1701+ monkeypatch .setattr (LLMAnalyzerBase , "_asleep_before_retry " , sleep )
16971702 structured_llm = _mock_tp4_structured_llm (
16981703 monkeypatch ,
16991704 [{"is_mismatch" : True , "confidence" : 1.7 }, {"is_mismatch" : False }],
@@ -1707,6 +1712,49 @@ def test_out_of_range_confidence_is_retried(self, monkeypatch: pytest.MonkeyPatc
17071712 assert result ["llm_call_log" ] == [{"node" : "mcp_tool_poisoning" , "ok" : True , "error" : None }]
17081713
17091714
1715+ class TestTP4Concurrency :
1716+ def test_batches_run_concurrently_within_the_shared_limit (
1717+ self , monkeypatch : pytest .MonkeyPatch
1718+ ) -> None :
1719+ """TP4 fans out its batches like the semantic analyzers instead of one by one."""
1720+ state = {
1721+ "manifest" : {"name" : "concurrency" , "description" : "Runs local calculations." },
1722+ "file_cache" : {
1723+ "tool.py" : "" .join (f"value_{ index } = { index } \n " for index in range (300 ))
1724+ },
1725+ "component_metadata" : [{"path" : "tool.py" , "type" : "python" }],
1726+ "use_llm" : True ,
1727+ "model_config" : {"default" : "test-model" },
1728+ }
1729+ monkeypatch .setenv ("SKILLSPECTOR_MAX_LLM_CONCURRENCY" , "2" )
1730+ monkeypatch .setattr (mcp_tool_poisoning , "TP4_MAX_BATCH_INPUT_TOKENS" , 256 )
1731+ monkeypatch .setattr (mcp_tool_poisoning , "TP4_MAX_BATCHES" , 4 )
1732+ monkeypatch .setattr (mcp_tool_poisoning , "TP4_MIN_CODE_TOKENS" , 1 )
1733+ monkeypatch .setattr (mcp_tool_poisoning , "get_max_input_tokens" , lambda _model : 2048 )
1734+ structured = _mock_tp4_structured_llm (
1735+ monkeypatch , [{"is_mismatch" : False } for _ in range (4 )]
1736+ )
1737+ in_flight = 0
1738+ peak = 0
1739+
1740+ async def slow_ainvoke (llm : _FakeStructuredLLM , prompt : str , collector : object ) -> object :
1741+ nonlocal in_flight , peak
1742+ in_flight += 1
1743+ peak = max (peak , in_flight )
1744+ await asyncio .sleep (0.05 )
1745+ in_flight -= 1
1746+ return llm .invoke_with_usage (prompt , collector )
1747+
1748+ monkeypatch .setattr (_FakeStructuredLLM , "ainvoke_with_usage" , slow_ainvoke )
1749+
1750+ result = node (state )
1751+
1752+ assert structured .calls == 4
1753+ assert peak == 2
1754+ semantic = [event for event in result ["inspection_ledger" ] if event ["phase" ] == "semantic" ]
1755+ assert sum (event ["outcome" ] is LedgerOutcome .COMPLETED for event in semantic ) == 4
1756+
1757+
17101758class TestTP4Telemetry :
17111759 """TP4 records llm_call_log so the report's degradation detector counts it
17121760 consistently with the semantic analyzers and the meta-analyzer."""
0 commit comments