2525@pytest .fixture
2626def openai_http_endpoint (monkeypatch ):
2727 requests = []
28- behavior = {"mode" : "clean" , "tp4_calls" : 0 }
28+ behavior = {"mode" : "clean" , "tp4_calls" : 0 , "tool_name" : "_TP4AnalysisResult" }
2929
3030 class Handler (BaseHTTPRequestHandler ):
3131 def log_message (self , * _args ):
@@ -35,17 +35,20 @@ def do_POST(self):
3535 body = json .loads (self .rfile .read (int (self .headers ["Content-Length" ])))
3636 requests .append (body )
3737 tools = body .get ("tools" , [])
38- is_tp4 = bool (tools and tools [0 ]["function" ]["name" ] == "_TP4AnalysisResult" )
38+ tool_name = behavior ["tool_name" ]
39+ is_tp4 = bool (tools and tools [0 ]["function" ]["name" ] == tool_name )
3940 if not is_tp4 :
4041 response_format = body .get ("response_format" , {})
41- is_tp4 = response_format .get ("json_schema" , {}).get ("name" ) == "_TP4AnalysisResult"
42+ is_tp4 = response_format .get ("json_schema" , {}).get ("name" ) == tool_name
4243 message = {"role" : "assistant" , "content" : json .dumps ({"findings" : []})}
4344 finish_reason = "stop"
4445 if is_tp4 :
4546 behavior ["tp4_calls" ] += 1
4647 mode = behavior ["mode" ]
47- if mode == "recover" and behavior ["tp4_calls" ] > 1 :
48+ if mode in { "recover" , "recover-multiple" } and behavior ["tp4_calls" ] > 1 :
4849 mode = "clean"
50+ if mode == "recover-multiple" :
51+ mode = "multiple-mismatch"
4952 if mode in {"refusal" , "recover" }:
5053 message = {"role" : "assistant" , "content" : "Cannot provide this assessment." }
5154 else :
@@ -59,19 +62,44 @@ def do_POST(self):
5962 }
6063 encoded = "not valid JSON" if mode == "malformed" else json .dumps (arguments )
6164 if tools :
62- message = {
63- "role" : "assistant" ,
64- "content" : None ,
65- "tool_calls" : [
65+ tool_calls = [
66+ {
67+ "id" : "call_tp4" ,
68+ "type" : "function" ,
69+ "function" : {
70+ "name" : tool_name ,
71+ "arguments" : encoded ,
72+ },
73+ }
74+ ]
75+ if mode .startswith ("multiple-" ):
76+ second_arguments = dict (arguments )
77+ if mode == "multiple-mismatch" :
78+ second_arguments .update (
79+ is_mismatch = True , mismatched_capabilities = ["persistence" ]
80+ )
81+ tool_calls .append (
6682 {
67- "id" : "call_tp4 " ,
83+ "id" : "call_tp4_second " ,
6884 "type" : "function" ,
6985 "function" : {
70- "name" : "_TP4AnalysisResult" ,
71- "arguments" : encoded ,
86+ "name" : (
87+ "UnexpectedAssessment"
88+ if mode == "multiple-unknown"
89+ else tool_name
90+ ),
91+ "arguments" : (
92+ "not valid JSON"
93+ if mode == "multiple-malformed"
94+ else json .dumps (second_arguments )
95+ ),
7296 },
7397 }
74- ],
98+ )
99+ message = {
100+ "role" : "assistant" ,
101+ "content" : None ,
102+ "tool_calls" : tool_calls ,
75103 }
76104 finish_reason = "tool_calls"
77105 else :
@@ -96,6 +124,7 @@ def do_POST(self):
96124 thread .start ()
97125 monkeypatch .setenv ("OPENAI_API_KEY" , "sk-loopback-test" )
98126 monkeypatch .setenv ("OPENAI_BASE_URL" , f"http://127.0.0.1:{ server .server_port } /v1" )
127+ monkeypatch .setenv ("SKILLSPECTOR_MODEL" , _MODEL )
99128 monkeypatch .setenv ("LANGSMITH_TRACING" , "false" )
100129 monkeypatch .delenv ("SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD" , raising = False )
101130 monkeypatch .delenv ("SKILLSPECTOR_REASONING_EFFORT" , raising = False )
@@ -118,6 +147,11 @@ def do_POST(self):
118147 ("refusal" , 4 , False ),
119148 ("malformed" , 4 , False ),
120149 ("out-of-range" , 4 , False ),
150+ ("multiple-clean" , 4 , False ),
151+ ("multiple-mismatch" , 4 , False ),
152+ ("multiple-unknown" , 4 , False ),
153+ ("multiple-malformed" , 4 , False ),
154+ ("recover-multiple" , 2 , True ),
121155 ],
122156)
123157@pytest .mark .parametrize ("refresh" , [False , True ])
@@ -140,6 +174,7 @@ def test_tp4_real_http_retries_and_deadline_refresh(
140174 "type" : "function" ,
141175 "function" : {"name" : "_TP4AnalysisResult" },
142176 }
177+ assert request ["parallel_tool_calls" ] is False
143178 assert "response_format" not in request
144179 assert "Report your result by calling" not in request ["messages" ][0 ]["content" ]
145180 if not complete :
@@ -162,13 +197,41 @@ def test_reasoning_effort_uses_existing_json_schema_route_over_real_http(
162197 assert "tool_choice" not in requests [0 ]
163198
164199
200+ def test_real_http_validation_uses_schema_title_as_tool_name (openai_http_endpoint ):
201+ requests , behavior = openai_http_endpoint
202+ behavior ["tool_name" ] = "GatewayTP4Assessment"
203+
204+ class TitledTP4Assessment (tp ._TP4AnalysisResult ):
205+ model_config = {"title" : "GatewayTP4Assessment" }
206+
207+ class TitledTP4Analyzer (tp ._TP4Analyzer ):
208+ response_schema = TitledTP4Assessment
209+
210+ analyzer = TitledTP4Analyzer (model = _MODEL , timeout = 30.0 )
211+ outcome = analyzer .run_batches_detailed ([tp .Batch (file_path = "format.py" , content = "Assess" )])
212+
213+ assert behavior ["tp4_calls" ] == 1
214+ assert outcome .successful and not outcome .failures
215+ assert isinstance (outcome .successful [0 ][1 ][0 ], TitledTP4Assessment )
216+ assert requests [0 ]["tool_choice" ] == {
217+ "type" : "function" ,
218+ "function" : {"name" : "GatewayTP4Assessment" },
219+ }
220+
221+
165222@pytest .mark .asyncio
166223@pytest .mark .parametrize (
167224 "mode,attempts,complete" ,
168225 [
169226 ("clean" , 1 , True ),
227+ ("mismatch" , 1 , True ),
170228 ("refusal" , 4 , False ),
171229 ("malformed" , 4 , False ),
230+ ("multiple-clean" , 4 , False ),
231+ ("multiple-mismatch" , 4 , False ),
232+ ("multiple-unknown" , 4 , False ),
233+ ("multiple-malformed" , 4 , False ),
234+ ("recover-multiple" , 2 , True ),
172235 ],
173236)
174237async def test_async_real_http_tool_results_use_same_validation (
@@ -190,9 +253,22 @@ async def test_async_real_http_tool_results_use_same_validation(
190253
191254
192255@pytest .mark .parametrize ("output_format" , ["json" , "sarif" ])
193- @pytest .mark .parametrize ("mode" , ["clean" , "mismatch" , "refusal" , "malformed" ])
256+ @pytest .mark .parametrize (
257+ "mode,attempts,complete" ,
258+ [
259+ ("clean" , 1 , True ),
260+ ("mismatch" , 1 , True ),
261+ ("refusal" , 4 , False ),
262+ ("malformed" , 4 , False ),
263+ ("multiple-clean" , 4 , False ),
264+ ("multiple-mismatch" , 4 , False ),
265+ ("multiple-unknown" , 4 , False ),
266+ ("multiple-malformed" , 4 , False ),
267+ ("recover-multiple" , 2 , True ),
268+ ],
269+ )
194270def test_full_scanner_graph_reports_tp4_http_results (
195- tmp_path : Path , openai_http_endpoint , output_format , mode
271+ tmp_path : Path , openai_http_endpoint , output_format , mode , attempts , complete
196272):
197273 from skillspector .graph import create_graph
198274
@@ -220,8 +296,8 @@ def test_full_scanner_graph_reports_tp4_http_results(
220296 for event in result ["analyzer_status_events" ]
221297 if event ["analyzer_id" ] == tp .ANALYZER_ID
222298 )
223- assert tp4_status ["status" ] == ("completed" if mode in { "clean" , "mismatch" } else "degraded" )
224- assert behavior ["tp4_calls" ] == ( 1 if mode in { "clean" , "mismatch" } else 4 )
299+ assert tp4_status ["status" ] == ("completed" if complete else "degraded" )
300+ assert behavior ["tp4_calls" ] == attempts
225301 findings = [f for f in result ["findings" ] if f .rule_id == "TP4" ]
226302 assert bool (findings ) is (mode == "mismatch" )
227303 if findings :
@@ -235,7 +311,6 @@ def test_full_scanner_graph_reports_tp4_http_results(
235311 if event ["analyzer_id" ] == tp .ANALYZER_ID and event ["path" ] == "format.py"
236312 )
237313 report = json .loads (result ["report_body" ])
238- complete = mode in {"clean" , "mismatch" }
239314 if output_format == "json" :
240315 assert report ["execution_successful" ] is True
241316 assert report ["analysis_completeness" ]["is_complete" ] is complete
@@ -249,4 +324,14 @@ def test_full_scanner_graph_reports_tp4_http_results(
249324 assert bool ([item for item in run ["results" ] if item ["ruleId" ] == "TP4" ]) is (
250325 mode == "mismatch"
251326 )
252- assert requests
327+ tp4_requests = [
328+ request
329+ for request in requests
330+ if request .get ("tools" ) and request ["tools" ][0 ]["function" ]["name" ] == "_TP4AnalysisResult"
331+ ]
332+ assert len (tp4_requests ) == attempts
333+ assert all (request ["model" ] == _MODEL for request in tp4_requests )
334+ assert all (
335+ request ["tool_choice" ] == {"type" : "function" , "function" : {"name" : "_TP4AnalysisResult" }}
336+ for request in tp4_requests
337+ )
0 commit comments