|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
| 3 | +import asyncio |
3 | 4 | import json |
4 | 5 | from typing import Any |
5 | 6 |
|
@@ -298,6 +299,68 @@ async def test_a_post_output_callback_failure_keeps_the_executed_output(streamed |
298 | 299 | assert completed._pending_session_write is None |
299 | 300 |
|
300 | 301 |
|
| 302 | +@pytest.mark.asyncio |
| 303 | +@pytest.mark.parametrize("streamed", [False, True]) |
| 304 | +async def test_callback_failure_preserves_model_order_for_completed_outputs(streamed: bool) -> None: |
| 305 | + second_committed = asyncio.Event() |
| 306 | + executions: list[str] = [] |
| 307 | + |
| 308 | + def mark_second_committed(ctx: Any) -> None: |
| 309 | + second_committed.set() |
| 310 | + |
| 311 | + @function_tool(needs_approval=True, custom_data_extractor=_boom_custom_data_extractor) |
| 312 | + async def first() -> str: |
| 313 | + await second_committed.wait() |
| 314 | + executions.append("first") |
| 315 | + return "first result" |
| 316 | + |
| 317 | + @function_tool(needs_approval=True, custom_data_extractor=mark_second_committed) |
| 318 | + async def second() -> str: |
| 319 | + executions.append("second") |
| 320 | + return "second result" |
| 321 | + |
| 322 | + agent = Agent( |
| 323 | + name="ordered callback recovery", |
| 324 | + tools=[first, second], |
| 325 | + output_guardrails=[always_fine], |
| 326 | + tool_use_behavior=StopAtTools(stop_at_tool_names=["first"]), |
| 327 | + model=ScriptedModel( |
| 328 | + [ |
| 329 | + ModelStep( |
| 330 | + output=[ |
| 331 | + function_call("first", {}, call_id="call_FIRST"), |
| 332 | + function_call("second", {}, call_id="call_SECOND"), |
| 333 | + ] |
| 334 | + ) |
| 335 | + ] |
| 336 | + ), |
| 337 | + ) |
| 338 | + session = SimpleListSession() |
| 339 | + parked = await _run(agent, "go", session, streamed=streamed) |
| 340 | + state = await RunState.from_json(agent, parked.to_state().to_json()) |
| 341 | + for interruption in state.get_interruptions(): |
| 342 | + state.approve(interruption) |
| 343 | + |
| 344 | + # The callback signals after the second output is committed, ensuring that |
| 345 | + # completion order differs from the model's call order before the failure. |
| 346 | + with pytest.raises(Exception, match="extractor boom"): |
| 347 | + await _run(agent, state, session, streamed=streamed) |
| 348 | + |
| 349 | + restored = await RunState.from_json(agent, json.loads(json.dumps(state.to_json()))) |
| 350 | + agent.output_guardrails = [] |
| 351 | + result = await _run(agent, restored, session, streamed=streamed) |
| 352 | + |
| 353 | + assert result.final_output == "first result" |
| 354 | + assert executions == ["second", "first"] |
| 355 | + assert [ |
| 356 | + (item["call_id"], item["output"]) |
| 357 | + for item in await session.get_items() |
| 358 | + if item.get("type") == "function_call_output" |
| 359 | + ] == [("call_FIRST", "first result"), ("call_SECOND", "second result")] |
| 360 | + completed = await RunState.from_json(agent, result.to_state().to_json()) |
| 361 | + assert completed._pending_session_write is None |
| 362 | + |
| 363 | + |
301 | 364 | @pytest.mark.parametrize("streamed", [False, True]) |
302 | 365 | async def test_checkpointed_mcp_approval_survives_callback_failure_and_json_retry( |
303 | 366 | streamed: bool, |
|
0 commit comments