Created
August 25, 2026 04:19
-
-
Save FU-max-boop/1eadecf110e7505ddaf9c67eb74f6053 to your computer and use it in GitHub Desktop.
Provider-neutral NextStepInterruption Session-write and guardrail-publication reproducer for openai-agents-python#4646
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """Provider-neutral observability extension for openai-agents-python #4615. | |
| Run this against a pinned SDK source tree. It demonstrates that a Session | |
| ambiguous-ack failure after an approved tool loses completed tool-guardrail | |
| results from live/JSON state and every later public result, even though hooks, | |
| guardrails, and the tool side effect each executed exactly once. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| from copy import deepcopy | |
| from dataclasses import dataclass | |
| from typing import Any, Literal | |
| from agents import ( | |
| Agent, | |
| RunConfig, | |
| Runner, | |
| RunState, | |
| ToolGuardrailFunctionOutput, | |
| function_tool, | |
| tool_input_guardrail, | |
| tool_output_guardrail, | |
| ) | |
| from agents.lifecycle import RunHooks | |
| from agents.testing import ScriptedModel, assistant_message, function_call | |
| Mode = Literal["run", "streamed"] | |
| Outcome = Literal["absent", "committed"] | |
| Transport = Literal["live", "json"] | |
| class AmbiguousAckSession: | |
| """In-memory Session that either rejects or commits before raising once.""" | |
| def __init__(self, session_id: str, outcome: Outcome) -> None: | |
| self.session_id = session_id | |
| self.outcome = outcome | |
| self.items: list[dict[str, Any]] = [] | |
| self.fail_next_add = False | |
| self.failed_batch: list[dict[str, Any]] | None = None | |
| async def get_items(self, limit: int | None = None) -> list[dict[str, Any]]: | |
| items = deepcopy(self.items) | |
| return items[-limit:] if limit is not None else items | |
| async def add_items(self, items: list[dict[str, Any]]) -> None: | |
| owned = deepcopy(items) | |
| if self.fail_next_add: | |
| self.fail_next_add = False | |
| self.failed_batch = owned | |
| if self.outcome == "committed": | |
| self.items.extend(deepcopy(owned)) | |
| raise RuntimeError("injected ambiguous renewed-interruption write failure") | |
| self.items.extend(owned) | |
| async def pop_item(self) -> dict[str, Any] | None: | |
| return self.items.pop() if self.items else None | |
| async def clear_session(self) -> None: | |
| self.items.clear() | |
| @dataclass(frozen=True) | |
| class MatrixCase: | |
| name: str | |
| initial_mode: Mode | |
| failing_mode: Mode | |
| retry_mode: Mode | |
| transport: Transport | |
| outcome: Outcome | |
| @dataclass(frozen=True) | |
| class CaseResult: | |
| name: str | |
| outcome: Outcome | |
| transport: Transport | |
| step_after_failure: str | |
| counters: tuple[int, int, int, int, int] | |
| model_calls_before_decision: int | |
| model_calls_final: int | |
| state_guardrails: tuple[int, int] | |
| pending_guardrails: tuple[int, int] | |
| pending_state_guardrails: tuple[int, int] | |
| final_guardrails: tuple[int, int] | |
| session_before_retry: tuple[int, int, int, int] | |
| session_final: tuple[int, int, int, int] | |
| replay_final: tuple[int, int, int, int] | |
| CASES = ( | |
| MatrixCase("RR-live-absent", "run", "run", "run", "live", "absent"), | |
| MatrixCase("RS-json-commit", "run", "run", "streamed", "json", "committed"), | |
| MatrixCase("SR-live-commit", "streamed", "streamed", "run", "live", "committed"), | |
| MatrixCase("SS-json-absent", "streamed", "streamed", "streamed", "json", "absent"), | |
| ) | |
| async def invoke( | |
| mode: Mode, | |
| agent: Agent[Any], | |
| input: str | RunState[Any], | |
| *, | |
| session: AmbiguousAckSession, | |
| run_config: RunConfig, | |
| hooks: RunHooks[Any], | |
| ) -> Any: | |
| if mode == "run": | |
| return await Runner.run( | |
| agent, | |
| input, | |
| session=session, | |
| run_config=run_config, | |
| hooks=hooks, | |
| ) | |
| result = Runner.run_streamed( | |
| agent, | |
| input, | |
| session=session, | |
| run_config=run_config, | |
| hooks=hooks, | |
| ) | |
| async for _ in result.stream_events(): | |
| pass | |
| return result | |
| def item_value(item: Any, key: str) -> Any: | |
| return item.get(key) if isinstance(item, dict) else getattr(item, key, None) | |
| def call_id(item: Any) -> str: | |
| value = item_value(item, "call_id") | |
| assert isinstance(value, str) | |
| return value | |
| def history_counts(items: list[Any]) -> tuple[int, int, int, int]: | |
| return tuple( | |
| sum( | |
| item_value(item, "type") == item_type | |
| and item_value(item, "call_id") == expected_call_id | |
| for item in items | |
| ) | |
| for expected_call_id, item_type in ( | |
| ("c-charge", "function_call"), | |
| ("c-charge", "function_call_output"), | |
| ("c-notify", "function_call"), | |
| ("c-notify", "function_call_output"), | |
| ) | |
| ) | |
| def state_guardrail_counts(state: RunState[Any]) -> tuple[int, int]: | |
| return ( | |
| len(state._tool_input_guardrail_results), | |
| len(state._tool_output_guardrail_results), | |
| ) | |
| def result_guardrail_counts(result: Any) -> tuple[int, int]: | |
| return ( | |
| len(result.tool_input_guardrail_results), | |
| len(result.tool_output_guardrail_results), | |
| ) | |
| async def run_case(case: MatrixCase) -> CaseResult: | |
| counters = {"effect": 0, "input": 0, "output": 0, "start": 0, "end": 0} | |
| @tool_input_guardrail | |
| def charge_input(_data: Any) -> ToolGuardrailFunctionOutput: | |
| counters["input"] += 1 | |
| return ToolGuardrailFunctionOutput.allow(output_info="charge-input-ok") | |
| @tool_output_guardrail | |
| def charge_output(_data: Any) -> ToolGuardrailFunctionOutput: | |
| counters["output"] += 1 | |
| return ToolGuardrailFunctionOutput.allow(output_info="charge-output-ok") | |
| @function_tool( | |
| needs_approval=True, | |
| tool_input_guardrails=[charge_input], | |
| tool_output_guardrails=[charge_output], | |
| ) | |
| async def charge(x: int) -> str: | |
| counters["effect"] += 1 | |
| return f"charged:{x}" | |
| @function_tool(needs_approval=True) | |
| async def notify() -> str: | |
| raise AssertionError("the rejected tool must not execute") | |
| class Hooks(RunHooks[Any]): | |
| async def on_tool_start(self, context: Any, agent: Any, tool: Any) -> None: | |
| if tool.name == "charge": | |
| counters["start"] += 1 | |
| async def on_tool_end( | |
| self, | |
| context: Any, | |
| agent: Any, | |
| tool: Any, | |
| result: Any, | |
| ) -> None: | |
| if tool.name == "charge": | |
| counters["end"] += 1 | |
| model = ScriptedModel( | |
| steps=[ | |
| [ | |
| function_call("charge", {"x": 7}, call_id="c-charge"), | |
| function_call("notify", {}, call_id="c-notify"), | |
| ], | |
| [assistant_message("done")], | |
| ] | |
| ) | |
| agent = Agent(name="agent", model=model, tools=[charge, notify]) | |
| session = AmbiguousAckSession(f"session-{case.name}", case.outcome) | |
| run_config = RunConfig(tracing_disabled=True) | |
| hooks = Hooks() | |
| paused = await invoke( | |
| case.initial_mode, | |
| agent, | |
| "charge and notify", | |
| session=session, | |
| run_config=run_config, | |
| hooks=hooks, | |
| ) | |
| state = paused.to_state() | |
| charge_approval = next( | |
| item for item in state.get_interruptions() if call_id(item) == "c-charge" | |
| ) | |
| state.approve(charge_approval) | |
| session.fail_next_add = True | |
| try: | |
| await invoke( | |
| case.failing_mode, | |
| agent, | |
| state, | |
| session=session, | |
| run_config=run_config, | |
| hooks=hooks, | |
| ) | |
| except RuntimeError as error: | |
| assert str(error) == "injected ambiguous renewed-interruption write failure" | |
| else: | |
| raise AssertionError("the injected resumed Session write did not fail") | |
| serialized = state.to_json() | |
| assert serialized["current_step"]["type"] == "next_step_interruption" | |
| assert [call_id(item) for item in state.get_interruptions()] == ["c-notify"] | |
| assert serialized["tool_input_guardrail_results"] == [] | |
| assert serialized["tool_output_guardrail_results"] == [] | |
| assert session.failed_batch is not None | |
| assert history_counts(session.failed_batch) == (0, 1, 0, 0) | |
| session_before_retry = history_counts(session.items) | |
| retry_state = ( | |
| await RunState.from_json(agent, serialized) | |
| if case.transport == "json" | |
| else state | |
| ) | |
| pending = await invoke( | |
| case.retry_mode, | |
| agent, | |
| retry_state, | |
| session=session, | |
| run_config=run_config, | |
| hooks=hooks, | |
| ) | |
| pending_state = pending.to_state() | |
| assert [call_id(item) for item in pending_state.get_interruptions()] == ["c-notify"] | |
| assert len(model.calls) == 1 | |
| assert history_counts(session.items) == session_before_retry | |
| notify_approval = pending_state.get_interruptions()[0] | |
| pending_state.reject( | |
| notify_approval, rejection_message="notification intentionally rejected" | |
| ) | |
| final = await invoke( | |
| case.retry_mode, | |
| agent, | |
| pending_state, | |
| session=session, | |
| run_config=run_config, | |
| hooks=hooks, | |
| ) | |
| result = CaseResult( | |
| name=case.name, | |
| outcome=case.outcome, | |
| transport=case.transport, | |
| step_after_failure=serialized["current_step"]["type"], | |
| counters=( | |
| counters["effect"], | |
| counters["input"], | |
| counters["output"], | |
| counters["start"], | |
| counters["end"], | |
| ), | |
| model_calls_before_decision=1, | |
| model_calls_final=len(model.calls), | |
| state_guardrails=state_guardrail_counts(state), | |
| pending_guardrails=result_guardrail_counts(pending), | |
| pending_state_guardrails=state_guardrail_counts(pending_state), | |
| final_guardrails=result_guardrail_counts(final), | |
| session_before_retry=session_before_retry, | |
| session_final=history_counts(session.items), | |
| replay_final=history_counts(final.to_input_list()), | |
| ) | |
| charge_output_count = 0 if case.outcome == "absent" else 1 | |
| assert result.step_after_failure == "next_step_interruption" | |
| assert result.counters == (1, 1, 1, 1, 1) | |
| assert result.model_calls_before_decision == 1 | |
| assert result.model_calls_final == 2 | |
| assert result.state_guardrails == (0, 0) | |
| assert result.pending_guardrails == (0, 0) | |
| assert result.pending_state_guardrails == (0, 0) | |
| assert result.final_guardrails == (0, 0) | |
| assert result.session_before_retry == (1, charge_output_count, 1, 0) | |
| assert result.session_final == (1, charge_output_count, 1, 1) | |
| assert result.replay_final == (1, 1, 1, 1) | |
| assert final.final_output == "done" | |
| return result | |
| async def main() -> None: | |
| for case in CASES: | |
| print(await run_case(case)) | |
| if __name__ == "__main__": | |
| asyncio.run(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment