Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Select an option

  • Save FU-max-boop/1eadecf110e7505ddaf9c67eb74f6053 to your computer and use it in GitHub Desktop.

Select an option

Save FU-max-boop/1eadecf110e7505ddaf9c67eb74f6053 to your computer and use it in GitHub Desktop.
Provider-neutral NextStepInterruption Session-write and guardrail-publication reproducer for openai-agents-python#4646
"""Provider-neutral observability extension for openai-agents-python #4615.
Run this against a pinned SDK source tree. It demonstrates that a Session
ambiguous-ack failure after an approved tool loses completed tool-guardrail
results from live/JSON state and every later public result, even though hooks,
guardrails, and the tool side effect each executed exactly once.
"""
from __future__ import annotations
import asyncio
from copy import deepcopy
from dataclasses import dataclass
from typing import Any, Literal
from agents import (
Agent,
RunConfig,
Runner,
RunState,
ToolGuardrailFunctionOutput,
function_tool,
tool_input_guardrail,
tool_output_guardrail,
)
from agents.lifecycle import RunHooks
from agents.testing import ScriptedModel, assistant_message, function_call
Mode = Literal["run", "streamed"]
Outcome = Literal["absent", "committed"]
Transport = Literal["live", "json"]
class AmbiguousAckSession:
"""In-memory Session that either rejects or commits before raising once."""
def __init__(self, session_id: str, outcome: Outcome) -> None:
self.session_id = session_id
self.outcome = outcome
self.items: list[dict[str, Any]] = []
self.fail_next_add = False
self.failed_batch: list[dict[str, Any]] | None = None
async def get_items(self, limit: int | None = None) -> list[dict[str, Any]]:
items = deepcopy(self.items)
return items[-limit:] if limit is not None else items
async def add_items(self, items: list[dict[str, Any]]) -> None:
owned = deepcopy(items)
if self.fail_next_add:
self.fail_next_add = False
self.failed_batch = owned
if self.outcome == "committed":
self.items.extend(deepcopy(owned))
raise RuntimeError("injected ambiguous renewed-interruption write failure")
self.items.extend(owned)
async def pop_item(self) -> dict[str, Any] | None:
return self.items.pop() if self.items else None
async def clear_session(self) -> None:
self.items.clear()
@dataclass(frozen=True)
class MatrixCase:
name: str
initial_mode: Mode
failing_mode: Mode
retry_mode: Mode
transport: Transport
outcome: Outcome
@dataclass(frozen=True)
class CaseResult:
name: str
outcome: Outcome
transport: Transport
step_after_failure: str
counters: tuple[int, int, int, int, int]
model_calls_before_decision: int
model_calls_final: int
state_guardrails: tuple[int, int]
pending_guardrails: tuple[int, int]
pending_state_guardrails: tuple[int, int]
final_guardrails: tuple[int, int]
session_before_retry: tuple[int, int, int, int]
session_final: tuple[int, int, int, int]
replay_final: tuple[int, int, int, int]
CASES = (
MatrixCase("RR-live-absent", "run", "run", "run", "live", "absent"),
MatrixCase("RS-json-commit", "run", "run", "streamed", "json", "committed"),
MatrixCase("SR-live-commit", "streamed", "streamed", "run", "live", "committed"),
MatrixCase("SS-json-absent", "streamed", "streamed", "streamed", "json", "absent"),
)
async def invoke(
mode: Mode,
agent: Agent[Any],
input: str | RunState[Any],
*,
session: AmbiguousAckSession,
run_config: RunConfig,
hooks: RunHooks[Any],
) -> Any:
if mode == "run":
return await Runner.run(
agent,
input,
session=session,
run_config=run_config,
hooks=hooks,
)
result = Runner.run_streamed(
agent,
input,
session=session,
run_config=run_config,
hooks=hooks,
)
async for _ in result.stream_events():
pass
return result
def item_value(item: Any, key: str) -> Any:
return item.get(key) if isinstance(item, dict) else getattr(item, key, None)
def call_id(item: Any) -> str:
value = item_value(item, "call_id")
assert isinstance(value, str)
return value
def history_counts(items: list[Any]) -> tuple[int, int, int, int]:
return tuple(
sum(
item_value(item, "type") == item_type
and item_value(item, "call_id") == expected_call_id
for item in items
)
for expected_call_id, item_type in (
("c-charge", "function_call"),
("c-charge", "function_call_output"),
("c-notify", "function_call"),
("c-notify", "function_call_output"),
)
)
def state_guardrail_counts(state: RunState[Any]) -> tuple[int, int]:
return (
len(state._tool_input_guardrail_results),
len(state._tool_output_guardrail_results),
)
def result_guardrail_counts(result: Any) -> tuple[int, int]:
return (
len(result.tool_input_guardrail_results),
len(result.tool_output_guardrail_results),
)
async def run_case(case: MatrixCase) -> CaseResult:
counters = {"effect": 0, "input": 0, "output": 0, "start": 0, "end": 0}
@tool_input_guardrail
def charge_input(_data: Any) -> ToolGuardrailFunctionOutput:
counters["input"] += 1
return ToolGuardrailFunctionOutput.allow(output_info="charge-input-ok")
@tool_output_guardrail
def charge_output(_data: Any) -> ToolGuardrailFunctionOutput:
counters["output"] += 1
return ToolGuardrailFunctionOutput.allow(output_info="charge-output-ok")
@function_tool(
needs_approval=True,
tool_input_guardrails=[charge_input],
tool_output_guardrails=[charge_output],
)
async def charge(x: int) -> str:
counters["effect"] += 1
return f"charged:{x}"
@function_tool(needs_approval=True)
async def notify() -> str:
raise AssertionError("the rejected tool must not execute")
class Hooks(RunHooks[Any]):
async def on_tool_start(self, context: Any, agent: Any, tool: Any) -> None:
if tool.name == "charge":
counters["start"] += 1
async def on_tool_end(
self,
context: Any,
agent: Any,
tool: Any,
result: Any,
) -> None:
if tool.name == "charge":
counters["end"] += 1
model = ScriptedModel(
steps=[
[
function_call("charge", {"x": 7}, call_id="c-charge"),
function_call("notify", {}, call_id="c-notify"),
],
[assistant_message("done")],
]
)
agent = Agent(name="agent", model=model, tools=[charge, notify])
session = AmbiguousAckSession(f"session-{case.name}", case.outcome)
run_config = RunConfig(tracing_disabled=True)
hooks = Hooks()
paused = await invoke(
case.initial_mode,
agent,
"charge and notify",
session=session,
run_config=run_config,
hooks=hooks,
)
state = paused.to_state()
charge_approval = next(
item for item in state.get_interruptions() if call_id(item) == "c-charge"
)
state.approve(charge_approval)
session.fail_next_add = True
try:
await invoke(
case.failing_mode,
agent,
state,
session=session,
run_config=run_config,
hooks=hooks,
)
except RuntimeError as error:
assert str(error) == "injected ambiguous renewed-interruption write failure"
else:
raise AssertionError("the injected resumed Session write did not fail")
serialized = state.to_json()
assert serialized["current_step"]["type"] == "next_step_interruption"
assert [call_id(item) for item in state.get_interruptions()] == ["c-notify"]
assert serialized["tool_input_guardrail_results"] == []
assert serialized["tool_output_guardrail_results"] == []
assert session.failed_batch is not None
assert history_counts(session.failed_batch) == (0, 1, 0, 0)
session_before_retry = history_counts(session.items)
retry_state = (
await RunState.from_json(agent, serialized)
if case.transport == "json"
else state
)
pending = await invoke(
case.retry_mode,
agent,
retry_state,
session=session,
run_config=run_config,
hooks=hooks,
)
pending_state = pending.to_state()
assert [call_id(item) for item in pending_state.get_interruptions()] == ["c-notify"]
assert len(model.calls) == 1
assert history_counts(session.items) == session_before_retry
notify_approval = pending_state.get_interruptions()[0]
pending_state.reject(
notify_approval, rejection_message="notification intentionally rejected"
)
final = await invoke(
case.retry_mode,
agent,
pending_state,
session=session,
run_config=run_config,
hooks=hooks,
)
result = CaseResult(
name=case.name,
outcome=case.outcome,
transport=case.transport,
step_after_failure=serialized["current_step"]["type"],
counters=(
counters["effect"],
counters["input"],
counters["output"],
counters["start"],
counters["end"],
),
model_calls_before_decision=1,
model_calls_final=len(model.calls),
state_guardrails=state_guardrail_counts(state),
pending_guardrails=result_guardrail_counts(pending),
pending_state_guardrails=state_guardrail_counts(pending_state),
final_guardrails=result_guardrail_counts(final),
session_before_retry=session_before_retry,
session_final=history_counts(session.items),
replay_final=history_counts(final.to_input_list()),
)
charge_output_count = 0 if case.outcome == "absent" else 1
assert result.step_after_failure == "next_step_interruption"
assert result.counters == (1, 1, 1, 1, 1)
assert result.model_calls_before_decision == 1
assert result.model_calls_final == 2
assert result.state_guardrails == (0, 0)
assert result.pending_guardrails == (0, 0)
assert result.pending_state_guardrails == (0, 0)
assert result.final_guardrails == (0, 0)
assert result.session_before_retry == (1, charge_output_count, 1, 0)
assert result.session_final == (1, charge_output_count, 1, 1)
assert result.replay_final == (1, 1, 1, 1)
assert final.final_output == "done"
return result
async def main() -> None:
for case in CASES:
print(await run_case(case))
if __name__ == "__main__":
asyncio.run(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment