|
56 | 56 | from agents.retry import ModelRetryAdviceRequest |
57 | 57 | from agents.usage import Usage |
58 | 58 | from tests.model_test_helpers import get_response_obj |
| 59 | +from tests.test_responses import get_text_message |
59 | 60 | from tests.testing_processor import fetch_ordered_spans |
60 | 61 |
|
61 | 62 |
|
| 63 | +@pytest.mark.allow_call_model_methods |
| 64 | +@pytest.mark.asyncio |
| 65 | +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) |
| 66 | +@pytest.mark.parametrize( |
| 67 | + ("counts", "expected"), |
| 68 | + [ |
| 69 | + ((None, 5, 12), (0, 5, 12)), |
| 70 | + ((7, None, 12), (7, 0, 12)), |
| 71 | + ((7, 5, None), (7, 5, 0)), |
| 72 | + ((None, None, None), (0, 0, 0)), |
| 73 | + ((0, 0, 0), (0, 0, 0)), |
| 74 | + ((7, 5, 12), (7, 5, 12)), |
| 75 | + ], |
| 76 | + ids=["null-input", "null-output", "null-total", "all-null", "zero", "valid"], |
| 77 | +) |
| 78 | +async def test_runner_normalizes_nullable_responses_usage( |
| 79 | + stream: bool, |
| 80 | + counts: tuple[int | None, int | None, int | None], |
| 81 | + expected: tuple[int, int, int], |
| 82 | +) -> None: |
| 83 | + usage = { |
| 84 | + "input_tokens": counts[0], |
| 85 | + "output_tokens": counts[1], |
| 86 | + "total_tokens": counts[2], |
| 87 | + "input_tokens_details": {"cached_tokens": 3}, |
| 88 | + "output_tokens_details": {"reasoning_tokens": 2}, |
| 89 | + } |
| 90 | + |
| 91 | + # Exercise provider parsing and usage conversion instead of a normalized scripted model. |
| 92 | + async def handler(request: httpx2.Request) -> httpx2.Response: |
| 93 | + payload = get_response_obj([get_text_message("Hello")]).model_dump() |
| 94 | + payload["usage"] = usage |
| 95 | + if stream: |
| 96 | + event = {"type": "response.completed", "response": payload, "sequence_number": 0} |
| 97 | + return httpx2.Response( |
| 98 | + 200, |
| 99 | + content=f"event: response.completed\ndata: {json.dumps(event)}\n\n", |
| 100 | + headers={"content-type": "text/event-stream"}, |
| 101 | + ) |
| 102 | + return httpx2.Response(200, json=payload) |
| 103 | + |
| 104 | + async with AsyncOpenAI( |
| 105 | + api_key="test-key", |
| 106 | + base_url="https://provider.example/v1", |
| 107 | + http_client=httpx2.AsyncClient(transport=httpx2.MockTransport(handler)), |
| 108 | + ) as client: |
| 109 | + agent = Agent( |
| 110 | + name="test", |
| 111 | + model=OpenAIResponsesModel(model="fake", openai_client=client), |
| 112 | + model_settings=ModelSettings(preserve_raw_usage=True), |
| 113 | + ) |
| 114 | + if stream: |
| 115 | + result = Runner.run_streamed(agent, "hi") |
| 116 | + async for _ in result.stream_events(): |
| 117 | + pass |
| 118 | + else: |
| 119 | + result = await Runner.run(agent, "hi") |
| 120 | + |
| 121 | + assert result.final_output == "Hello" |
| 122 | + normalized = result.context_wrapper.usage |
| 123 | + assert normalized.requests == 1 |
| 124 | + assert (normalized.input_tokens, normalized.output_tokens, normalized.total_tokens) == expected |
| 125 | + assert normalized.input_tokens_details.cached_tokens == 3 |
| 126 | + assert normalized.output_tokens_details.reasoning_tokens == 2 |
| 127 | + assert result.raw_responses[0].raw_usage == usage |
| 128 | + if expected == (0, 0, 0): |
| 129 | + assert normalized.request_usage_entries == [] |
| 130 | + else: |
| 131 | + assert len(normalized.request_usage_entries) == 1 |
| 132 | + entry = normalized.request_usage_entries[0] |
| 133 | + assert (entry.input_tokens, entry.output_tokens, entry.total_tokens) == expected |
| 134 | + assert entry.input_tokens_details.cached_tokens == 3 |
| 135 | + assert entry.output_tokens_details.reasoning_tokens == 2 |
| 136 | + |
| 137 | + |
62 | 138 | async def _run_responses_model_with_custom_base_url( |
63 | 139 | model_settings: ModelSettings | dict[str, Any] | None = None, |
64 | 140 | ) -> dict[str, Any]: |
|
0 commit comments