Skip to content

Commit 06298d5

Browse files
authored
fix(tracing): retry rate-limited trace exports (Fixes #5023) (#5052)
1 parent 51bcea7 commit 06298d5

2 files changed

Lines changed: 204 additions & 23 deletions

File tree

‎src/agents/tracing/processors.py‎

Lines changed: 72 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ class BackendSpanExporter(TracingExporter):
5252
}
5353
)
5454
_OPENAI_TRACING_USAGE_SPAN_TYPES = frozenset({"generation"})
55+
# 4xx statuses that are transient, mirroring the OpenAI client's own retry policy:
56+
# request timeout, conflict, and rate limiting.
57+
_RETRYABLE_CLIENT_STATUS_CODES = frozenset({408, 409, 429})
5558
_UNSERIALIZABLE = object()
5659

5760
def __init__(
@@ -167,7 +170,9 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float
167170
# Exponential backoff loop
168171
attempt = 0
169172
delay = self.base_delay
173+
retry_after: float | None = None
170174
while True:
175+
retry_after = None
171176
request_timeout = self._timeout_for_deadline(deadline)
172177
if deadline is not None and request_timeout is None:
173178
logger.warning(
@@ -191,8 +196,18 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float
191196
logger.debug("Exported %s items", len(grouped))
192197
break
193198

194-
# If the response is a client error (4xx), we won't retry
195-
if 400 <= response.status_code < 500:
199+
allows_retry = self._server_allows_retry(response)
200+
# A rate limit, request timeout, or conflict is transient: retry it
201+
# like a server error, waiting at least the advertised Retry-After,
202+
# unless the server says outright not to retry.
203+
if response.status_code in self._RETRYABLE_CLIENT_STATUS_CODES and allows_retry:
204+
retry_after = self._retry_after_seconds(response)
205+
logger.warning(
206+
"[non-fatal] Tracing: client error %s, retrying.",
207+
response.status_code,
208+
)
209+
# Any other client error (4xx) won't be retried
210+
elif 400 <= response.status_code < 500:
196211
if _debug.DONT_LOG_MODEL_DATA or _debug.DONT_LOG_TOOL_DATA:
197212
logger.error(
198213
"[non-fatal] Tracing client error %s. Response data is redacted.",
@@ -205,11 +220,18 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float
205220
response.text,
206221
)
207222
break
208-
209-
# For 5xx or other unexpected codes, treat it as transient and retry
210-
logger.warning(
211-
"[non-fatal] Tracing: server error %s, retrying.", response.status_code
212-
)
223+
elif not allows_retry:
224+
logger.error(
225+
"[non-fatal] Tracing: server forbade retry for %s.",
226+
response.status_code,
227+
)
228+
break
229+
else:
230+
# For 5xx or other unexpected codes, treat it as transient and retry
231+
logger.warning(
232+
"[non-fatal] Tracing: server error %s, retrying.",
233+
response.status_code,
234+
)
213235
except httpx2.RequestError as exc:
214236
# Network or other I/O error, we'll retry
215237
log_model_and_tool_action_warning(
@@ -225,10 +247,40 @@ def _export_with_deadline(self, items: list[Trace | Span[Any]], deadline: float
225247

226248
# Exponential backoff + jitter
227249
sleep_time = delay + random.uniform(0, 0.1 * delay) # 10% jitter
250+
if retry_after is not None:
251+
# Honour the server's wait, bounded by max_delay so a large
252+
# Retry-After cannot stall the export thread.
253+
sleep_time = min(max(sleep_time, retry_after), self.max_delay)
228254
if not self._sleep_before_retry(sleep_time, deadline):
229255
break
230256
delay = min(delay * 2, self.max_delay)
231257

258+
def _server_allows_retry(self, response: httpx2.Response) -> bool:
259+
# Mirror the OpenAI client: an explicit `x-should-retry: false` wins over the
260+
# status code classification.
261+
headers = getattr(response, "headers", None)
262+
if headers is None:
263+
return True
264+
should_retry = headers.get("x-should-retry")
265+
if not isinstance(should_retry, str):
266+
return True
267+
return should_retry.strip().lower() != "false"
268+
269+
def _retry_after_seconds(self, response: httpx2.Response) -> float | None:
270+
# Imported lazily: the models package pulls in the OpenAI client, which must not
271+
# become an import-time dependency of tracing.
272+
from ..models._retry_runtime import parse_retry_after_ms, parse_retry_after_value
273+
274+
headers = getattr(response, "headers", None)
275+
if headers is None:
276+
return None
277+
retry_after = parse_retry_after_ms(headers.get("retry-after-ms"))
278+
if retry_after is None:
279+
retry_after = parse_retry_after_value(headers.get("retry-after"))
280+
if retry_after is None or not math.isfinite(retry_after):
281+
return None
282+
return retry_after
283+
232284
def _timeout_for_deadline(self, deadline: float | None) -> httpx2.Timeout | None:
233285
if deadline is None:
234286
return None
@@ -241,27 +293,25 @@ def _timeout_for_deadline(self, deadline: float | None) -> httpx2.Timeout | None
241293
return httpx2.Timeout(remaining, connect=connect_timeout)
242294

243295
def _sleep_before_retry(self, sleep_time: float, deadline: float | None) -> bool:
244-
if deadline is None:
245-
if self._shutdown_event.wait(sleep_time):
246-
logger.warning(
247-
"[non-fatal] Tracing: shutdown requested during retry backoff, giving up."
248-
)
249-
return False
250-
return not self._shutdown_event.is_set()
251-
252-
remaining = deadline - time.monotonic()
253-
if remaining <= 0:
296+
remaining = None if deadline is None else deadline - time.monotonic()
297+
if remaining is not None and remaining <= 0:
254298
logger.warning("[non-fatal] Tracing: export deadline reached before retry, giving up.")
255299
return False
256300

257-
if sleep_time >= remaining:
258-
time.sleep(remaining)
301+
wait_for = sleep_time if remaining is None else min(sleep_time, remaining)
302+
if deadline is None and self._shutdown_event.wait(wait_for):
259303
logger.warning(
260-
"[non-fatal] Tracing: export deadline reached during retry backoff, giving up."
304+
"[non-fatal] Tracing: shutdown requested during retry backoff, giving up."
261305
)
262306
return False
263-
264-
time.sleep(sleep_time)
307+
if deadline is not None:
308+
# The final drain runs after shutdown is requested; its deadline bounds retries.
309+
time.sleep(wait_for)
310+
if remaining is not None and (sleep_time >= remaining or time.monotonic() >= deadline):
311+
logger.warning(
312+
"[non-fatal] Tracing: export deadline reached during retry backoff, giving up."
313+
)
314+
return False
265315
return True
266316

267317
def _should_sanitize_for_openai_tracing_api(self) -> bool:

‎tests/test_trace_processor.py‎

Lines changed: 132 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,22 @@ def _export_with_deadline(
284284
assert seen_deadlines[0] is not None
285285

286286

287+
@patch("httpx2.Client")
288+
def test_batch_trace_processor_timed_shutdown_retries_final_drain(mock_client) -> None:
289+
transient = MagicMock(status_code=503, headers={})
290+
success = MagicMock(status_code=200, headers={})
291+
mock_client.return_value.post.side_effect = [transient, success]
292+
293+
exporter = BackendSpanExporter(api_key="test_key", max_retries=2, base_delay=0.001)
294+
processor = BatchTraceProcessor(exporter=exporter)
295+
processor._queue.put_nowait(get_span(processor))
296+
297+
processor.shutdown(timeout=1.0)
298+
299+
assert mock_client.return_value.post.call_count == 2
300+
exporter.close()
301+
302+
287303
def test_batch_trace_processor_survives_exporter_exception():
288304
"""A failing exporter must not kill the background worker thread.
289305
@@ -564,6 +580,101 @@ def text(self) -> str:
564580
exporter.close()
565581

566582

583+
@patch("httpx2.Client")
584+
@pytest.mark.parametrize("status_code", [408, 409, 429])
585+
def test_backend_span_exporter_retries_transient_client_errors(mock_client, status_code: int):
586+
"""A rate limit, timeout, or conflict is transient, as the OpenAI client treats it;
587+
dropping the batch on the first response loses every trace in it."""
588+
mock_response = MagicMock()
589+
mock_response.status_code = status_code
590+
mock_response.headers = {}
591+
mock_client.return_value.post.return_value = mock_response
592+
593+
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
594+
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
595+
exporter.export([get_span(mock_processor())])
596+
597+
assert mock_client.return_value.post.call_count == 3
598+
assert wait_for_retry.call_count == 2
599+
exporter.close()
600+
601+
602+
@patch("httpx2.Client")
603+
def test_backend_span_exporter_rate_limit_obeys_x_should_retry_false(mock_client, caplog):
604+
"""An explicit `x-should-retry: false` wins over the status classification, as in the
605+
OpenAI client."""
606+
mock_response = MagicMock()
607+
mock_response.status_code = 429
608+
mock_response.headers = {"x-should-retry": "false", "retry-after": "1"}
609+
mock_response.text = "rate limited"
610+
mock_client.return_value.post.return_value = mock_response
611+
612+
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
613+
with (
614+
patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry,
615+
caplog.at_level(logging.ERROR, logger="openai.agents"),
616+
):
617+
exporter.export([get_span(mock_processor())])
618+
619+
mock_client.return_value.post.assert_called_once()
620+
wait_for_retry.assert_not_called()
621+
assert "Tracing client error 429" in caplog.text
622+
exporter.close()
623+
624+
625+
@patch("httpx2.Client")
626+
def test_backend_span_exporter_rate_limit_recovers_after_retry(mock_client):
627+
ok_response = MagicMock()
628+
ok_response.status_code = 200
629+
limited_response = MagicMock()
630+
limited_response.status_code = 429
631+
limited_response.headers = {}
632+
mock_client.return_value.post.side_effect = [limited_response, ok_response]
633+
634+
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
635+
with patch.object(exporter._shutdown_event, "wait", return_value=False):
636+
exporter.export([get_span(mock_processor())])
637+
638+
assert mock_client.return_value.post.call_count == 2
639+
exporter.close()
640+
641+
642+
@patch("httpx2.Client")
643+
@pytest.mark.parametrize(
644+
("headers", "expected_wait"),
645+
[
646+
({"retry-after": "3"}, 3.0),
647+
({"retry-after-ms": "1500"}, 1.5),
648+
# Larger than max_delay: bounded so the export thread cannot stall.
649+
({"retry-after": "600"}, 5.0),
650+
# Unparsable or negative values fall back to exponential backoff.
651+
({"retry-after": "soon"}, None),
652+
({"retry-after": "-1"}, None),
653+
],
654+
)
655+
def test_backend_span_exporter_rate_limit_honours_retry_after(
656+
mock_client, headers: dict[str, str], expected_wait: float | None
657+
):
658+
limited_response = MagicMock()
659+
limited_response.status_code = 429
660+
limited_response.headers = headers
661+
ok_response = MagicMock()
662+
ok_response.status_code = 200
663+
mock_client.return_value.post.side_effect = [limited_response, ok_response]
664+
665+
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=5.0)
666+
with patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry:
667+
exporter.export([get_span(mock_processor())])
668+
669+
assert mock_client.return_value.post.call_count == 2
670+
waited = wait_for_retry.call_args.args[0]
671+
if expected_wait is None:
672+
assert 0.1 <= waited <= 0.11
673+
else:
674+
assert waited == pytest.approx(expected_wait)
675+
exporter.close()
676+
677+
567678
@patch("httpx2.Client")
568679
def test_backend_span_exporter_5xx_retry(mock_client):
569680
mock_response = MagicMock()
@@ -583,14 +694,34 @@ def test_backend_span_exporter_5xx_retry(mock_client):
583694
exporter.close()
584695

585696

697+
@patch("httpx2.Client")
698+
def test_backend_span_exporter_5xx_obeys_x_should_retry_false(mock_client, caplog):
699+
mock_response = MagicMock()
700+
mock_response.status_code = 503
701+
mock_response.headers = {"x-should-retry": "false"}
702+
mock_client.return_value.post.return_value = mock_response
703+
704+
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=0.1, max_delay=0.2)
705+
with (
706+
patch.object(exporter._shutdown_event, "wait", return_value=False) as wait_for_retry,
707+
caplog.at_level(logging.ERROR, logger="openai.agents"),
708+
):
709+
exporter.export([get_span(mock_processor())])
710+
711+
mock_client.return_value.post.assert_called_once()
712+
wait_for_retry.assert_not_called()
713+
assert "server forbade retry for 503" in caplog.text
714+
exporter.close()
715+
716+
586717
@patch("httpx2.Client")
587718
def test_backend_span_exporter_deadline_stops_during_5xx_retry_backoff(mock_client):
588719
mock_response = MagicMock()
589720
mock_response.status_code = 504
590721
mock_client.return_value.post.return_value = mock_response
591722

592723
exporter = BackendSpanExporter(api_key="test_key", max_retries=3, base_delay=1.0)
593-
with patch("time.sleep") as sleep_for_retry:
724+
with patch("agents.tracing.processors.time.sleep") as sleep_for_retry:
594725
exporter._export_with_deadline(
595726
[get_span(mock_processor())], deadline=time.monotonic() + 0.01
596727
)

0 commit comments

Comments
 (0)