Skip to content

Commit c0ac04e

Browse files
speedstorm1copybara-github
authored andcommitted
fix: Honor HttpRetryOptions for aiohttp connection errors in async requests
fixes #3160 PiperOrigin-RevId: 1003738102
1 parent 1a64450 commit c0ac04e

3 files changed

Lines changed: 247 additions & 3 deletions

File tree

‎google/genai/_api_client.py‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,17 @@
7070

7171

7272
has_aiohttp = False
73+
# aiohttp connection errors that HttpRetryOptions retries, like httpx.
74+
_AIOHTTP_TRANSIENT_EXC: tuple[type[Exception], ...] = ()
7375
try:
7476
import aiohttp
7577

7678
has_aiohttp = True
79+
_AIOHTTP_TRANSIENT_EXC = (
80+
aiohttp.ClientConnectorError,
81+
aiohttp.ClientOSError,
82+
aiohttp.ServerDisconnectedError,
83+
)
7784
except ImportError:
7885
pass
7986

@@ -632,7 +639,8 @@ def retry_args(options: Optional[HttpRetryOptions]) -> _common.StringDict:
632639
retriable_codes = options.http_status_codes or _RETRY_HTTP_STATUS_CODES
633640
retry = tenacity.retry_if_exception(
634641
lambda e: (isinstance(e, errors.APIError) and e.code in retriable_codes)
635-
or isinstance(e, _HTTPX_TRANSIENT_EXC),
642+
or isinstance(e, _HTTPX_TRANSIENT_EXC)
643+
or isinstance(e, _AIOHTTP_TRANSIENT_EXC),
636644
)
637645
wait = tenacity.wait_exponential_jitter(
638646
initial=_delay_or_default(options.initial_delay, _RETRY_INITIAL_DELAY),
@@ -1637,7 +1645,11 @@ def _request(
16371645
return self._retry(self._request_once, http_request, stream) # type: ignore[no-any-return]
16381646

16391647
async def _async_request_once(
1640-
self, http_request: HttpRequest, stream: bool = False
1648+
self,
1649+
http_request: HttpRequest,
1650+
stream: bool = False,
1651+
*,
1652+
implicit_retry: bool = True,
16411653
) -> HttpResponse:
16421654
data: Optional[bytes] = None
16431655

@@ -1685,6 +1697,8 @@ async def _async_request_once(
16851697
aiohttp.ServerDisconnectedError,
16861698
auth_exceptions.TransportError,
16871699
) as e:
1700+
if not implicit_retry or self._http_options.retry_options:
1701+
raise # HttpRetryOptions owns retries; tenacity decides.
16881702
await asyncio.sleep(1 + random.randint(0, 9))
16891703
logger.info('Retrying due to aiohttp error: %s' % e)
16901704
# Retrieve the SSL context from the session.
@@ -1763,6 +1777,8 @@ async def _async_request_once(
17631777
aiohttp.ServerDisconnectedError,
17641778
auth_exceptions.TransportError,
17651779
) as e:
1780+
if not implicit_retry or self._http_options.retry_options:
1781+
raise # HttpRetryOptions owns retries; tenacity decides.
17661782
await asyncio.sleep(1 + random.randint(0, 9))
17671783
logger.info('Retrying due to aiohttp error: %s' % e)
17681784
# Retrieve the SSL context from the session.
@@ -1819,7 +1835,12 @@ async def _async_request(
18191835
if parameter_model.retry_options:
18201836
retry_kwargs = retry_args(parameter_model.retry_options)
18211837
retry = tenacity.AsyncRetrying(**retry_kwargs)
1822-
return await retry(self._async_request_once, http_request, stream) # type: ignore[no-any-return]
1838+
return await retry( # type: ignore[no-any-return]
1839+
self._async_request_once,
1840+
http_request,
1841+
stream,
1842+
implicit_retry=False,
1843+
)
18231844
return await self._async_retry( # type: ignore[no-any-return]
18241845
self._async_request_once, http_request, stream
18251846
)
Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
16+
"""Tests that HttpRetryOptions governs retries of aiohttp connection errors."""
17+
18+
import asyncio
19+
from unittest import mock
20+
21+
from google.oauth2 import credentials
22+
import pytest
23+
24+
from ... import _api_client as api_client
25+
from ... import types
26+
27+
aiohttp = pytest.importorskip('aiohttp')
28+
29+
30+
def _patch_auth_default():
31+
return mock.patch(
32+
'google.auth.default',
33+
return_value=(credentials.Credentials('magic_token'), 'test_project'),
34+
autospec=True,
35+
)
36+
37+
38+
async def _aiohttp_async_response(status: int):
39+
"""Has to return a coroutine hence async."""
40+
response = mock.Mock(spec=aiohttp.ClientResponse)
41+
response.status = status
42+
response.headers = {'status-code': str(status)}
43+
response.text.return_value = 'test'
44+
return response
45+
46+
47+
def _client(retry_options=None):
48+
return api_client.BaseApiClient(
49+
vertexai=True,
50+
project='test_project',
51+
location='global',
52+
http_options=types.HttpOptions(retry_options=retry_options),
53+
)
54+
55+
56+
def test_retry_args_retries_aiohttp_transport_errors():
57+
"""HttpRetryOptions retries aiohttp connection errors, like httpx."""
58+
59+
retry = api_client.retry_args(types.HttpRetryOptions())['retry']
60+
61+
assert retry.predicate(aiohttp.ServerDisconnectedError())
62+
assert retry.predicate(aiohttp.ClientOSError())
63+
assert retry.predicate(
64+
aiohttp.ClientConnectorError(
65+
connection_key=aiohttp.client_reqrep.ConnectionKey(
66+
'localhost', 80, False, True, None, None, None
67+
),
68+
os_error=OSError(),
69+
)
70+
)
71+
72+
assert not retry.predicate(aiohttp.ClientPayloadError())
73+
assert not retry.predicate(ValueError('not a transport error'))
74+
75+
76+
@mock.patch.object(aiohttp.ClientSession, 'request', autospec=True)
77+
def test_aiohttp_single_attempt_server_disconnected_error_executes_once(
78+
mock_request,
79+
):
80+
"""attempts=1 means no retries, so a dropped connection is not resent."""
81+
82+
api_client.has_aiohttp = True
83+
84+
async def run():
85+
mock_request.side_effect = aiohttp.ServerDisconnectedError()
86+
client = _client(types.HttpRetryOptions(attempts=1))
87+
88+
with _patch_auth_default():
89+
with pytest.raises(aiohttp.ServerDisconnectedError):
90+
await client.async_request(
91+
http_method='POST', path='path', request_dict={}
92+
)
93+
assert mock_request.call_count == 1
94+
95+
asyncio.run(run())
96+
97+
98+
@mock.patch.object(aiohttp.ClientSession, 'request', autospec=True)
99+
def test_aiohttp_single_attempt_server_disconnected_error_executes_once_at_request_level(
100+
mock_request,
101+
):
102+
"""Request-level attempts=1 also disables the implicit aiohttp reconnect."""
103+
104+
api_client.has_aiohttp = True
105+
106+
async def run():
107+
mock_request.side_effect = aiohttp.ServerDisconnectedError()
108+
client = _client()
109+
110+
with _patch_auth_default():
111+
with pytest.raises(aiohttp.ServerDisconnectedError):
112+
await client.async_request(
113+
http_method='POST',
114+
path='path',
115+
request_dict={},
116+
http_options={'retry_options': {'attempts': 1}},
117+
)
118+
assert mock_request.call_count == 1
119+
120+
asyncio.run(run())
121+
122+
123+
@mock.patch.object(aiohttp.ClientSession, 'request', autospec=True)
124+
def test_aiohttp_single_attempt_streamed_server_disconnected_error_executes_once(
125+
mock_request,
126+
):
127+
"""The streaming branch honors attempts=1 the same way."""
128+
129+
api_client.has_aiohttp = True
130+
131+
async def run():
132+
mock_request.side_effect = aiohttp.ServerDisconnectedError()
133+
client = _client(types.HttpRetryOptions(attempts=1))
134+
135+
with _patch_auth_default():
136+
with pytest.raises(aiohttp.ServerDisconnectedError):
137+
await client.async_request_streamed(
138+
http_method='POST', path='path', request_dict={}
139+
)
140+
assert mock_request.call_count == 1
141+
142+
asyncio.run(run())
143+
144+
145+
@mock.patch.object(aiohttp.ClientSession, 'request', autospec=True)
146+
def test_aiohttp_retries_server_disconnected_error_up_to_attempts(
147+
mock_request,
148+
):
149+
"""With retry options, tenacity retries aiohttp errors up to `attempts`."""
150+
151+
api_client.has_aiohttp = True
152+
153+
async def run():
154+
mock_request.side_effect = aiohttp.ServerDisconnectedError()
155+
client = _client(
156+
types.HttpRetryOptions(
157+
attempts=3, initial_delay=0, max_delay=0, jitter=0
158+
)
159+
)
160+
161+
with _patch_auth_default():
162+
with pytest.raises(aiohttp.ServerDisconnectedError):
163+
await client.async_request(
164+
http_method='POST', path='path', request_dict={}
165+
)
166+
assert mock_request.call_count == 3
167+
168+
asyncio.run(run())
169+
170+
171+
@mock.patch.object(aiohttp.ClientSession, 'request', autospec=True)
172+
def test_aiohttp_retries_server_disconnected_error_retries_successfully(
173+
mock_request,
174+
):
175+
"""A dropped connection is still retried when attempts > 1."""
176+
177+
api_client.has_aiohttp = True
178+
179+
async def run():
180+
mock_request.side_effect = (
181+
aiohttp.ServerDisconnectedError(),
182+
_aiohttp_async_response(200),
183+
)
184+
client = _client(
185+
types.HttpRetryOptions(
186+
attempts=2, initial_delay=0, max_delay=0, jitter=0
187+
)
188+
)
189+
190+
with _patch_auth_default():
191+
response = await client.async_request(
192+
http_method='GET', path='path', request_dict={}
193+
)
194+
assert mock_request.call_count == 2
195+
assert response.headers['status-code'] == '200'
196+
197+
asyncio.run(run())
198+
199+
200+
@mock.patch.object(aiohttp.ClientSession, 'request', autospec=True)
201+
def test_aiohttp_without_retry_options_reconnects_once(mock_request):
202+
"""Without retry options, the legacy single reconnect is kept."""
203+
204+
api_client.has_aiohttp = True
205+
206+
async def run():
207+
mock_request.side_effect = (
208+
aiohttp.ServerDisconnectedError(),
209+
_aiohttp_async_response(200),
210+
)
211+
client = _client()
212+
213+
with _patch_auth_default(), mock.patch.object(
214+
asyncio, 'sleep', new=mock.AsyncMock()
215+
):
216+
response = await client.async_request(
217+
http_method='GET', path='path', request_dict={}
218+
)
219+
assert mock_request.call_count == 2
220+
assert response.headers['status-code'] == '200'
221+
222+
asyncio.run(run())

‎google/genai/tests/client/test_retries.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,6 +1342,7 @@ async def run():
13421342
http_method='GET', path='path', request_dict={}
13431343
)
13441344
mock_request.assert_called()
1345+
assert mock_request.call_count == 2
13451346
assert response.headers['status-code'] == '200'
13461347

13471348
asyncio.run(run())

0 commit comments

Comments
 (0)