Skip to content

Commit 5d90c7e

Browse files
speedstorm1copybara-github
authored andcommitted
test: run live module integration tests against both backends
PiperOrigin-RevId: 974743067
1 parent 56d8dec commit 5d90c7e

3 files changed

Lines changed: 62 additions & 20 deletions

File tree

‎google/genai/tests/conftest.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,13 @@ def replays_prefix():
116116
return 'test'
117117

118118

119+
# Overridden by test packages whose Vertex model is not served at the configured
120+
# location. Returning None leaves GOOGLE_CLOUD_LOCATION as-is.
121+
@pytest.fixture
122+
def location_override():
123+
return None
124+
125+
119126
def _get_replay_id(use_vertex: bool, replays_prefix: str) -> str:
120127
test_name_ending = os.environ.get('PYTEST_CURRENT_TEST').split('::')[-1]
121128
test_name = (
@@ -127,7 +134,7 @@ def _get_replay_id(use_vertex: bool, replays_prefix: str) -> str:
127134

128135

129136
@pytest.fixture
130-
def client(use_vertex, replays_prefix, http_options, request):
137+
def client(use_vertex, replays_prefix, http_options, request, location_override):
131138
mode = request.config.getoption('--mode')
132139
if mode not in ['auto', 'record', 'replay', 'api', 'tap']:
133140
raise ValueError('Invalid mode: ' + mode)
@@ -171,8 +178,10 @@ def client(use_vertex, replays_prefix, http_options, request):
171178
# Get private arg.
172179
private = request.config.getoption('--private')
173180

174-
location_override = None
175-
if use_vertex and 'tunings' in replays_prefix:
181+
if not use_vertex:
182+
location_override = None
183+
elif location_override is None and 'tunings' in replays_prefix:
184+
# Tuning jobs are not supported on the global endpoint.
176185
if os.environ.get('GOOGLE_CLOUD_LOCATION') == 'global':
177186
location_override = 'us-central1'
178187

‎google/genai/tests/live_api/conftest.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,16 @@ def http_options():
2020
Normally injected by pytest_helper.setup(); live tests use the SDK defaults.
2121
"""
2222
return None
23+
24+
25+
@pytest.fixture
26+
def location_override():
27+
"""Pins the Vertex client to a region for the live model.
28+
29+
gemini-live-2.5-flash-native-audio is not served on the global endpoint: a
30+
setup there is rejected with 1008 "Publisher model ... was not found". It is
31+
available in us-central1, us-east5 and europe-west4. The Agent Platform
32+
wrapper sets GOOGLE_CLOUD_LOCATION=global for the shared suite, so the live
33+
tests override it here rather than changing the wrapper.
34+
"""
35+
return 'us-central1'

‎google/genai/tests/live_api/test_live_session.py‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,17 @@
4444

4545
pytest_plugins = ('pytest_asyncio',)
4646

47-
# The only live model family currently served. It is audio-native and rejects a
48-
# TEXT response modality outright ("The requested combination of response
49-
# modalities (TEXT) is not supported by the model"), so these tests request
50-
# AUDIO and turn on output transcription to get an assertable text signal.
51-
LIVE_MODEL = 'gemini-3.1-flash-live-preview'
47+
# Live models are backend-specific: gemini-3.1-flash-live-preview is served only
48+
# on the Gemini API, and gemini-live-2.5-flash-native-audio only on Vertex, and
49+
# there only regionally -- see the location_override fixture in this package's
50+
# conftest. Both are audio-native and reject a TEXT response modality outright
51+
# ("The requested combination of response modalities (TEXT) is not supported by
52+
# the model"), so these tests request AUDIO and turn on output transcription to
53+
# get an assertable text signal.
54+
LIVE_MODELS = {
55+
False: 'gemini-3.1-flash-live-preview',
56+
True: 'gemini-live-2.5-flash-native-audio',
57+
}
5258

5359
# A live turn is an open-ended stream with no built-in deadline. Without this
5460
# bound a wedged receive would hang the nightly rather than fail it.
@@ -121,12 +127,13 @@ async def _say(session, text: str) -> None:
121127
)
122128

123129

130+
@pytest.mark.parametrize('use_vertex', [False, True])
124131
@pytest.mark.asyncio
125-
async def test_text_input(client):
132+
async def test_text_input(client, use_vertex):
126133
"""A single text turn produces audio output and a matching transcription."""
127134
try:
128135
async with client.aio.live.connect(
129-
model=LIVE_MODEL, config=_base_config()
136+
model=LIVE_MODELS[use_vertex], config=_base_config()
130137
) as session:
131138
await _say(session, 'Say hello.')
132139
turn = await _receive_turn(session)
@@ -138,12 +145,13 @@ async def test_text_input(client):
138145
raise
139146

140147

148+
@pytest.mark.parametrize('use_vertex', [False, True])
141149
@pytest.mark.asyncio
142-
async def test_multi_turn(client):
150+
async def test_multi_turn(client, use_vertex):
143151
"""A second turn in the same session can see the first turn's context."""
144152
try:
145153
async with client.aio.live.connect(
146-
model=LIVE_MODEL, config=_base_config()
154+
model=LIVE_MODELS[use_vertex], config=_base_config()
147155
) as session:
148156
await _say(session, 'Remember the number 42. Just acknowledge it.')
149157
first = await _receive_turn(session)
@@ -162,8 +170,9 @@ async def test_multi_turn(client):
162170
raise
163171

164172

173+
@pytest.mark.parametrize('use_vertex', [False, True])
165174
@pytest.mark.asyncio
166-
async def test_function_calling(client):
175+
async def test_function_calling(client, use_vertex):
167176
"""The model requests a declared tool, and the session accepts its result."""
168177
turn_on_the_lights = types.FunctionDeclaration(
169178
name='turn_on_the_lights',
@@ -176,7 +185,7 @@ async def test_function_calling(client):
176185

177186
try:
178187
async with client.aio.live.connect(
179-
model=LIVE_MODEL, config=config
188+
model=LIVE_MODELS[use_vertex], config=config
180189
) as session:
181190
await _say(session, 'Please turn on the lights.')
182191
turn = await _receive_turn(session)
@@ -194,20 +203,31 @@ async def test_function_calling(client):
194203
]
195204
)
196205
follow_up = await _receive_turn(session)
197-
assert follow_up.transcript.strip(), (
198-
'expected the model to respond after the tool result'
199-
)
206+
if not use_vertex:
207+
# Vertex accepts the tool result and completes the turn, but emits an
208+
# empty transcription and no audio for it, so only the Gemini API can be
209+
# asserted on content here. Confirmed at the raw protocol level: the
210+
# follow-up carries outputTranscription with empty text, then
211+
# generationComplete and turnComplete.
212+
assert follow_up.transcript.strip(), (
213+
'expected the model to respond after the tool result'
214+
)
200215
except Exception as e: # pylint: disable=broad-except
201216
_skip_if_quota_exhausted(e)
202217
raise
203218

204219

220+
@pytest.mark.parametrize('use_vertex', [False])
205221
@pytest.mark.asyncio
206-
async def test_send_tool_response_without_id_raises(client):
207-
"""The Gemini API backend requires an id on every FunctionResponse."""
222+
async def test_send_tool_response_without_id_raises(client, use_vertex):
223+
"""The Gemini API backend requires an id on every FunctionResponse.
224+
225+
Gemini API only: live.py:409 puts this validation in the non-vertexai branch
226+
of send_tool_response, so Vertex accepts an id-less FunctionResponse.
227+
"""
208228
try:
209229
async with client.aio.live.connect(
210-
model=LIVE_MODEL, config=_base_config()
230+
model=LIVE_MODELS[use_vertex], config=_base_config()
211231
) as session:
212232
with pytest.raises(ValueError, match='must have an `id` field'):
213233
await session.send_tool_response(

0 commit comments

Comments
 (0)