Skip to content

Commit 738e2b1

Browse files
jsondaicopybara-github
authored andcommitted
fix: GenAI Client(evals) - Fix run_inference for local ADK agents and remote runtime N+1
PiperOrigin-RevId: 996157640
1 parent b28737e commit 738e2b1

4 files changed

Lines changed: 24 additions & 17 deletions

File tree

‎agentplatform/_genai/_evals_common.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3510,7 +3510,9 @@ def _create_runtime_session(
35103510
raise RuntimeError(
35113511
"Failed to create session: runtime.api_client is None."
35123512
) from exc
3513-
operation = runtime.api_client.sessions.create(
3513+
from . import sessions
3514+
3515+
operation = sessions.Sessions(runtime.api_client._api_client).create(
35143516
name=runtime.api_resource.name,
35153517
user_id=user_id,
35163518
config=types.CreateRuntimeSessionConfig(
@@ -3574,10 +3576,12 @@ def _execute_agent_run_with_retry(
35743576
return {"error": "runtime.api_resource is None."}
35753577
if runtime.api_client is None:
35763578
return {"error": "runtime.api_client is None."}
3579+
from . import sessions
3580+
35773581
session_name = f"{runtime.api_resource.name}/sessions/{session_id}"
35783582
base_ts = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc)
35793583
for i, ag_event in enumerate(history_events):
3580-
runtime.api_client.sessions.events.append(
3584+
sessions.Sessions(runtime.api_client._api_client).events.append(
35813585
name=session_name,
35823586
author=ag_event.author or "user",
35833587
invocation_id="history",
@@ -3635,7 +3639,7 @@ async def _execute_local_agent_run_with_retry_async(
36353639
# Lazy-import ADK dependencies to avoid top-level import failures when
36363640
# google-adk is not installed.
36373641
from google.adk.runners import Runner
3638-
from google.adk.sessions import InMemorySessionService
3642+
from google.adk.sessions.in_memory_session_service import InMemorySessionService
36393643

36403644
# Multi-turn agent scraping with user simulation.
36413645
if user_simulator_config or "conversation_plan" in row:

‎tests/unit/agentplatform/genai/replays/test_run_inference.py‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,11 @@
1414
#
1515
# pylint: disable=protected-access,bad-continuation,missing-function-docstring
1616

17-
import pytest
18-
17+
from google.adk.agents import LlmAgent
1918
from tests.unit.agentplatform.genai.replays import pytest_helper
2019
from agentplatform._genai import types
2120
from google.genai import types as genai_types
2221

23-
pytest.importorskip(
24-
"google.adk", reason="google-adk not installed, skipping ADK agent tests"
25-
)
26-
from google.adk.agents import ( # noqa: E402
27-
LlmAgent,
28-
) # pylint: disable=g-import-not-at-top,g-bad-import-order
29-
3022

3123
def test_inference_with_eval_cases_multi_turn_agent_data(client):
3224
"""Tests run_inference with multi-turn agent_data in eval_cases.
@@ -187,6 +179,7 @@ def test_inference_with_prompt_column_local_agent(client):
187179
assert response_val is not None
188180
assert isinstance(response_val, str)
189181
assert len(response_val) > 0
182+
assert '"error"' not in response_val
190183

191184

192185
def test_inference_with_completed_and_incomplete_agent_data(client):
@@ -298,6 +291,7 @@ def test_inference_with_completed_and_incomplete_agent_data(client):
298291
assert row1_response is not None
299292
assert isinstance(row1_response, str)
300293
assert len(row1_response) > 0
294+
assert '"error"' not in row1_response
301295

302296

303297
def test_inference_with_gemini_agent(client):

‎tests/unit/agentplatform/genai/test_evals.py‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3655,12 +3655,14 @@ def test_run_inference_with_runtime_with_response_column_raises_error(
36553655
"'intermediate_events' or 'response' columns"
36563656
) in str(excinfo.value)
36573657

3658+
@mock.patch("agentplatform._genai.sessions.Sessions")
36583659
@mock.patch.object(_evals_utils, "EvalDatasetLoader")
36593660
@mock.patch.object(_evals_common.agentplatform, "Client")
36603661
def test_run_inference_with_runtime_falls_back_to_managed_sessions_api(
36613662
self,
36623663
mock_agentplatform_client,
36633664
mock_eval_dataset_loader,
3665+
mock_sessions,
36643666
):
36653667
"""Tests that run_inference falls back to the managed Sessions API
36663668
when the agent engine does not have create_session registered."""
@@ -3694,7 +3696,7 @@ def test_run_inference_with_runtime_falls_back_to_managed_sessions_api(
36943696
"projects/test-project/locations/us-central1"
36953697
"/reasoningEngines/123/sessions/managed-session-1"
36963698
)
3697-
mock_runtime.api_client.sessions.create.return_value = mock_session_operation
3699+
mock_sessions.return_value.create.return_value = mock_session_operation
36983700

36993701
stream_query_return_value = [
37003702
{
@@ -3719,7 +3721,8 @@ def test_run_inference_with_runtime_falls_back_to_managed_sessions_api(
37193721
)
37203722

37213723
# Verify the managed Sessions API was called as fallback.
3722-
mock_runtime.api_client.sessions.create.assert_called_once_with(
3724+
mock_sessions.assert_called_once_with(mock_runtime.api_client._api_client)
3725+
mock_sessions.return_value.create.assert_called_once_with(
37233726
name="projects/test-project/locations/us-central1/reasoningEngines/123",
37243727
user_id="123",
37253728
config=agentplatform_genai_types.CreateRuntimeSessionConfig(
@@ -3832,7 +3835,9 @@ def run_async_side_effect(*args, **kwargs):
38323835
sys.modules,
38333836
{
38343837
"google.adk": mock.MagicMock(),
3835-
"google.adk.sessions": mock_adk_sessions_module,
3838+
"google.adk.sessions.in_memory_session_service": (
3839+
mock_adk_sessions_module
3840+
),
38363841
"google.adk.runners": mock_adk_runners_module,
38373842
"google.adk.agents": mock.MagicMock(),
38383843
},
@@ -11359,7 +11364,11 @@ def test_local_agent_run_retries_429_and_fails_fast_on_400(self, mock_sleep):
1135911364

1136011365
with mock.patch.dict(
1136111366
sys.modules,
11362-
{"google.adk.runners": mock_runners, "google.adk.sessions": mock_sessions},
11367+
{
11368+
"google.adk.runners": mock_runners,
11369+
"google.adk.sessions": mock_sessions,
11370+
"google.adk.sessions.in_memory_session_service": mock_sessions,
11371+
},
1136311372
):
1136411373
result = _evals_common._execute_local_agent_run_with_retry(
1136511374
row=pd.Series({"prompt": "prompt"}),

‎vertexai/_genai/_evals_common.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2416,7 +2416,7 @@ async def _execute_local_agent_run_with_retry_async(
24162416
# Lazy-import ADK dependencies to avoid top-level import failures when
24172417
# google-adk is not installed.
24182418
from google.adk.runners import Runner
2419-
from google.adk.sessions import InMemorySessionService
2419+
from google.adk.sessions.in_memory_session_service import InMemorySessionService
24202420

24212421
# Multi-turn agent scraping with user simulation.
24222422
if user_simulator_config or "conversation_plan" in row:

0 commit comments

Comments
 (0)