Skip to content

Commit 25f5214

Browse files
svfatcopybara-github
authored andcommitted
fix: log an error when a model returns multiple candidates
Merge google#6519 Only the first candidate was ever used; the extra ones were dropped silently. In streaming (both LiteLLM and Gemini) this also narrows what reaches the caller: candidates or choices carrying a non-zero index are skipped instead of being concatenated into the first candidate's text. PiperOrigin-RevId: 976670666
1 parent c7ef8cf commit 25f5214

4 files changed

Lines changed: 434 additions & 3 deletions

File tree

‎src/google/adk/models/google_llm.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,16 +300,41 @@ async def generate_content_async(
300300
# only difference is bidi rely on complete_turn flag to detect end while
301301
# sse depends on finish_reason.
302302
aggregator = StreamingResponseAggregator()
303+
multiple_candidates_logged = False
304+
last_usage_metadata = None
303305
async with Aclosing(responses) as agen:
304306
async for response in agen:
305307
if logger.isEnabledFor(logging.DEBUG):
306308
logger.debug(_build_response_log(response))
309+
if response.usage_metadata:
310+
last_usage_metadata = response.usage_metadata
311+
if (
312+
not multiple_candidates_logged
313+
and response.candidates
314+
and (
315+
len(response.candidates) > 1
316+
or any(c.index for c in response.candidates)
317+
)
318+
):
319+
multiple_candidates_logged = True
320+
logger.error(
321+
'Multiple candidates found in streaming response but only the'
322+
' first one will be used.'
323+
)
324+
if response.candidates:
325+
response.candidates = [
326+
c for c in response.candidates if not c.index
327+
]
328+
if not response.candidates:
329+
continue
307330
async with Aclosing(
308331
aggregator.process_response(response)
309332
) as aggregator_gen:
310333
async for llm_response in aggregator_gen:
311334
yield llm_response
312335
if (close_result := aggregator.close()) is not None:
336+
if last_usage_metadata:
337+
close_result.usage_metadata = last_usage_metadata
313338
# Populate cache metadata in the final aggregated response for
314339
# streaming
315340
if cache_metadata and cache_manager is not None:
@@ -328,6 +353,12 @@ async def generate_content_async(
328353
if logger.isEnabledFor(logging.DEBUG):
329354
logger.debug(_build_response_log(response))
330355

356+
if response.candidates and len(response.candidates) > 1:
357+
logger.error(
358+
'Multiple candidates found in response but only the first one'
359+
' will be used.'
360+
)
361+
331362
llm_response = LlmResponse.create(response)
332363
if cache_metadata and cache_manager is not None:
333364
cache_manager.populate_cache_metadata_in_response(

‎src/google/adk/models/lite_llm.py‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2311,11 +2311,13 @@ def _has_meaningful_signal(message: Message | Delta | None) -> bool:
23112311
"Unexpected response type from LiteLLM: %r" % (type(response),)
23122312
)
23132313

2314-
choices = response.get("choices")
2315-
if not choices:
2314+
# Extra candidates arrive as extra choices, either in the same chunk or in
2315+
# chunks carrying only a non-zero index; only the first candidate is used.
2316+
choices = response.get("choices") or []
2317+
choice = next((c for c in choices if not c.get("index")), None)
2318+
if choice is None:
23162319
yield None, None
23172320
else:
2318-
choice = choices[0]
23192321
finish_reason = choice.get("finish_reason")
23202322
if message_field == "delta":
23212323
message = choice.get("delta")
@@ -2439,6 +2441,11 @@ def _model_response_to_generate_content_response(
24392441
message = None
24402442
finish_reason = None
24412443
if (choices := response.get("choices")) and choices:
2444+
if len(choices) > 1:
2445+
logger.error(
2446+
"Multiple choices found in response but only the first one will be"
2447+
" used."
2448+
)
24422449
first_choice = choices[0]
24432450
message = first_choice.get("message", None)
24442451
finish_reason = first_choice.get("finish_reason", None)
@@ -3243,6 +3250,7 @@ async def generate_content_async(
32433250
grounding_metadata = None
32443251
last_finish_reason: str | None = None
32453252
fallback_index = 0
3253+
multiple_choices_logged = False
32463254

32473255
def _finalize_tool_call_response(
32483256
*, model_version: str, finish_reason: str
@@ -3338,6 +3346,16 @@ def _reset_stream_buffers() -> None:
33383346
last_finish_reason = None
33393347

33403348
async for part in await self.llm_client.acompletion(**completion_args):
3349+
part_choices = part.get("choices") or []
3350+
if not multiple_choices_logged and (
3351+
len(part_choices) > 1
3352+
or any(choice.get("index") for choice in part_choices)
3353+
):
3354+
multiple_choices_logged = True
3355+
logger.error(
3356+
"Multiple choices found in streaming response but only the first"
3357+
" one will be used."
3358+
)
33413359
# Grounding metadata can arrive on the first chunk (search queries) or
33423360
# the final chunk (supports); keep the latest non-empty one.
33433361
part_grounding = _extract_grounding_metadata(part)

‎tests/unittests/models/test_google_llm.py‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,100 @@ async def mock_coro():
453453
mock_client.aio.models.generate_content.assert_called_once()
454454

455455

456+
@pytest.mark.asyncio
457+
async def test_generate_content_async_multiple_candidates_logs_error(
458+
gemini_llm, llm_request, generate_content_response, caplog
459+
):
460+
generate_content_response.candidates = [
461+
types.Candidate(
462+
content=Content(
463+
role="model", parts=[Part.from_text(text="First candidate")]
464+
),
465+
finish_reason=types.FinishReason.STOP,
466+
),
467+
types.Candidate(
468+
content=Content(
469+
role="model", parts=[Part.from_text(text="Second candidate")]
470+
),
471+
finish_reason=types.FinishReason.STOP,
472+
),
473+
]
474+
475+
with mock.patch.object(gemini_llm, "api_client") as mock_client:
476+
477+
async def mock_coro():
478+
return generate_content_response
479+
480+
mock_client.aio.models.generate_content.return_value = mock_coro()
481+
482+
with caplog.at_level(logging.ERROR):
483+
responses = [
484+
response
485+
async for response in gemini_llm.generate_content_async(
486+
llm_request, stream=False
487+
)
488+
]
489+
490+
mock_client.aio.models.generate_content.assert_called_once()
491+
assert len(responses) == 1
492+
assert responses[0].content.parts[0].text == "First candidate"
493+
errors = [
494+
record
495+
for record in caplog.records
496+
if "Multiple candidates found in response" in record.getMessage()
497+
]
498+
assert len(errors) == 1
499+
assert errors[0].name == "google_adk.google.adk.models.google_llm"
500+
501+
502+
@pytest.mark.asyncio
503+
async def test_generate_content_async_stream_multiple_candidates_logs_error(
504+
gemini_llm, llm_request, caplog
505+
):
506+
with mock.patch.object(gemini_llm, "api_client") as mock_client:
507+
mock_responses = [
508+
types.GenerateContentResponse(
509+
candidates=[
510+
types.Candidate(
511+
content=Content(
512+
role="model", parts=[Part.from_text(text=first)]
513+
),
514+
finish_reason=None,
515+
),
516+
types.Candidate(
517+
content=Content(
518+
role="model", parts=[Part.from_text(text=second)]
519+
),
520+
finish_reason=None,
521+
),
522+
]
523+
)
524+
for first, second in (("Hello", "World"), ("Hello2", "World2"))
525+
]
526+
527+
async def mock_coro():
528+
return MockAsyncIterator(mock_responses)
529+
530+
mock_client.aio.models.generate_content_stream.return_value = mock_coro()
531+
532+
with caplog.at_level(logging.ERROR):
533+
_ = [
534+
response
535+
async for response in gemini_llm.generate_content_async(
536+
llm_request, stream=True
537+
)
538+
]
539+
540+
errors = [
541+
record
542+
for record in caplog.records
543+
if "Multiple candidates found in streaming response"
544+
in record.getMessage()
545+
]
546+
assert len(errors) == 1
547+
assert errors[0].name == "google_adk.google.adk.models.google_llm"
548+
549+
456550
@pytest.mark.asyncio
457551
async def test_generate_content_async_stream(gemini_llm, llm_request):
458552
with mock.patch.object(gemini_llm, "api_client") as mock_client:
@@ -3043,3 +3137,182 @@ async def __aexit__(self, *args):
30433137
# The log is still emitted and still useful.
30443138
assert "gemini-2.5-flash" in caplog.text
30453139
assert "Modality.AUDIO" in caplog.text
3140+
3141+
3142+
@pytest.mark.asyncio
3143+
async def test_generate_content_async_stream_secondary_candidate_chunk(
3144+
gemini_llm, llm_request, caplog
3145+
):
3146+
"""Test a candidate streamed in its own chunk is detected and skipped."""
3147+
with mock.patch.object(gemini_llm, "api_client") as mock_client:
3148+
mock_responses = [
3149+
types.GenerateContentResponse(
3150+
candidates=[
3151+
types.Candidate(
3152+
index=0,
3153+
content=Content(
3154+
role="model", parts=[Part.from_text(text="Hello")]
3155+
),
3156+
finish_reason=None,
3157+
)
3158+
]
3159+
),
3160+
types.GenerateContentResponse(
3161+
candidates=[
3162+
types.Candidate(
3163+
index=1,
3164+
content=Content(
3165+
role="model", parts=[Part.from_text(text="Other")]
3166+
),
3167+
finish_reason=None,
3168+
)
3169+
]
3170+
),
3171+
types.GenerateContentResponse(
3172+
candidates=[
3173+
types.Candidate(
3174+
index=0,
3175+
content=Content(
3176+
role="model", parts=[Part.from_text(text=" world")]
3177+
),
3178+
finish_reason=types.FinishReason.STOP,
3179+
)
3180+
]
3181+
),
3182+
]
3183+
3184+
async def mock_coro():
3185+
return MockAsyncIterator(mock_responses)
3186+
3187+
mock_client.aio.models.generate_content_stream.return_value = mock_coro()
3188+
3189+
with caplog.at_level(logging.ERROR):
3190+
responses = [
3191+
response
3192+
async for response in gemini_llm.generate_content_async(
3193+
llm_request, stream=True
3194+
)
3195+
]
3196+
3197+
assert responses[-1].content.parts[0].text == "Hello world"
3198+
errors = [
3199+
record
3200+
for record in caplog.records
3201+
if "Multiple candidates found in streaming response"
3202+
in record.getMessage()
3203+
]
3204+
assert len(errors) == 1
3205+
3206+
3207+
@pytest.mark.asyncio
3208+
async def test_generate_content_async_stream_secondary_candidate_chunk_is_skipped(
3209+
gemini_llm, llm_request
3210+
):
3211+
"""Test a chunk holding only secondary candidates yields no partial response."""
3212+
with mock.patch.object(gemini_llm, "api_client") as mock_client:
3213+
mock_responses = [
3214+
types.GenerateContentResponse(
3215+
candidates=[
3216+
types.Candidate(
3217+
index=0,
3218+
content=Content(
3219+
role="model", parts=[Part.from_text(text="Hello")]
3220+
),
3221+
finish_reason=None,
3222+
)
3223+
]
3224+
),
3225+
types.GenerateContentResponse(
3226+
candidates=[
3227+
types.Candidate(
3228+
index=1,
3229+
content=Content(
3230+
role="model", parts=[Part.from_text(text="Other")]
3231+
),
3232+
finish_reason=None,
3233+
)
3234+
]
3235+
),
3236+
types.GenerateContentResponse(
3237+
candidates=[
3238+
types.Candidate(
3239+
index=0,
3240+
content=Content(
3241+
role="model", parts=[Part.from_text(text=" world")]
3242+
),
3243+
finish_reason=types.FinishReason.STOP,
3244+
)
3245+
]
3246+
),
3247+
]
3248+
3249+
async def mock_coro():
3250+
return MockAsyncIterator(mock_responses)
3251+
3252+
mock_client.aio.models.generate_content_stream.return_value = mock_coro()
3253+
3254+
responses = [
3255+
response
3256+
async for response in gemini_llm.generate_content_async(
3257+
llm_request, stream=True
3258+
)
3259+
]
3260+
3261+
assert len(responses) == 3
3262+
assert [r.content.parts[0].text for r in responses] == [
3263+
"Hello",
3264+
" world",
3265+
"Hello world",
3266+
]
3267+
3268+
3269+
@pytest.mark.asyncio
3270+
async def test_generate_content_async_stream_secondary_candidate_chunk_preserves_usage_metadata(
3271+
gemini_llm, llm_request
3272+
):
3273+
"""Test that usage metadata on a chunk holding only secondary candidates is preserved."""
3274+
with mock.patch.object(gemini_llm, "api_client") as mock_client:
3275+
mock_responses = [
3276+
types.GenerateContentResponse(
3277+
candidates=[
3278+
types.Candidate(
3279+
index=0,
3280+
content=Content(
3281+
role="model", parts=[Part.from_text(text="Hello")]
3282+
),
3283+
finish_reason=None,
3284+
)
3285+
]
3286+
),
3287+
types.GenerateContentResponse(
3288+
candidates=[
3289+
types.Candidate(
3290+
index=1,
3291+
content=Content(
3292+
role="model", parts=[Part.from_text(text="Other")]
3293+
),
3294+
finish_reason=types.FinishReason.STOP,
3295+
)
3296+
],
3297+
usage_metadata=types.GenerateContentResponseUsageMetadata(
3298+
prompt_token_count=10,
3299+
candidates_token_count=5,
3300+
total_token_count=15,
3301+
),
3302+
),
3303+
]
3304+
3305+
async def mock_coro():
3306+
return MockAsyncIterator(mock_responses)
3307+
3308+
mock_client.aio.models.generate_content_stream.return_value = mock_coro()
3309+
3310+
responses = [
3311+
response
3312+
async for response in gemini_llm.generate_content_async(
3313+
llm_request, stream=True
3314+
)
3315+
]
3316+
3317+
assert responses[-1].usage_metadata is not None
3318+
assert responses[-1].usage_metadata.total_token_count == 15

0 commit comments

Comments
 (0)