Skip to content

Commit 83e43ea

Browse files
yyyu-googlecopybara-github
authored andcommitted
feat: allow automatic resumption of continuation token for in generate content and generate content stream. Default behavior for this feature is disabled
PiperOrigin-RevId: 996730675
1 parent ccd5dd3 commit 83e43ea

7 files changed

Lines changed: 2619 additions & 332 deletions

File tree

‎google/genai/_extra_utils.py‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -777,3 +777,232 @@ def get_usage_header(
777777

778778
http_options.headers = existing_headers
779779
return config_model
780+
781+
782+
def should_enable_automatic_continuation(
783+
config: Optional[types.GenerateContentConfigOrDict],
784+
*,
785+
default_enabled: bool = True,
786+
) -> bool:
787+
"""Returns whether automatic continuation token resumption is enabled."""
788+
if config is None:
789+
return default_enabled
790+
if isinstance(config, dict):
791+
automatic_continuation = config.get('automatic_continuation')
792+
else:
793+
automatic_continuation = getattr(config, 'automatic_continuation', None)
794+
if automatic_continuation is None:
795+
return default_enabled
796+
return bool(automatic_continuation)
797+
798+
799+
def is_resumable_finish_reason(
800+
finish_reason: Optional[types.FinishReason],
801+
) -> bool:
802+
"""Returns True if finish_reason is eligible for automatic continuation."""
803+
return finish_reason == types.FinishReason.CONTINUATION
804+
805+
806+
def should_continue_generation(
807+
response: Optional[types.GenerateContentResponse],
808+
) -> Optional[bytes]:
809+
"""Returns continuation_token if generation should auto-resume, else None."""
810+
if not response or not response.candidates:
811+
return None
812+
candidate = response.candidates[0]
813+
if not candidate.continuation_token:
814+
return None
815+
if is_resumable_finish_reason(candidate.finish_reason):
816+
return candidate.continuation_token
817+
return None
818+
819+
820+
def prepare_continuation_config(
821+
base_config: Optional[types.GenerateContentConfigOrDict],
822+
continuation_token: Optional[bytes],
823+
*,
824+
clear_automatic_continuation: bool = False,
825+
) -> Optional[types.GenerateContentConfig]:
826+
"""Returns a shallow copy of base_config with continuation_token set."""
827+
has_auto_cont = (
828+
base_config.get('automatic_continuation') is not None
829+
if isinstance(base_config, dict)
830+
else getattr(base_config, 'automatic_continuation', None) is not None
831+
)
832+
need_clear_auto_cont = clear_automatic_continuation and has_auto_cont
833+
if (
834+
not continuation_token
835+
and not need_clear_auto_cont
836+
and (
837+
base_config is None
838+
or isinstance(base_config, types.GenerateContentConfig)
839+
)
840+
):
841+
return base_config
842+
if base_config is None:
843+
return types.GenerateContentConfig(continuation_token=continuation_token)
844+
if isinstance(base_config, dict):
845+
copied_dict: dict[str, Any] = dict(base_config)
846+
if continuation_token:
847+
copied_dict['continuation_token'] = continuation_token
848+
if need_clear_auto_cont:
849+
copied_dict['automatic_continuation'] = None
850+
return types.GenerateContentConfig(**copied_dict)
851+
updates: dict[str, Any] = {}
852+
if continuation_token:
853+
updates['continuation_token'] = continuation_token
854+
if need_clear_auto_cont:
855+
updates['automatic_continuation'] = None
856+
return base_config.model_copy(update=updates)
857+
858+
859+
def merge_modality_token_counts(
860+
prev_list: list[types.ModalityTokenCount],
861+
curr_list: list[types.ModalityTokenCount],
862+
) -> list[types.ModalityTokenCount]:
863+
"""Sums token_count per modality across two lists of ModalityTokenCount."""
864+
counts_by_modality: dict[Any, int] = {}
865+
order: list[Any] = []
866+
for item in prev_list + curr_list:
867+
if item.modality not in counts_by_modality:
868+
counts_by_modality[item.modality] = 0
869+
order.append(item.modality)
870+
counts_by_modality[item.modality] += item.token_count or 0
871+
return [
872+
types.ModalityTokenCount(
873+
modality=mod, token_count=counts_by_modality[mod]
874+
)
875+
for mod in order
876+
]
877+
878+
879+
def merge_safety_ratings(
880+
prev_list: list[types.SafetyRating],
881+
curr_list: list[types.SafetyRating],
882+
) -> list[types.SafetyRating]:
883+
"""Deduplicates safety ratings by category, keeping the latest hop's rating."""
884+
by_category: dict[Any, types.SafetyRating] = {}
885+
order: list[Any] = []
886+
for rating in prev_list + curr_list:
887+
if rating.category not in by_category:
888+
order.append(rating.category)
889+
by_category[rating.category] = rating
890+
return [by_category[cat] for cat in order]
891+
892+
893+
def merge_candidates(
894+
prev_candidates: list[types.Candidate],
895+
curr_candidates: list[types.Candidate],
896+
) -> list[types.Candidate]:
897+
"""Merges candidates element-wise across continuation hops."""
898+
if not prev_candidates:
899+
return curr_candidates
900+
if not curr_candidates:
901+
return prev_candidates
902+
result: list[types.Candidate] = []
903+
for prev_candidate, curr_candidate in zip(prev_candidates, curr_candidates):
904+
merged_candidate = merge_pydantic_models(prev_candidate, curr_candidate)
905+
result.append(merged_candidate)
906+
result.extend(prev_candidates[len(curr_candidates) :])
907+
result.extend(curr_candidates[len(prev_candidates) :])
908+
return result
909+
910+
911+
def merge_pydantic_models(prev: C, curr: C) -> C:
912+
"""Recursively merges two Pydantic BaseModel instances across continuation hops."""
913+
merged_data: dict[str, Any] = {}
914+
for field_name in type(curr).model_fields:
915+
prev_val = getattr(prev, field_name, None)
916+
curr_val = getattr(curr, field_name, None)
917+
918+
# Terminal hop state on Candidate must reflect the final hop even if None
919+
# e.g., continuation_token is None when the final hop finishes with STOP
920+
if isinstance(curr, types.Candidate) and field_name in (
921+
'continuation_token',
922+
'finish_reason',
923+
'finish_message',
924+
):
925+
merged_data[field_name] = curr_val
926+
continue
927+
928+
if curr_val is None:
929+
merged_data[field_name] = prev_val
930+
elif prev_val is None:
931+
merged_data[field_name] = curr_val
932+
elif isinstance(prev_val, _common.BaseModel) and isinstance(
933+
curr_val, _common.BaseModel
934+
):
935+
merged_data[field_name] = merge_pydantic_models(prev_val, curr_val)
936+
elif (
937+
isinstance(prev_val, (int, float))
938+
and not isinstance(prev_val, bool)
939+
and isinstance(curr_val, (int, float))
940+
and not isinstance(curr_val, bool)
941+
and (
942+
isinstance(curr, types.GenerateContentResponseUsageMetadata)
943+
or field_name == 'token_count'
944+
or field_name.endswith('_count')
945+
or field_name.endswith('_sum')
946+
)
947+
):
948+
merged_data[field_name] = prev_val + curr_val
949+
elif isinstance(prev_val, list) and isinstance(curr_val, list):
950+
if field_name == 'candidates':
951+
merged_data[field_name] = merge_candidates(prev_val, curr_val)
952+
elif prev_val and isinstance(prev_val[0], types.ModalityTokenCount):
953+
merged_data[field_name] = merge_modality_token_counts(
954+
prev_val, curr_val
955+
)
956+
elif curr_val and isinstance(curr_val[0], types.ModalityTokenCount):
957+
merged_data[field_name] = merge_modality_token_counts(
958+
prev_val, curr_val
959+
)
960+
elif field_name == 'safety_ratings':
961+
merged_data[field_name] = merge_safety_ratings(prev_val, curr_val)
962+
else:
963+
# Concatenates Content.parts (keeping each hop's Part distinct and
964+
# preserving empty placeholder Parts), GroundingChunks, Citations, etc.
965+
merged_data[field_name] = prev_val + curr_val
966+
else:
967+
merged_data[field_name] = curr_val
968+
969+
return curr.model_copy(update=merged_data)
970+
971+
972+
def merge_continuation_responses(
973+
responses: list[types.GenerateContentResponse],
974+
config: Optional[types.GenerateContentConfigOrDict] = None,
975+
) -> types.GenerateContentResponse:
976+
"""Merges a sequence of continuation hop responses into a single response."""
977+
if not responses:
978+
return types.GenerateContentResponse()
979+
if len(responses) == 1:
980+
return responses[0]
981+
merged = responses[0]
982+
for resp in responses[1:]:
983+
merged = merge_pydantic_models(merged, resp)
984+
985+
if config is not None:
986+
response_schema = (
987+
config.get('response_schema')
988+
if isinstance(config, dict)
989+
else getattr(config, 'response_schema', None)
990+
)
991+
response_json_schema = (
992+
config.get('response_json_schema')
993+
if isinstance(config, dict)
994+
else getattr(config, 'response_json_schema', None)
995+
)
996+
if response_schema is not None or response_json_schema is not None:
997+
config_dict = {
998+
'response_schema': response_schema,
999+
'response_json_schema': response_json_schema,
1000+
}
1001+
reparsed = types.GenerateContentResponse._from_response(
1002+
response=merged.model_dump(exclude_none=True),
1003+
kwargs={'config': config_dict},
1004+
)
1005+
merged.parsed = reparsed.parsed
1006+
1007+
return merged
1008+

0 commit comments

Comments
 (0)