Skip to content

Commit b27f3cd

Browse files
kashyapkumarcopybara-github
authored andcommitted
feat: accumulate streaming continuation usage metadata and expose initial_prompt_token_count
PiperOrigin-RevId: 997482740
1 parent 90bb32b commit b27f3cd

5 files changed

Lines changed: 355 additions & 31 deletions

File tree

‎google/genai/_extra_utils.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -946,8 +946,7 @@ def merge_pydantic_models(prev: C, curr: C) -> C:
946946
and isinstance(curr_val, (int, float))
947947
and not isinstance(curr_val, bool)
948948
and (
949-
isinstance(curr, types.GenerateContentResponseUsageMetadata)
950-
or field_name == 'token_count'
949+
field_name == 'token_count'
951950
or field_name.endswith('_count')
952951
or field_name.endswith('_sum')
953952
)
@@ -973,7 +972,18 @@ def merge_pydantic_models(prev: C, curr: C) -> C:
973972
else:
974973
merged_data[field_name] = curr_val
975974

976-
return curr.model_copy(update=merged_data)
975+
merged = curr.model_copy(update=merged_data)
976+
if (
977+
isinstance(prev, types.GenerateContentResponseUsageMetadata)
978+
and isinstance(curr, types.GenerateContentResponseUsageMetadata)
979+
and isinstance(merged, types.GenerateContentResponseUsageMetadata)
980+
):
981+
merged._initial_prompt_token_count = (
982+
prev.initial_prompt_token_count
983+
if prev.initial_prompt_token_count is not None
984+
else curr.initial_prompt_token_count
985+
)
986+
return merged
977987

978988

979989
def merge_continuation_responses(

‎google/genai/models.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6147,6 +6147,9 @@ def _generate_content_stream_with_continuation(
61476147
)
61486148
continuation_token: Optional[bytes] = None
61496149
hop_finish_reason: Optional[types.FinishReason] = None
6150+
accumulated_usage: Optional[types.GenerateContentResponseUsageMetadata] = (
6151+
None
6152+
)
61506153
is_first_hop = True
61516154

61526155
while is_first_hop or (
@@ -6160,6 +6163,7 @@ def _generate_content_stream_with_continuation(
61606163
)
61616164
hop_finish_reason = None
61626165
continuation_token = None
6166+
hop_usage: Optional[types.GenerateContentResponseUsageMetadata] = None
61636167

61646168
try:
61656169
for chunk in self._generate_content_stream(
@@ -6171,6 +6175,15 @@ def _generate_content_stream_with_continuation(
61716175
hop_finish_reason = candidate.finish_reason
61726176
if candidate.continuation_token:
61736177
continuation_token = candidate.continuation_token
6178+
if chunk.usage_metadata is not None:
6179+
if accumulated_usage is not None:
6180+
merged_usage = _extra_utils.merge_pydantic_models(
6181+
accumulated_usage, chunk.usage_metadata
6182+
)
6183+
chunk = chunk.model_copy(update={'usage_metadata': merged_usage})
6184+
hop_usage = merged_usage
6185+
else:
6186+
hop_usage = chunk.usage_metadata
61746187
yield chunk
61756188
except Exception: # pylint: disable=broad-exception-caught
61766189
# If the stream fails after emitting a checkpoint continuation_token
@@ -6182,6 +6195,8 @@ def _generate_content_stream_with_continuation(
61826195
and _extra_utils.is_resumable_finish_reason(hop_finish_reason)
61836196
):
61846197
raise
6198+
if hop_usage is not None:
6199+
accumulated_usage = hop_usage
61856200

61866201
def generate_content(
61876202
self,
@@ -8390,6 +8405,9 @@ async def _generate_content_stream_with_continuation(
83908405
)
83918406
continuation_token: Optional[bytes] = None
83928407
hop_finish_reason: Optional[types.FinishReason] = None
8408+
accumulated_usage: Optional[types.GenerateContentResponseUsageMetadata] = (
8409+
None
8410+
)
83938411
is_first_hop = True
83948412

83958413
while is_first_hop or (
@@ -8403,6 +8421,7 @@ async def _generate_content_stream_with_continuation(
84038421
)
84048422
hop_finish_reason = None
84058423
continuation_token = None
8424+
hop_usage: Optional[types.GenerateContentResponseUsageMetadata] = None
84068425

84078426
try:
84088427
response_stream = await self._generate_content_stream(
@@ -8415,6 +8434,15 @@ async def _generate_content_stream_with_continuation(
84158434
hop_finish_reason = candidate.finish_reason
84168435
if candidate.continuation_token:
84178436
continuation_token = candidate.continuation_token
8437+
if chunk.usage_metadata is not None:
8438+
if accumulated_usage is not None:
8439+
merged_usage = _extra_utils.merge_pydantic_models(
8440+
accumulated_usage, chunk.usage_metadata
8441+
)
8442+
chunk = chunk.model_copy(update={'usage_metadata': merged_usage})
8443+
hop_usage = merged_usage
8444+
else:
8445+
hop_usage = chunk.usage_metadata
84188446
yield chunk
84198447
except Exception: # pylint: disable=broad-exception-caught
84208448
# If the stream fails after emitting a checkpoint continuation_token
@@ -8426,6 +8454,8 @@ async def _generate_content_stream_with_continuation(
84268454
and _extra_utils.is_resumable_finish_reason(hop_finish_reason)
84278455
):
84288456
raise
8457+
if hop_usage is not None:
8458+
accumulated_usage = hop_usage
84298459

84308460
async def generate_content(
84318461
self,

0 commit comments

Comments
 (0)