3030GROUP_INDEX_KEY = "index"
3131GROUP_HAS_REASONING_KEY = "has_reasoning"
3232GROUP_TOKEN_COUNT_KEY = "token_count" # ruff:ignore[hardcoded-password-string] # nosec B105 - compaction metadata key, not a credential
33+ GROUP_TOKEN_COUNT_BASIS_KEY = "token_count_basis" # ruff:ignore[hardcoded-password-string] # nosec B105 - compaction metadata key, not a credential
34+ # Bumped whenever the token-estimation serialization changes. Cached counts
35+ # stamped with an older basis are recomputed instead of reused: annotations
36+ # survive Message.to_dict()/from_dict() round-trips, so without the basis a
37+ # transcript annotated before a serialization change keeps its stale counts.
38+ TOKEN_COUNT_BASIS_VERSION = 2
3339EXCLUDED_KEY = "_excluded"
3440EXCLUDE_REASON_KEY = "_exclude_reason"
3541SUMMARY_OF_MESSAGE_IDS_KEY = "_summary_of_message_ids"
@@ -458,7 +464,13 @@ def _write_group_annotation(
458464 token_count : int | None = None
459465 if existing_raw_annotation is not None :
460466 raw_token_count = existing_raw_annotation .get (GROUP_TOKEN_COUNT_KEY )
461- if isinstance (raw_token_count , int ) or raw_token_count is None :
467+ raw_token_count_basis = existing_raw_annotation .get (GROUP_TOKEN_COUNT_BASIS_KEY )
468+ # Preserve a cached count only when it was computed with the current
469+ # serialization basis; counts from older bases are stale and get
470+ # recomputed (see TOKEN_COUNT_BASIS_VERSION).
471+ if raw_token_count_basis == TOKEN_COUNT_BASIS_VERSION and (
472+ isinstance (raw_token_count , int ) or raw_token_count is None
473+ ):
462474 token_count = raw_token_count
463475 unknown_fields = {
464476 key : value
@@ -470,6 +482,7 @@ def _write_group_annotation(
470482 GROUP_INDEX_KEY ,
471483 GROUP_HAS_REASONING_KEY ,
472484 GROUP_TOKEN_COUNT_KEY ,
485+ GROUP_TOKEN_COUNT_BASIS_KEY ,
473486 }
474487 }
475488
@@ -479,6 +492,7 @@ def _write_group_annotation(
479492 GROUP_INDEX_KEY : index ,
480493 GROUP_HAS_REASONING_KEY : has_reasoning ,
481494 GROUP_TOKEN_COUNT_KEY : token_count ,
495+ GROUP_TOKEN_COUNT_BASIS_KEY : TOKEN_COUNT_BASIS_VERSION if token_count is not None else None ,
482496 }
483497 annotation .update (unknown_fields )
484498 message .additional_properties [GROUP_ANNOTATION_KEY ] = annotation
@@ -512,6 +526,10 @@ def _token_count(message: Message) -> int | None:
512526 if annotation is None :
513527 return None
514528 token_count = annotation .get (GROUP_TOKEN_COUNT_KEY )
529+ # Counts stamped with an older serialization basis are stale: they were
530+ # computed from a different serialized form and must be recomputed.
531+ if annotation .get (GROUP_TOKEN_COUNT_BASIS_KEY ) != TOKEN_COUNT_BASIS_VERSION :
532+ return None
515533 return token_count if isinstance (token_count , int ) else None
516534
517535
@@ -520,6 +538,7 @@ def _write_token_count(message: Message, token_count: int) -> None:
520538 if annotation is None :
521539 return
522540 annotation [GROUP_TOKEN_COUNT_KEY ] = token_count
541+ annotation [GROUP_TOKEN_COUNT_BASIS_KEY ] = TOKEN_COUNT_BASIS_VERSION
523542 message .additional_properties [GROUP_ANNOTATION_KEY ] = annotation
524543
525544
@@ -694,17 +713,54 @@ def annotate_message_groups(
694713 return _ordered_group_ids_from_annotations (messages )
695714
696715
716+ _OPAQUE_REASONING_KEYS : Final [frozenset [str ]] = frozenset ({"encrypted_content" })
717+
718+
719+ def _strip_opaque_reasoning_payload (value : Any ) -> Any :
720+ """Recursively drop opaque reasoning members, keeping clear-text payloads."""
721+ if isinstance (value , dict ):
722+ filtered = {
723+ key : _strip_opaque_reasoning_payload (item )
724+ for key , item in value .items ()
725+ if key not in _OPAQUE_REASONING_KEYS
726+ }
727+ return filtered or None
728+ if isinstance (value , list ):
729+ filtered = [item for item in (_strip_opaque_reasoning_payload (item ) for item in value ) if item is not None ]
730+ return filtered or None
731+ return value
732+
733+
697734def _serialize_content (content : Content ) -> dict [str , Any ]:
698735 payload = content .to_dict (exclude_none = True )
699736 payload .pop ("raw_representation" , None )
700737 # ``items`` mirrors ``result`` for function_result content; exclude it
701738 # to avoid double-counting tokens during estimation.
702739 payload .pop ("items" , None )
703- # ``protected_data`` holds provider-encrypted reasoning payloads (e.g.
704- # OpenAI ``encrypted_content``, Anthropic thinking ``signature``) that the
705- # model replays opaquely and never tokenises; exclude them so estimation
706- # measures the text the model actually sees.
707- payload .pop ("protected_data" , None )
740+ # ``protected_data`` carries provider reasoning payloads. JSON-serialised
741+ # reasoning_details (Chat Completions) are replayed to the provider as
742+ # clear text -- ``summary``, ``reasoning_text`` and nested ``reasoning.text``
743+ # are part of the context the provider receives, so only their opaque
744+ # members (``encrypted_content``) are excluded and the clear text stays
745+ # counted. Anything that is not such a JSON structure (Anthropic thinking
746+ # ``signature``, Responses API ``encrypted_content`` blobs) is replayed
747+ # opaquely and never tokenised; exclude it so estimation measures the text
748+ # the model actually sees.
749+ protected_data = payload .get ("protected_data" )
750+ if isinstance (protected_data , str ) and protected_data :
751+ try :
752+ reasoning_payload = json .loads (protected_data )
753+ except ValueError :
754+ reasoning_payload = None
755+ filtered = (
756+ _strip_opaque_reasoning_payload (reasoning_payload ) if isinstance (reasoning_payload , (dict , list )) else None
757+ )
758+ if filtered is None :
759+ payload .pop ("protected_data" , None )
760+ else :
761+ payload ["protected_data" ] = json .dumps (filtered , ensure_ascii = False )
762+ else :
763+ payload .pop ("protected_data" , None )
708764 additional_properties = payload .get ("additional_properties" )
709765 if isinstance (additional_properties , dict ) and "encrypted_content" in additional_properties :
710766 payload ["additional_properties" ] = {
0 commit comments