Skip to content

Commit efb44d8

Browse files
committed
fix(compaction): count clear-text reasoning details and invalidate stale token caches
Review follow-up for the protected-data token exclusion: - reasoning_details stored in protected_data are JSON-serialised and replayed to the provider as clear text (summary, reasoning_text and nested reasoning.text), so only opaque members (encrypted_content) are excluded from the estimate; non-JSON payloads (Anthropic signature, Responses API encrypted blobs) are still dropped entirely. - group token counts are stamped with a serialization basis version; counts cached under an older basis (e.g. sessions annotated before a serialization change) are recomputed instead of reused.
1 parent 5c5568b commit efb44d8

2 files changed

Lines changed: 121 additions & 6 deletions

File tree

‎python/packages/core/agent_framework/_compaction.py‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@
3030
GROUP_INDEX_KEY = "index"
3131
GROUP_HAS_REASONING_KEY = "has_reasoning"
3232
GROUP_TOKEN_COUNT_KEY = "token_count" # ruff:ignore[hardcoded-password-string] # nosec B105 - compaction metadata key, not a credential
33+
GROUP_TOKEN_COUNT_BASIS_KEY = "token_count_basis" # ruff:ignore[hardcoded-password-string] # nosec B105 - compaction metadata key, not a credential
34+
# Bumped whenever the token-estimation serialization changes. Cached counts
35+
# stamped with an older basis are recomputed instead of reused: annotations
36+
# survive Message.to_dict()/from_dict() round-trips, so without the basis a
37+
# transcript annotated before a serialization change keeps its stale counts.
38+
TOKEN_COUNT_BASIS_VERSION = 2
3339
EXCLUDED_KEY = "_excluded"
3440
EXCLUDE_REASON_KEY = "_exclude_reason"
3541
SUMMARY_OF_MESSAGE_IDS_KEY = "_summary_of_message_ids"
@@ -458,7 +464,13 @@ def _write_group_annotation(
458464
token_count: int | None = None
459465
if existing_raw_annotation is not None:
460466
raw_token_count = existing_raw_annotation.get(GROUP_TOKEN_COUNT_KEY)
461-
if isinstance(raw_token_count, int) or raw_token_count is None:
467+
raw_token_count_basis = existing_raw_annotation.get(GROUP_TOKEN_COUNT_BASIS_KEY)
468+
# Preserve a cached count only when it was computed with the current
469+
# serialization basis; counts from older bases are stale and get
470+
# recomputed (see TOKEN_COUNT_BASIS_VERSION).
471+
if raw_token_count_basis == TOKEN_COUNT_BASIS_VERSION and (
472+
isinstance(raw_token_count, int) or raw_token_count is None
473+
):
462474
token_count = raw_token_count
463475
unknown_fields = {
464476
key: value
@@ -470,6 +482,7 @@ def _write_group_annotation(
470482
GROUP_INDEX_KEY,
471483
GROUP_HAS_REASONING_KEY,
472484
GROUP_TOKEN_COUNT_KEY,
485+
GROUP_TOKEN_COUNT_BASIS_KEY,
473486
}
474487
}
475488

@@ -479,6 +492,7 @@ def _write_group_annotation(
479492
GROUP_INDEX_KEY: index,
480493
GROUP_HAS_REASONING_KEY: has_reasoning,
481494
GROUP_TOKEN_COUNT_KEY: token_count,
495+
GROUP_TOKEN_COUNT_BASIS_KEY: TOKEN_COUNT_BASIS_VERSION if token_count is not None else None,
482496
}
483497
annotation.update(unknown_fields)
484498
message.additional_properties[GROUP_ANNOTATION_KEY] = annotation
@@ -512,6 +526,10 @@ def _token_count(message: Message) -> int | None:
512526
if annotation is None:
513527
return None
514528
token_count = annotation.get(GROUP_TOKEN_COUNT_KEY)
529+
# Counts stamped with an older serialization basis are stale: they were
530+
# computed from a different serialized form and must be recomputed.
531+
if annotation.get(GROUP_TOKEN_COUNT_BASIS_KEY) != TOKEN_COUNT_BASIS_VERSION:
532+
return None
515533
return token_count if isinstance(token_count, int) else None
516534

517535

@@ -520,6 +538,7 @@ def _write_token_count(message: Message, token_count: int) -> None:
520538
if annotation is None:
521539
return
522540
annotation[GROUP_TOKEN_COUNT_KEY] = token_count
541+
annotation[GROUP_TOKEN_COUNT_BASIS_KEY] = TOKEN_COUNT_BASIS_VERSION
523542
message.additional_properties[GROUP_ANNOTATION_KEY] = annotation
524543

525544

@@ -694,17 +713,54 @@ def annotate_message_groups(
694713
return _ordered_group_ids_from_annotations(messages)
695714

696715

716+
_OPAQUE_REASONING_KEYS: Final[frozenset[str]] = frozenset({"encrypted_content"})
717+
718+
719+
def _strip_opaque_reasoning_payload(value: Any) -> Any:
720+
"""Recursively drop opaque reasoning members, keeping clear-text payloads."""
721+
if isinstance(value, dict):
722+
filtered = {
723+
key: _strip_opaque_reasoning_payload(item)
724+
for key, item in value.items()
725+
if key not in _OPAQUE_REASONING_KEYS
726+
}
727+
return filtered or None
728+
if isinstance(value, list):
729+
filtered = [item for item in (_strip_opaque_reasoning_payload(item) for item in value) if item is not None]
730+
return filtered or None
731+
return value
732+
733+
697734
def _serialize_content(content: Content) -> dict[str, Any]:
698735
payload = content.to_dict(exclude_none=True)
699736
payload.pop("raw_representation", None)
700737
# ``items`` mirrors ``result`` for function_result content; exclude it
701738
# to avoid double-counting tokens during estimation.
702739
payload.pop("items", None)
703-
# ``protected_data`` holds provider-encrypted reasoning payloads (e.g.
704-
# OpenAI ``encrypted_content``, Anthropic thinking ``signature``) that the
705-
# model replays opaquely and never tokenises; exclude them so estimation
706-
# measures the text the model actually sees.
707-
payload.pop("protected_data", None)
740+
# ``protected_data`` carries provider reasoning payloads. JSON-serialised
741+
# reasoning_details (Chat Completions) are replayed to the provider as
742+
# clear text -- ``summary``, ``reasoning_text`` and nested ``reasoning.text``
743+
# are part of the context the provider receives, so only their opaque
744+
# members (``encrypted_content``) are excluded and the clear text stays
745+
# counted. Anything that is not such a JSON structure (Anthropic thinking
746+
# ``signature``, Responses API ``encrypted_content`` blobs) is replayed
747+
# opaquely and never tokenised; exclude it so estimation measures the text
748+
# the model actually sees.
749+
protected_data = payload.get("protected_data")
750+
if isinstance(protected_data, str) and protected_data:
751+
try:
752+
reasoning_payload = json.loads(protected_data)
753+
except ValueError:
754+
reasoning_payload = None
755+
filtered = (
756+
_strip_opaque_reasoning_payload(reasoning_payload) if isinstance(reasoning_payload, (dict, list)) else None
757+
)
758+
if filtered is None:
759+
payload.pop("protected_data", None)
760+
else:
761+
payload["protected_data"] = json.dumps(filtered, ensure_ascii=False)
762+
else:
763+
payload.pop("protected_data", None)
708764
additional_properties = payload.get("additional_properties")
709765
if isinstance(additional_properties, dict) and "encrypted_content" in additional_properties:
710766
payload["additional_properties"] = {

‎python/packages/core/tests/core/test_compaction.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,62 @@ def test_annotate_message_groups_token_counts_ignore_encrypted_content_in_additi
472472
assert _token_count(with_message) == _token_count(without_message)
473473

474474

475+
def test_annotate_message_groups_token_counts_keep_clear_text_reasoning_details() -> None:
476+
import json
477+
478+
clear_text = "visible reasoning " * 128
479+
with_details = Content.from_text_reasoning(
480+
id="rs_1",
481+
text="",
482+
protected_data=json.dumps([{"type": "reasoning.text", "text": clear_text, "encrypted_content": "A" * 4096}]),
483+
)
484+
summary_only = Content.from_text_reasoning(
485+
id="rs_1",
486+
text="",
487+
protected_data=json.dumps([{"type": "reasoning.text", "text": clear_text}]),
488+
)
489+
without_details = Content.from_text_reasoning(id="rs_1", text="")
490+
final_answer = Content.from_text("final answer")
491+
492+
with_message = Message(role="assistant", contents=[with_details, final_answer])
493+
summary_only_message = Message(role="assistant", contents=[summary_only, final_answer])
494+
bare_message = Message(role="assistant", contents=[without_details, final_answer])
495+
496+
annotate_message_groups([with_message], tokenizer=CharacterEstimatorTokenizer())
497+
annotate_message_groups([summary_only_message], tokenizer=CharacterEstimatorTokenizer())
498+
annotate_message_groups([bare_message], tokenizer=CharacterEstimatorTokenizer())
499+
500+
# The opaque encrypted member is excluded while the clear-text reasoning
501+
# the provider replays stays counted.
502+
assert _token_count(with_message) == _token_count(summary_only_message)
503+
assert _token_count(with_message) > _token_count(bare_message)
504+
505+
506+
def test_annotate_token_counts_recomputes_counts_from_stale_serialization_basis() -> None:
507+
508+
from agent_framework._compaction import TOKEN_COUNT_BASIS_VERSION, _write_group_annotation, annotate_token_counts
509+
510+
message = Message(role="assistant", contents=[Content.from_text("final answer " * 64)])
511+
_write_group_annotation(message, group_id="g0", kind="assistant_text", index=0, has_reasoning=False)
512+
annotation = message.additional_properties[GROUP_ANNOTATION_KEY]
513+
assert isinstance(annotation, dict)
514+
# Simulate a count cached by a pre-fix serialization basis: value present,
515+
# no basis stamp (annotations survive to_dict/from_dict round-trips).
516+
annotation[GROUP_TOKEN_COUNT_KEY] = 1
517+
annotation.pop("token_count_basis", None)
518+
assert _token_count(message) == 1
519+
520+
annotate_token_counts([message], tokenizer=CharacterEstimatorTokenizer())
521+
522+
annotation = message.additional_properties[GROUP_ANNOTATION_KEY]
523+
assert isinstance(annotation, dict)
524+
assert annotation[GROUP_TOKEN_COUNT_KEY] != 1
525+
assert annotation["token_count_basis"] == TOKEN_COUNT_BASIS_VERSION
526+
527+
475528
def test_extend_compaction_messages_preserves_existing_annotations_and_tokens() -> None:
529+
from agent_framework._compaction import TOKEN_COUNT_BASIS_VERSION
530+
476531
tokenizer = CharacterEstimatorTokenizer()
477532
messages = [_assistant_function_call("c3")]
478533
annotate_message_groups(messages)
@@ -482,6 +537,10 @@ def test_extend_compaction_messages_preserves_existing_annotations_and_tokens()
482537
annotation = messages[0].additional_properties.get(GROUP_ANNOTATION_KEY)
483538
if isinstance(annotation, dict):
484539
annotation[GROUP_TOKEN_COUNT_KEY] = old_token_count
540+
# Counts cached under the current serialization basis are preserved
541+
# across extensions; counts without the current basis stamp are stale
542+
# and get recomputed (see test_annotate_token_counts_recomputes_...).
543+
annotation["token_count_basis"] = TOKEN_COUNT_BASIS_VERSION
485544

486545
extend_compaction_messages(messages, [_tool_result("c3", "ok")], tokenizer=tokenizer)
487546

0 commit comments

Comments
 (0)