Skip to content

Commit d90355f

Browse files
Lubaoshuaimoonbox3
andauthored
Python: exclude encrypted reasoning payloads from compaction token counts (#8354)
* Python: exclude encrypted reasoning payloads from compaction token counts The compaction token counter serialises each message and tokenises the JSON. Content.protected_data (OpenAI encrypted_content, Anthropic thinking signature, Gemini thought_signature, OpenAI reasoning_details) is opaque ciphertext the model replays but never tokenises, so counting it inflated every token-aware compaction decision. The same blob can also ride in additional_properties as encrypted_content. Both are now excluded from serialised content, matching the existing exclusions for raw_representation and mirrored items. Fixes #8348 * fix(compaction): count clear-text reasoning details and invalidate stale token caches Review follow-up for the protected-data token exclusion: - reasoning_details stored in protected_data are JSON-serialised and replayed to the provider as clear text (summary, reasoning_text and nested reasoning.text), so only opaque members (encrypted_content) are excluded from the estimate; non-JSON payloads (Anthropic signature, Responses API encrypted blobs) are still dropped entirely. - group token counts are stamped with a serialization basis version; counts cached under an older basis (e.g. sessions annotated before a serialization change) are recomputed instead of reused. * fix(compaction): resolve pyright/mypy typing errors in token estimation - annotate the recursive reasoning-payload filter with explicit dict[Any, Any] / list[Any] casts so iterating the narrowed Any dict does not leak Unknown types under pyright strict - cast additional_properties before the encrypted_content filter - assert non-None token counts in the clear-text regression test so the ordering comparison does not involve optional integers --------- Co-authored-by: Evan Mattson <[email protected]>
1 parent 999dda7 commit d90355f

2 files changed

Lines changed: 179 additions & 1 deletion

File tree

‎python/packages/core/agent_framework/_compaction.py‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@
3030
GROUP_INDEX_KEY = "index"
3131
GROUP_HAS_REASONING_KEY = "has_reasoning"
3232
GROUP_TOKEN_COUNT_KEY = "token_count" # ruff:ignore[hardcoded-password-string] # nosec B105 - compaction metadata key, not a credential
33+
GROUP_TOKEN_COUNT_BASIS_KEY = "token_count_basis" # ruff:ignore[hardcoded-password-string] # nosec B105 - compaction metadata key, not a credential
34+
# Bumped whenever the token-estimation serialization changes. Cached counts
35+
# stamped with an older basis are recomputed instead of reused: annotations
36+
# survive Message.to_dict()/from_dict() round-trips, so without the basis a
37+
# transcript annotated before a serialization change keeps its stale counts.
38+
TOKEN_COUNT_BASIS_VERSION = 2
3339
EXCLUDED_KEY = "_excluded"
3440
EXCLUDE_REASON_KEY = "_exclude_reason"
3541
SUMMARY_OF_MESSAGE_IDS_KEY = "_summary_of_message_ids"
@@ -496,7 +502,13 @@ def _write_group_annotation(
496502
token_count: int | None = None
497503
if existing_raw_annotation is not None:
498504
raw_token_count = existing_raw_annotation.get(GROUP_TOKEN_COUNT_KEY)
499-
if isinstance(raw_token_count, int) or raw_token_count is None:
505+
raw_token_count_basis = existing_raw_annotation.get(GROUP_TOKEN_COUNT_BASIS_KEY)
506+
# Preserve a cached count only when it was computed with the current
507+
# serialization basis; counts from older bases are stale and get
508+
# recomputed (see TOKEN_COUNT_BASIS_VERSION).
509+
if raw_token_count_basis == TOKEN_COUNT_BASIS_VERSION and (
510+
isinstance(raw_token_count, int) or raw_token_count is None
511+
):
500512
token_count = raw_token_count
501513
unknown_fields = {
502514
key: value
@@ -508,6 +520,7 @@ def _write_group_annotation(
508520
GROUP_INDEX_KEY,
509521
GROUP_HAS_REASONING_KEY,
510522
GROUP_TOKEN_COUNT_KEY,
523+
GROUP_TOKEN_COUNT_BASIS_KEY,
511524
}
512525
}
513526

@@ -517,6 +530,7 @@ def _write_group_annotation(
517530
GROUP_INDEX_KEY: index,
518531
GROUP_HAS_REASONING_KEY: has_reasoning,
519532
GROUP_TOKEN_COUNT_KEY: token_count,
533+
GROUP_TOKEN_COUNT_BASIS_KEY: TOKEN_COUNT_BASIS_VERSION if token_count is not None else None,
520534
}
521535
annotation.update(unknown_fields)
522536
message.additional_properties[GROUP_ANNOTATION_KEY] = annotation
@@ -550,6 +564,10 @@ def _token_count(message: Message) -> int | None:
550564
if annotation is None:
551565
return None
552566
token_count = annotation.get(GROUP_TOKEN_COUNT_KEY)
567+
# Counts stamped with an older serialization basis are stale: they were
568+
# computed from a different serialized form and must be recomputed.
569+
if annotation.get(GROUP_TOKEN_COUNT_BASIS_KEY) != TOKEN_COUNT_BASIS_VERSION:
570+
return None
553571
return token_count if isinstance(token_count, int) else None
554572

555573

@@ -558,6 +576,7 @@ def _write_token_count(message: Message, token_count: int) -> None:
558576
if annotation is None:
559577
return
560578
annotation[GROUP_TOKEN_COUNT_KEY] = token_count
579+
annotation[GROUP_TOKEN_COUNT_BASIS_KEY] = TOKEN_COUNT_BASIS_VERSION
561580
message.additional_properties[GROUP_ANNOTATION_KEY] = annotation
562581

563582

@@ -732,12 +751,68 @@ def annotate_message_groups(
732751
return _ordered_group_ids_from_annotations(messages)
733752

734753

754+
_OPAQUE_REASONING_KEYS: Final[frozenset[str]] = frozenset({"encrypted_content"})
755+
756+
757+
def _strip_opaque_reasoning_payload(value: Any) -> Any:
758+
"""Recursively drop opaque reasoning members, keeping clear-text payloads."""
759+
if isinstance(value, dict):
760+
entries = cast("dict[Any, Any]", value)
761+
filtered: dict[Any, Any] = {}
762+
for key, item in entries.items():
763+
if key in _OPAQUE_REASONING_KEYS:
764+
continue
765+
stripped = _strip_opaque_reasoning_payload(item)
766+
if stripped is not None:
767+
filtered[key] = stripped
768+
return filtered or None
769+
if isinstance(value, list):
770+
entries = cast("list[Any]", value)
771+
kept = [
772+
stripped
773+
for stripped in (_strip_opaque_reasoning_payload(entry) for entry in entries)
774+
if stripped is not None
775+
]
776+
return kept or None
777+
return value
778+
779+
735780
def _serialize_content(content: Content) -> dict[str, Any]:
736781
payload = content.to_dict(exclude_none=True)
737782
payload.pop("raw_representation", None)
738783
# ``items`` mirrors ``result`` for function_result content; exclude it
739784
# to avoid double-counting tokens during estimation.
740785
payload.pop("items", None)
786+
# ``protected_data`` carries provider reasoning payloads. JSON-serialised
787+
# reasoning_details (Chat Completions) are replayed to the provider as
788+
# clear text -- ``summary``, ``reasoning_text`` and nested ``reasoning.text``
789+
# are part of the context the provider receives, so only their opaque
790+
# members (``encrypted_content``) are excluded and the clear text stays
791+
# counted. Anything that is not such a JSON structure (Anthropic thinking
792+
# ``signature``, Responses API ``encrypted_content`` blobs) is replayed
793+
# opaquely and never tokenised; exclude it so estimation measures the text
794+
# the model actually sees.
795+
protected_data = payload.get("protected_data")
796+
if isinstance(protected_data, str) and protected_data:
797+
try:
798+
reasoning_payload = json.loads(protected_data)
799+
except ValueError:
800+
reasoning_payload = None
801+
filtered = (
802+
_strip_opaque_reasoning_payload(reasoning_payload) if isinstance(reasoning_payload, (dict, list)) else None
803+
)
804+
if filtered is None:
805+
payload.pop("protected_data", None)
806+
else:
807+
payload["protected_data"] = json.dumps(filtered, ensure_ascii=False)
808+
else:
809+
payload.pop("protected_data", None)
810+
additional_properties = payload.get("additional_properties")
811+
if isinstance(additional_properties, dict) and "encrypted_content" in additional_properties:
812+
typed_properties = cast("dict[str, Any]", additional_properties)
813+
payload["additional_properties"] = {
814+
key: value for key, value in typed_properties.items() if key != "encrypted_content"
815+
}
741816
return payload
742817

743818

‎python/packages/core/tests/core/test_compaction.py‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,106 @@ def test_annotate_message_groups_with_tokenizer_adds_token_counts() -> None:
435435
assert isinstance(_token_count(messages[1]), int)
436436

437437

438+
def test_annotate_message_groups_token_counts_ignore_protected_data() -> None:
439+
encrypted_blob = "A" * 4096
440+
with_payload = Message(
441+
role="assistant",
442+
contents=[
443+
Content.from_text_reasoning(id="rs_1", text="", protected_data=encrypted_blob),
444+
Content.from_text("final answer"),
445+
],
446+
)
447+
without_payload = Message(
448+
role="assistant",
449+
contents=[
450+
Content.from_text_reasoning(id="rs_1", text=""),
451+
Content.from_text("final answer"),
452+
],
453+
)
454+
455+
annotate_message_groups([with_payload], tokenizer=CharacterEstimatorTokenizer())
456+
annotate_message_groups([without_payload], tokenizer=CharacterEstimatorTokenizer())
457+
458+
assert _token_count(with_payload) == _token_count(without_payload)
459+
460+
461+
def test_annotate_message_groups_token_counts_ignore_encrypted_content_in_additional_properties() -> None:
462+
encrypted_blob = "A" * 4096
463+
with_blob = Content.from_text_reasoning(id="rs_1", text="")
464+
with_blob.additional_properties["encrypted_content"] = encrypted_blob
465+
without_blob = Content.from_text_reasoning(id="rs_1", text="")
466+
467+
with_message = Message(role="assistant", contents=[with_blob, Content.from_text("final answer")])
468+
without_message = Message(role="assistant", contents=[without_blob, Content.from_text("final answer")])
469+
470+
annotate_message_groups([with_message], tokenizer=CharacterEstimatorTokenizer())
471+
annotate_message_groups([without_message], tokenizer=CharacterEstimatorTokenizer())
472+
473+
assert _token_count(with_message) == _token_count(without_message)
474+
475+
476+
def test_annotate_message_groups_token_counts_keep_clear_text_reasoning_details() -> None:
477+
import json
478+
479+
clear_text = "visible reasoning " * 128
480+
with_details = Content.from_text_reasoning(
481+
id="rs_1",
482+
text="",
483+
protected_data=json.dumps([{"type": "reasoning.text", "text": clear_text, "encrypted_content": "A" * 4096}]),
484+
)
485+
summary_only = Content.from_text_reasoning(
486+
id="rs_1",
487+
text="",
488+
protected_data=json.dumps([{"type": "reasoning.text", "text": clear_text}]),
489+
)
490+
without_details = Content.from_text_reasoning(id="rs_1", text="")
491+
final_answer = Content.from_text("final answer")
492+
493+
with_message = Message(role="assistant", contents=[with_details, final_answer])
494+
summary_only_message = Message(role="assistant", contents=[summary_only, final_answer])
495+
bare_message = Message(role="assistant", contents=[without_details, final_answer])
496+
497+
annotate_message_groups([with_message], tokenizer=CharacterEstimatorTokenizer())
498+
annotate_message_groups([summary_only_message], tokenizer=CharacterEstimatorTokenizer())
499+
annotate_message_groups([bare_message], tokenizer=CharacterEstimatorTokenizer())
500+
501+
# The opaque encrypted member is excluded while the clear-text reasoning
502+
# the provider replays stays counted.
503+
with_count = _token_count(with_message)
504+
summary_count = _token_count(summary_only_message)
505+
bare_count = _token_count(bare_message)
506+
assert with_count is not None
507+
assert summary_count is not None
508+
assert bare_count is not None
509+
assert with_count == summary_count
510+
assert with_count > bare_count
511+
512+
513+
def test_annotate_token_counts_recomputes_counts_from_stale_serialization_basis() -> None:
514+
515+
from agent_framework._compaction import TOKEN_COUNT_BASIS_VERSION, _write_group_annotation, annotate_token_counts
516+
517+
message = Message(role="assistant", contents=[Content.from_text("final answer " * 64)])
518+
_write_group_annotation(message, group_id="g0", kind="assistant_text", index=0, has_reasoning=False)
519+
annotation = message.additional_properties[GROUP_ANNOTATION_KEY]
520+
assert isinstance(annotation, dict)
521+
# Simulate a count cached by a pre-fix serialization basis: value present,
522+
# no basis stamp (annotations survive to_dict/from_dict round-trips).
523+
annotation[GROUP_TOKEN_COUNT_KEY] = 1
524+
annotation.pop("token_count_basis", None)
525+
assert _token_count(message) == 1
526+
527+
annotate_token_counts([message], tokenizer=CharacterEstimatorTokenizer())
528+
529+
annotation = message.additional_properties[GROUP_ANNOTATION_KEY]
530+
assert isinstance(annotation, dict)
531+
assert annotation[GROUP_TOKEN_COUNT_KEY] != 1
532+
assert annotation["token_count_basis"] == TOKEN_COUNT_BASIS_VERSION
533+
534+
438535
def test_extend_compaction_messages_preserves_existing_annotations_and_tokens() -> None:
536+
from agent_framework._compaction import TOKEN_COUNT_BASIS_VERSION
537+
439538
tokenizer = CharacterEstimatorTokenizer()
440539
messages = [_assistant_function_call("c3")]
441540
annotate_message_groups(messages)
@@ -445,6 +544,10 @@ def test_extend_compaction_messages_preserves_existing_annotations_and_tokens()
445544
annotation = messages[0].additional_properties.get(GROUP_ANNOTATION_KEY)
446545
if isinstance(annotation, dict):
447546
annotation[GROUP_TOKEN_COUNT_KEY] = old_token_count
547+
# Counts cached under the current serialization basis are preserved
548+
# across extensions; counts without the current basis stamp are stale
549+
# and get recomputed (see test_annotate_token_counts_recomputes_...).
550+
annotation["token_count_basis"] = TOKEN_COUNT_BASIS_VERSION
448551

449552
extend_compaction_messages(messages, [_tool_result("c3", "ok")], tokenizer=tokenizer)
450553

0 commit comments

Comments
 (0)