Skip to content

Commit 8e5fb9c

Browse files
fix: count the tokens billed by responses.compact
run_compaction makes a real billed model call and read only compacted.output, so compacted.usage was discarded and those tokens never reached the run's totals. On a long conversation the compaction pass is often the largest call in the run, so the reported total can be a fraction of what was billed, and requests undercounts by one per pass. The session now accumulates compaction usage in a public compaction_usage attribute. Accounting is written defensively: a missing or unexpected usage shape is logged and skipped rather than raised, because usage bookkeeping must never break a run. Fixes #4434 Claude-Session: https://claude.ai/code/session_01KJrBqn4AgyHTazfdUuXbuA
1 parent 9aba900 commit 8e5fb9c

2 files changed

Lines changed: 151 additions & 0 deletions

File tree

‎src/agents/memory/openai_responses_compaction_session.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
from ..logger import log_model_and_tool_action_warning
1212
from ..models._openai_shared import get_default_openai_client
1313
from ..run_internal.items import normalize_input_items_for_api
14+
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
15+
16+
from ..usage import Usage
1417
from .openai_conversations_session import OpenAIConversationsSession
1518
from .session import (
1619
OpenAIResponsesCompactionArgs,
@@ -122,6 +125,13 @@ def __init__(
122125

123126
self.session_id = session_id
124127
self.underlying_session = underlying_session
128+
self.compaction_usage = Usage()
129+
"""Tokens billed by responses.compact calls made by this session.
130+
131+
Compaction is a real model call, so it is billed, but it happens outside the
132+
agent run loop and therefore never reaches ``RunResult.context_wrapper.usage``.
133+
This accumulates it so the cost is visible rather than silently dropped.
134+
"""
125135
self._client = client
126136
self.model = model
127137
self.compaction_mode = compaction_mode
@@ -226,6 +236,8 @@ async def run_compaction(self, args: OpenAIResponsesCompactionArgs | None = None
226236

227237
compacted = await self.client.responses.compact(**compact_kwargs)
228238

239+
self._record_compaction_usage(getattr(compacted, "usage", None))
240+
229241
output_items = _strip_orphaned_assistant_ids(
230242
_normalize_compaction_output_items(compacted.output or [])
231243
)
@@ -247,6 +259,52 @@ async def run_compaction(self, args: OpenAIResponsesCompactionArgs | None = None
247259
len(self._compaction_candidate_items or []),
248260
)
249261

262+
def _record_compaction_usage(self, compacted_usage: Any) -> None:
263+
"""Accumulate the tokens a compaction pass billed.
264+
265+
Accounting must never break a run, so anything unexpected on the usage
266+
object is logged and skipped rather than raised.
267+
"""
268+
if compacted_usage is None:
269+
return
270+
271+
def _count(name: str) -> int | None:
272+
value = getattr(compacted_usage, name, None)
273+
if value is None:
274+
return 0
275+
return value if isinstance(value, int) and not isinstance(value, bool) else None
276+
277+
input_tokens = _count("input_tokens")
278+
output_tokens = _count("output_tokens")
279+
total_tokens = _count("total_tokens")
280+
if input_tokens is None or output_tokens is None or total_tokens is None:
281+
logger.debug(
282+
"compact: skipping usage for %s, unexpected token fields", self._response_id
283+
)
284+
return
285+
286+
usage = Usage(
287+
requests=1,
288+
input_tokens=input_tokens,
289+
output_tokens=output_tokens,
290+
total_tokens=total_tokens or input_tokens + output_tokens,
291+
)
292+
input_details = getattr(compacted_usage, "input_tokens_details", None)
293+
if isinstance(input_details, InputTokensDetails):
294+
usage.input_tokens_details = input_details
295+
output_details = getattr(compacted_usage, "output_tokens_details", None)
296+
if isinstance(output_details, OutputTokensDetails):
297+
usage.output_tokens_details = output_details
298+
299+
self.compaction_usage.add(usage)
300+
logger.debug(
301+
"compact: usage for %s (input=%s, output=%s, total=%s)",
302+
self._response_id,
303+
input_tokens,
304+
output_tokens,
305+
usage.total_tokens,
306+
)
307+
250308
async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]:
251309
return await self.underlying_session.get_items(limit)
252310

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
"""Compaction is a billed model call, so its tokens must not be discarded."""
2+
3+
from __future__ import annotations
4+
5+
from unittest.mock import AsyncMock, MagicMock
6+
7+
import pytest
8+
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
9+
10+
from agents.memory.openai_responses_compaction_session import (
11+
OpenAIResponsesCompactionSession,
12+
)
13+
from agents.memory.session import Session
14+
15+
16+
def _compact_response(input_tokens: int, output_tokens: int) -> MagicMock:
17+
response = MagicMock()
18+
response.output = []
19+
response.usage = MagicMock()
20+
response.usage.input_tokens = input_tokens
21+
response.usage.output_tokens = output_tokens
22+
response.usage.total_tokens = input_tokens + output_tokens
23+
response.usage.input_tokens_details = InputTokensDetails(cached_tokens=0, cache_write_tokens=0)
24+
response.usage.output_tokens_details = OutputTokensDetails(reasoning_tokens=0)
25+
return response
26+
27+
28+
def _session(client: MagicMock) -> OpenAIResponsesCompactionSession:
29+
return OpenAIResponsesCompactionSession(
30+
session_id="s1",
31+
underlying_session=MagicMock(spec=Session),
32+
client=client,
33+
model="gpt-4.1",
34+
should_trigger_compaction=lambda _args: True,
35+
)
36+
37+
38+
def test_compaction_usage_starts_empty() -> None:
39+
session = _session(MagicMock())
40+
assert session.compaction_usage.total_tokens == 0
41+
assert session.compaction_usage.requests == 0
42+
43+
44+
@pytest.mark.asyncio
45+
async def test_compaction_usage_is_recorded() -> None:
46+
client = MagicMock()
47+
client.responses.compact = AsyncMock(return_value=_compact_response(150_000, 42_000))
48+
session = _session(client)
49+
session._response_id = "resp_1"
50+
session._session_items = []
51+
session._compaction_candidate_items = []
52+
53+
await session.run_compaction()
54+
55+
assert session.compaction_usage.requests == 1
56+
assert session.compaction_usage.input_tokens == 150_000
57+
assert session.compaction_usage.output_tokens == 42_000
58+
assert session.compaction_usage.total_tokens == 192_000
59+
60+
61+
@pytest.mark.asyncio
62+
async def test_compaction_usage_accumulates_across_passes() -> None:
63+
client = MagicMock()
64+
client.responses.compact = AsyncMock(
65+
side_effect=[_compact_response(100, 10), _compact_response(200, 20)]
66+
)
67+
session = _session(client)
68+
session._response_id = "resp_1"
69+
session._session_items = []
70+
session._compaction_candidate_items = []
71+
72+
await session.run_compaction()
73+
await session.run_compaction()
74+
75+
assert session.compaction_usage.requests == 2
76+
assert session.compaction_usage.total_tokens == 330
77+
78+
79+
@pytest.mark.asyncio
80+
async def test_missing_usage_is_tolerated() -> None:
81+
response = MagicMock()
82+
response.output = []
83+
response.usage = None
84+
client = MagicMock()
85+
client.responses.compact = AsyncMock(return_value=response)
86+
session = _session(client)
87+
session._response_id = "resp_1"
88+
session._session_items = []
89+
session._compaction_candidate_items = []
90+
91+
await session.run_compaction()
92+
93+
assert session.compaction_usage.total_tokens == 0

0 commit comments

Comments
 (0)