2929
3030import base64
3131import json
32+ import time
3233from typing import Any , Literal , TypeGuard , cast
3334
3435from cryptography .fernet import Fernet , InvalidToken
4041from ...memory .openai_responses_compaction_session import OpenAIResponsesCompactionSession
4142from ...memory .session import (
4243 OpenAIResponsesCompactionArgs ,
44+ Session ,
4345 SessionABC ,
4446 _call_session_method ,
47+ _CompactionSnapshot ,
4548 _get_session_wrapper ,
4649)
4750from ...memory .session_settings import SessionSettings , resolve_session_limit
@@ -116,6 +119,8 @@ class EncryptedSession(SessionABC):
116119 the cumulative number of items retrieved and unwrapped per ``get_items`` call;
117120 overlapping backfill windows count again. This does not bound item byte size,
118121 backend-internal work, elapsed time, or ``pop_item`` work.
122+ Successful automatic compaction on native SQLite and SQLAlchemy stores reclaims a
123+ contiguous prefix of authenticated expired envelopes in bounded batches.
119124
120125 Note: Expired tokens are rejected based on the system clock of the application server.
121126 To avoid valid tokens being rejected due to clock drift, ensure all servers in
@@ -159,6 +164,9 @@ def __init__(
159164 self .cipher = _derive_session_fernet_key (master , session_id )
160165 self ._kid = "hkdf-v1"
161166 self ._ver = 1
167+ # One-use read budget for envelopes omitted by the latest history read.
168+ # This is not evidence of completeness or permission to replace history.
169+ self ._compaction_read_overhead = 0
162170
163171 def __getattr__ (self , name : str ) -> Any :
164172 # Expose compaction only when the underlying session actually supports it.
@@ -179,10 +187,9 @@ async def _run_compaction(
179187 await session ._run_compaction (
180188 args ,
181189 wrapper = wrapper ,
182- read_items = lambda : _call_session_method (
183- self .get_items , wrapper = _get_session_wrapper (self , wrapper )
184- ),
190+ read_items = lambda limit : self ._read_compaction_items (limit , wrapper = wrapper ),
185191 prepare_items = self ._encrypt_items ,
192+ read_snapshot = self ._get_compaction_snapshot ,
186193 )
187194
188195 async def _defer_encrypted_compaction (
@@ -196,10 +203,83 @@ async def _defer_encrypted_compaction(
196203 await session ._defer_compaction (
197204 response_id ,
198205 store ,
199- read_items = lambda : _call_session_method (
206+ read_items = lambda limit : self ._read_compaction_items (limit , wrapper = wrapper ),
207+ )
208+
209+ async def _get_compaction_snapshot (self , limit : int ) -> _CompactionSnapshot | None :
210+ backend : Session = self .underlying_session
211+ if isinstance (backend , OpenAIResponsesCompactionSession ):
212+ backend = backend .underlying_session
213+ reader = getattr (backend , "_get_compaction_snapshot" , None )
214+ if reader is None :
215+ return None
216+ # Authenticate timestamps before authorizing deletion; InvalidToken alone
217+ # can also mean a wrong key, corruption or a future timestamp.
218+ expired_before = int (time .time ()) - self .ttl
219+ cipher = self .cipher
220+
221+ def is_expired (item : TResponseInputItem ) -> bool :
222+ if not _is_encrypted_envelope (item ):
223+ return False
224+ try :
225+ return cipher .extract_timestamp (item ["payload" ]) < expired_before
226+ except (InvalidToken , TypeError ):
227+ return False
228+
229+ raw : _CompactionSnapshot | None = await reader (limit , prune_prefix = is_expired )
230+ if raw is None :
231+ return None
232+ items : list [TResponseInputItem ] = []
233+ positions : list [int ] = []
234+ retained_boundary = 0
235+ for index , encrypted_item in enumerate (raw .items ):
236+ item = self ._unwrap (encrypted_item )
237+ if item is not None :
238+ items .append (item )
239+ positions .append (index )
240+ elif not is_expired (encrypted_item ):
241+ # An unreadable envelope is not necessarily expired. Keep it and
242+ # everything before it outside the eligible replacement suffix.
243+ retained_boundary = index + 1
244+ items .clear ()
245+ positions .clear ()
246+
247+ async def replace_suffix (start : int , output : list [TResponseInputItem ]) -> bool :
248+ # Observed expired rows may be part of the selected raw suffix.
249+ # The transaction separately reclaims the authenticated expired prefix.
250+ raw_start = retained_boundary if start == 0 else positions [start ]
251+ return await raw .replace_suffix (raw_start , self ._encrypt_items (output ))
252+
253+ return _CompactionSnapshot (items , raw .complete and retained_boundary == 0 , replace_suffix )
254+
255+ async def _read_compaction_items (
256+ self , limit : int | None , * , wrapper : RunContextWrapper [Any ] | None = None
257+ ) -> tuple [list [TResponseInputItem ], bool ]:
258+ if limit is None :
259+ # A default-limited policy read must not discard overhead already
260+ # observed by a larger model-history read.
261+ overhead = self ._compaction_read_overhead
262+ items = await _call_session_method (
200263 self .get_items , wrapper = _get_session_wrapper (self , wrapper )
264+ )
265+ self ._compaction_read_overhead = max (overhead , self ._compaction_read_overhead )
266+ return items , False
267+ # Reuse only overhead already observed during ordinary history retrieval.
268+ # Consume it before I/O; current raw completeness and plaintext visibility
269+ # still decide whether compaction can replace the stored history.
270+ raw_limit = limit + self ._compaction_read_overhead
271+ self ._compaction_read_overhead = 0
272+ items = cast (
273+ list [TResponseInputItem ],
274+ await _call_session_method (
275+ self .underlying_session .get_items ,
276+ raw_limit ,
277+ wrapper = _get_session_wrapper (self .underlying_session , wrapper ),
201278 ),
202279 )
280+ if len (items ) >= raw_limit :
281+ return [], False
282+ return self ._unwrap_valid_items (items ), True
203283
204284 @property
205285 def session_settings (self ) -> SessionSettings | None :
@@ -251,6 +331,7 @@ async def get_items(
251331 * ,
252332 wrapper : RunContextWrapper [Any ] | None = None ,
253333 ) -> list [TResponseInputItem ]:
334+ self ._compaction_read_overhead = 0
254335 wrapper = _get_session_wrapper (self .underlying_session , wrapper )
255336 effective_limit = resolve_session_limit (limit , self .session_settings )
256337 remaining = self .max_scan_items
@@ -272,6 +353,7 @@ async def get_items(
272353 ),
273354 )
274355 valid_items = self ._unwrap_valid_items (encrypted_items )
356+ self ._compaction_read_overhead = len (encrypted_items ) - len (valid_items )
275357 if (
276358 positive_limit
277359 and effective_limit is not None
@@ -300,7 +382,9 @@ async def get_items(
300382 wrapper = wrapper ,
301383 ),
302384 )
303- return self ._unwrap_valid_items (encrypted_items )
385+ valid_items = self ._unwrap_valid_items (encrypted_items )
386+ self ._compaction_read_overhead = len (encrypted_items ) - len (valid_items )
387+ return valid_items
304388
305389 def _encrypt_items (self , items : list [TResponseInputItem ]) -> list [TResponseInputItem ]:
306390 return cast (list [TResponseInputItem ], [self ._wrap (item ) for item in items ])
0 commit comments