Skip to content

Commit 8e338e5

Browse files
authored
fix(extensions): keep SQLite usage capture off the event loop (#4981)
1 parent 465860b commit 8e338e5

2 files changed

Lines changed: 665 additions & 40 deletions

File tree

‎src/agents/extensions/memory/advanced_sqlite_session.py‎

Lines changed: 101 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@
55
import logging
66
import re
77
import sqlite3
8+
import threading
89
import time
10+
from collections.abc import Callable, Coroutine
11+
from concurrent.futures import Future
912
from contextlib import closing
1013
from pathlib import Path
11-
from typing import Any, ClassVar, cast
14+
from typing import Any, ClassVar, TypeVar, cast
1215

1316
from agents.result import RunResult
1417
from agents.usage import Usage
@@ -25,6 +28,8 @@
2528
from ...memory.session_settings import SessionSettings, resolve_session_limit
2629
from ...memory.sqlite_session import _await_mutation
2730

31+
_T = TypeVar("_T")
32+
2833

2934
def _allow_all_sqlite_actions(
3035
_action: int,
@@ -73,6 +78,13 @@ def __init__(
7378
logger: The logger to use. Defaults to the module logger
7479
**kwargs: Additional keyword arguments to pass to the superclass
7580
""" # noqa: E501
81+
# Publish branch and generation together so caller-thread snapshots cannot
82+
# combine a cleared branch with the new generation from a worker update.
83+
self._branch_state: tuple[str, int] = ("main", 0)
84+
# Order history mutations and usage capture before dispatching workers.
85+
# The bookkeeping lock never protects SQLite work or an await.
86+
self._turn_operation_lock = threading.Lock()
87+
self._turn_operation_tail: Future[None] | None = None
7688
self._create_structure_tables_on_init = create_tables
7789
try:
7890
super().__init__(
@@ -87,13 +99,16 @@ def __init__(
8799
except BaseException:
88100
pass
89101
raise
90-
self._current_branch_id = "main"
91-
# Synchronized with the durable session_clear_generations row whenever a
92-
# branch pointer is established or a write begins. A mismatch means
93-
# another instance cleared the session, so the local pointer resets to main.
94-
self._generation = 0
95102
self._logger = logger if logger is not None else logging.getLogger(__name__)
96103

104+
@property
105+
def _current_branch_id(self) -> str:
106+
return self._branch_state[0]
107+
108+
@property
109+
def _generation(self) -> int:
110+
return self._branch_state[1]
111+
97112
def _init_db_for_connection(self, conn: sqlite3.Connection) -> None:
98113
"""Initialize base tables only after validating advanced-table ownership."""
99114
if self._create_structure_tables_on_init:
@@ -104,6 +119,7 @@ def _init_db_for_connection(self, conn: sqlite3.Connection) -> None:
104119
self._claim_structure_tables(conn)
105120
self._create_schema_for_connection(conn)
106121
conn.commit()
122+
self._refresh_branch_after_external_clear(conn, initialize=False)
107123

108124
def _commit_branch_pointer(self, branch_id: str, generation: int) -> bool:
109125
"""Set the current-branch pointer unless a clear has committed meanwhile.
@@ -123,11 +139,9 @@ def _commit_branch_pointer(self, branch_id: str, generation: int) -> bool:
123139
).fetchone()
124140
durable_generation = row[0] if row is not None else 0
125141
if durable_generation != generation:
126-
self._generation = durable_generation
127-
self._current_branch_id = "main"
142+
self._branch_state = ("main", durable_generation)
128143
return False
129-
self._generation = durable_generation
130-
self._current_branch_id = branch_id
144+
self._branch_state = (branch_id, durable_generation)
131145
return True
132146

133147
# The structure tables that record which base-table pair owns a database file, and the
@@ -344,6 +358,33 @@ def _init_structure_tables(self, conn: sqlite3.Connection) -> None:
344358
ON turn_usage(session_id, branch_id, user_turn_number)
345359
""")
346360

361+
def _run_ordered_turn_operation(
362+
self, worker: Callable[..., _T], *args: Any
363+
) -> Coroutine[Any, Any, _T]:
364+
"""Reserve history/capture order before the caller first yields.
365+
366+
Completion futures are independent of any event loop, like the session's
367+
thread-local database connections. A failed operation still releases its
368+
successor; each caller owns cancellation settlement via _await_mutation.
369+
"""
370+
completion: Future[None] = Future()
371+
with self._turn_operation_lock:
372+
predecessor = self._turn_operation_tail
373+
self._turn_operation_tail = completion
374+
375+
async def _run() -> _T:
376+
try:
377+
if predecessor is not None:
378+
await asyncio.wrap_future(predecessor)
379+
return await asyncio.to_thread(worker, *args)
380+
finally:
381+
completion.set_result(None)
382+
with self._turn_operation_lock:
383+
if self._turn_operation_tail is completion:
384+
self._turn_operation_tail = None
385+
386+
return _run()
387+
347388
async def add_items(self, items: list[TResponseInputItem]) -> None:
348389
"""Add items to the session.
349390
@@ -356,17 +397,24 @@ async def add_items(self, items: list[TResponseInputItem]) -> None:
356397
if not items:
357398
return
358399

400+
branch_id, generation = self._branch_state
401+
359402
def _add_items_sync():
360403
"""Synchronous helper to add items and structure metadata together."""
361404
with self._write_connection() as conn:
362405
self._refresh_branch_after_external_clear(conn)
406+
# Match pop_item's reset behavior after a clear, while keeping a
407+
# later branch switch from redirecting an already queued append.
408+
target_branch = (
409+
self._current_branch_id if self._generation != generation else branch_id
410+
)
363411
# Keep both writes in one transaction so metadata failures do not leave orphans.
364412
self._insert_items(conn, items)
365-
self._insert_structure_metadata(conn, items)
413+
self._insert_structure_metadata(conn, items, target_branch)
366414
conn.commit()
367415

368416
try:
369-
await _await_mutation(asyncio.to_thread(_add_items_sync))
417+
await _await_mutation(self._run_ordered_turn_operation(_add_items_sync))
370418
except Exception as exc:
371419
log_model_and_tool_action_error(self._logger, "Failed to add session items", exc)
372420
raise
@@ -473,8 +521,7 @@ async def pop_item(self) -> TResponseInputItem | None:
473521
# Snapshot the current branch at call time so a concurrent
474522
# switch_to_branch() cannot redirect this pop to a different branch once
475523
# it has been dispatched to the worker thread.
476-
branch_id = self._current_branch_id
477-
generation = self._generation
524+
branch_id, generation = self._branch_state
478525

479526
def _pop_item_sync():
480527
with self._write_connection() as conn:
@@ -551,7 +598,7 @@ def _pop_item_sync():
551598
# Drop corrupted JSON entries and keep looking for a valid item.
552599
continue
553600

554-
return await _await_mutation(asyncio.to_thread(_pop_item_sync))
601+
return await _await_mutation(self._run_ordered_turn_operation(_pop_item_sync))
555602

556603
async def clear_session(self) -> None:
557604
"""Clear all items for this session.
@@ -611,8 +658,7 @@ def _clear_session_sync():
611658
# the pointer still references a deleted branch. Bumping the
612659
# generation invalidates any in-flight switch/create that
613660
# captured the pre-clear generation.
614-
self._generation = generation
615-
self._current_branch_id = "main"
661+
self._branch_state = ("main", generation)
616662

617663
await _await_mutation(asyncio.to_thread(_clear_session_sync))
618664

@@ -621,6 +667,8 @@ async def store_run_usage(self, result: RunResult) -> None:
621667
622668
This is designed to be called after `Runner.run()` completes.
623669
Session-level usage can be aggregated from turn data when needed.
670+
Appends and pops started later on this session instance cannot change
671+
which turn this call captures.
624672
625673
Args:
626674
result: The result from the run
@@ -635,14 +683,24 @@ async def store_run_usage(self, result: RunResult) -> None:
635683
# write is skipped. The anchor is scoped to this branch/turn, so
636684
# unrelated removals (e.g. delete_branch on another branch) do
637685
# not drop this write.
638-
current_turn, branch_id, turn_anchor = self._capture_current_turn()
639-
# Only update turn-level usage - session usage is aggregated on demand
640-
await self._update_turn_usage_internal(
641-
current_turn,
642-
result.context_wrapper.usage,
643-
branch_id=branch_id,
644-
turn_anchor=turn_anchor,
686+
branch_id, generation = self._branch_state
687+
usage = result.context_wrapper.usage
688+
capture = self._run_ordered_turn_operation(
689+
self._capture_current_turn, branch_id, generation
645690
)
691+
692+
async def _store_usage() -> None:
693+
current_turn, captured_branch, turn_anchor = await capture
694+
# Only update turn-level usage; session usage is aggregated on demand.
695+
await self._update_turn_usage_internal(
696+
current_turn,
697+
usage,
698+
branch_id=captured_branch,
699+
turn_anchor=turn_anchor,
700+
)
701+
702+
# Capture and write must both settle before caller cancellation propagates.
703+
await _await_mutation(_store_usage())
646704
except Exception as e:
647705

648706
def diagnostic_extra() -> dict[str, object]:
@@ -655,17 +713,20 @@ def diagnostic_extra() -> dict[str, object]:
655713
diagnostic_extra=diagnostic_extra,
656714
)
657715

658-
def _capture_current_turn(self) -> tuple[int, str, int | None]:
716+
def _capture_current_turn(self, branch_id: str, generation: int) -> tuple[int, str, int | None]:
659717
"""Return (current_turn, branch_id, turn_anchor) in one locked read.
660718
661719
``turn_anchor`` is the smallest ``message_structure.id`` of the current
662-
turn on the current branch (``None`` if the turn has no rows). Because
720+
turn on the captured branch (``None`` if the turn has no rows). Because
663721
ids are monotonic and never reused, it uniquely identifies this turn
664722
incarnation, so a later pop+recreate that reuses the numeric turn id
665723
yields a different anchor.
666724
"""
667725
with self._locked_connection() as conn:
668-
branch_id = self._resolve_read_branch(conn, None)
726+
self._refresh_branch_after_external_clear(conn, initialize=False)
727+
if self._generation != generation:
728+
# A clear invalidates usage belonging to the previous session history.
729+
return 0, branch_id, None
669730
with closing(conn.cursor()) as cursor:
670731
cursor.execute(
671732
"""
@@ -792,7 +853,9 @@ def _insert_structure_metadata(
792853
self,
793854
conn: sqlite3.Connection,
794855
items: list[TResponseInputItem],
856+
branch_id: str | None = None,
795857
) -> None:
858+
target_branch = self._current_branch_id if branch_id is None else branch_id
796859
# Get the IDs of messages we just inserted, in order.
797860
with closing(conn.cursor()) as cursor:
798861
cursor.execute(
@@ -830,7 +893,7 @@ def _insert_structure_metadata(
830893
FROM message_structure
831894
WHERE session_id = ? AND branch_id = ?
832895
""",
833-
(self.session_id, self._current_branch_id),
896+
(self.session_id, target_branch),
834897
)
835898
result = cursor.fetchone()
836899
current_turn = result[0] if result else 0
@@ -856,7 +919,7 @@ def _insert_structure_metadata(
856919
(
857920
self.session_id,
858921
msg_id,
859-
self._current_branch_id,
922+
target_branch,
860923
msg_type,
861924
seq_start + i + 1,
862925
item_turn,
@@ -1212,7 +1275,9 @@ def _delete_sync():
12121275
return usage_deleted, structure_deleted, orphaned_messages_deleted
12131276

12141277
usage_deleted, structure_deleted, orphaned_messages_deleted = await _await_mutation(
1215-
asyncio.to_thread(_delete_sync)
1278+
# Earlier queued appends must settle before deletion, or their captured
1279+
# branch target could recreate rows after this deletion commits.
1280+
self._run_ordered_turn_operation(_delete_sync)
12161281
)
12171282

12181283
self._logger.info(
@@ -1349,8 +1414,7 @@ def _refresh_branch_after_external_clear(
13491414
).fetchone()
13501415
generation = row[0] if row is not None else 0
13511416
if generation != self._generation:
1352-
self._generation = generation
1353-
self._current_branch_id = "main"
1417+
self._branch_state = ("main", generation)
13541418

13551419
def _resolve_read_branch(
13561420
self,
@@ -1420,8 +1484,7 @@ def _copy_sync() -> tuple[str, Any, str, int]:
14201484
conn.execute("BEGIN IMMEDIATE")
14211485
self._ensure_branch_reservations_table(conn)
14221486
self._refresh_branch_after_external_clear(conn)
1423-
source_branch_id = self._current_branch_id
1424-
generation = self._generation
1487+
source_branch_id, generation = self._branch_state
14251488
with closing(conn.cursor()) as cursor:
14261489
cursor.execute(
14271490
f"""
@@ -1927,15 +1990,15 @@ async def _update_turn_usage_internal(
19271990
turn reused the same numeric id. Because the check is scoped to
19281991
this branch/turn, unrelated removals (e.g. delete_branch on
19291992
another branch) do not drop this write. ``None`` means the branch
1930-
had no turn when it was read, so there is nothing to attribute
1931-
the usage to and the write is skipped.
1993+
had no turn when it was read or a session clear invalidated the
1994+
pending capture, so the write is skipped.
19321995
"""
19331996

19341997
target_branch = branch_id if branch_id is not None else self._current_branch_id
19351998

19361999
if turn_anchor is None:
1937-
# ``_capture_current_turn`` returns no anchor only when the branch has no
1938-
# turn rows; recording usage would invent a phantom turn 0.
2000+
# A missing anchor means there is no captured turn or a session clear
2001+
# invalidated the capture; neither case can safely receive usage.
19392002
self._logger.debug("Skipping usage store: no current turn on branch %r", target_branch)
19402003
return
19412004

0 commit comments

Comments
 (0)