Skip to content

Commit 0b6fcd8

Browse files
authored
fix(sessions): close SQLiteSession connections owned by exited worker threads (#5090)
* fix(memory): close SQLiteSession connections owned by exited worker threads A file-backed session registers one sqlite connection per worker thread so close() can release connections it does not own, but nothing removed an entry when the owning thread exited. The thread-local slot dies with the thread, so the registry became the only reference to a connection nobody could reach, and an application running one event loop per turn accumulated open connections until the process ran out of file descriptors. Record the owning thread for each tracked connection and close the ones whose owner has exited when a new worker registers a connection. * fix(memory): reap retired connections before allocating a new one Move the dead-owner sweep ahead of sqlite3.connect() and _configure_connection() so a worker releases its retired predecessors' handles before allocating its own, and evict a connection from the registries only after closing it so an interrupted close leaves it tracked for later cleanup. Cover the burst case with a regression test: three workers hold at a barrier so they retire together, and the next allocation must find every retired connection already closed while the stored history stays usable.
1 parent 581863a commit 0b6fcd8

2 files changed

Lines changed: 152 additions & 0 deletions

File tree

‎src/agents/memory/sqlite_session.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ def __init__(
5858
self.messages_table = messages_table
5959
self._local = threading.local()
6060
self._connections: set[sqlite3.Connection] = set()
61+
self._connection_owners: dict[sqlite3.Connection, threading.Thread] = {}
6162
self._quarantined_connections: set[sqlite3.Connection] = set()
6263
self._connections_lock = threading.Lock()
6364
self._closed = False
@@ -152,6 +153,7 @@ def _invalidate_connection(self, conn: sqlite3.Connection) -> None:
152153

153154
with self._connections_lock:
154155
self._connections.discard(conn)
156+
self._connection_owners.pop(conn, None)
155157
if close_failed:
156158
self._quarantined_connections.add(conn)
157159
else:
@@ -171,6 +173,10 @@ def _get_connection(self) -> sqlite3.Connection:
171173
else:
172174
# Use thread-local connections for file databases
173175
if not hasattr(self._local, "connection"):
176+
# Release retired workers' connections before opening the replacement, so a
177+
# burst of exits cannot exhaust the descriptors this allocation needs.
178+
with self._connections_lock:
179+
self._close_connections_from_exited_threads()
174180
connection = sqlite3.connect(
175181
str(self.db_path),
176182
check_same_thread=False,
@@ -179,11 +185,27 @@ def _get_connection(self) -> sqlite3.Connection:
179185
self._local.connection = connection
180186
with self._connections_lock:
181187
self._connections.add(connection)
188+
self._connection_owners[connection] = threading.current_thread()
182189
assert isinstance(self._local.connection, sqlite3.Connection), (
183190
f"Expected sqlite3.Connection, got {type(self._local.connection)}"
184191
)
185192
return self._local.connection
186193

194+
def _close_connections_from_exited_threads(self) -> None:
195+
"""Close tracked connections whose owning worker thread has exited."""
196+
# Callers hold _connections_lock. A worker's thread-local connection is
197+
# unreachable once its thread is gone, so this registry is the only reference.
198+
for conn, owner in list(self._connection_owners.items()):
199+
if owner.is_alive():
200+
continue
201+
try:
202+
conn.close()
203+
except Exception:
204+
self._quarantined_connections.add(conn)
205+
# Evicted after the close, so an interrupt cannot drop an open connection.
206+
del self._connection_owners[conn]
207+
self._connections.discard(conn)
208+
187209
@staticmethod
188210
def _configure_connection(conn: sqlite3.Connection) -> None:
189211
"""Enable WAL, retrying its transient cross-process initialization lock."""
@@ -541,6 +563,7 @@ def close(self) -> None:
541563
del self._local.connection
542564

543565
with self._connections_lock:
566+
self._connection_owners.clear()
544567
has_unclosed_connections = bool(self._quarantined_connections)
545568
if not has_unclosed_connections and self._lock_path is not None:
546569
with self._connections_lock:

‎tests/memory/test_session.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,135 @@ async def test_sqlite_session_close_closes_worker_thread_connections():
269269
connections[0].execute("SELECT 1")
270270

271271

272+
def test_sqlite_session_reaps_connections_from_exited_worker_threads():
273+
"""Test that a session closes connections owned by exited worker threads."""
274+
with tempfile.TemporaryDirectory() as temp_dir:
275+
db_path = Path(temp_dir) / "exited_worker_threads.db"
276+
session = SQLiteSession("exited_worker_threads", db_path)
277+
278+
try:
279+
# Each asyncio.run shuts down its default executor, so the worker that served
280+
# the session exits and its connection is unreachable from the next turn on.
281+
asyncio.run(session.add_items([{"role": "user", "content": "turn 0"}]))
282+
(abandoned,) = session._connections
283+
284+
for turn in range(1, 5):
285+
asyncio.run(session.add_items([{"role": "user", "content": f"turn {turn}"}]))
286+
assert len(session._connections) == 1
287+
288+
assert abandoned not in session._connections
289+
with pytest.raises(sqlite3.ProgrammingError):
290+
abandoned.execute("SELECT 1")
291+
292+
items = asyncio.run(session.get_items())
293+
assert [item["content"] for item in items] == [f"turn {turn}" for turn in range(5)]
294+
finally:
295+
session.close()
296+
297+
298+
def test_sqlite_session_closes_retired_connections_before_the_next_allocation(monkeypatch):
299+
"""Test that connections left by a burst of retired workers close before the next one opens."""
300+
with tempfile.TemporaryDirectory() as temp_dir:
301+
db_path = Path(temp_dir) / "retired_workers.db"
302+
session = SQLiteSession("retired_workers", db_path)
303+
# Each worker holds at the barrier until all of them have registered a
304+
# connection, so they retire together instead of one at a time.
305+
registered = threading.Barrier(3)
306+
307+
def serve_one_turn(index: int) -> None:
308+
async def turn() -> None:
309+
await session.add_items([{"role": "user", "content": f"worker {index}"}])
310+
await asyncio.to_thread(registered.wait)
311+
312+
# asyncio.run waits for its executor to shut down, so this worker has
313+
# exited by the time the thread is joined.
314+
asyncio.run(turn())
315+
316+
workers = [threading.Thread(target=serve_one_turn, args=(index,)) for index in range(3)]
317+
try:
318+
for worker in workers:
319+
worker.start()
320+
for worker in workers:
321+
worker.join(timeout=5)
322+
assert not worker.is_alive()
323+
324+
retired = list(session._connections)
325+
assert len(retired) == 3
326+
327+
def still_open() -> int:
328+
open_count = 0
329+
for connection in retired:
330+
try:
331+
connection.execute("SELECT 1")
332+
except sqlite3.ProgrammingError:
333+
continue
334+
open_count += 1
335+
return open_count
336+
337+
retired_open_at_allocation: list[int] = []
338+
connect = sqlite3.connect
339+
340+
def recording_connect(*args: Any, **kwargs: Any) -> sqlite3.Connection:
341+
retired_open_at_allocation.append(still_open())
342+
return connect(*args, **kwargs)
343+
344+
with monkeypatch.context() as allocation:
345+
allocation.setattr(sqlite3, "connect", recording_connect)
346+
asyncio.run(session.add_items([{"role": "user", "content": "after they retired"}]))
347+
348+
# The replacement connection must not be allocated while the retired ones
349+
# still hold their descriptors.
350+
assert retired_open_at_allocation == [0]
351+
assert len(session._connections) == 1
352+
353+
# The workers write concurrently, so only the final turn has a fixed position.
354+
contents = [item["content"] for item in asyncio.run(session.get_items())]
355+
assert sorted(contents[:-1]) == ["worker 0", "worker 1", "worker 2"]
356+
assert contents[-1] == "after they retired"
357+
finally:
358+
registered.abort()
359+
for worker in workers:
360+
worker.join(timeout=5)
361+
session.close()
362+
363+
364+
@pytest.mark.asyncio
365+
async def test_sqlite_session_keeps_connections_of_live_worker_threads():
366+
"""Test that connections owned by running worker threads are never closed."""
367+
with tempfile.TemporaryDirectory() as temp_dir:
368+
db_path = Path(temp_dir) / "live_worker_threads.db"
369+
session = SQLiteSession("live_worker_threads", db_path)
370+
# A second running loop has its own default executor, so its worker is a
371+
# different thread that stays alive for as long as that loop does.
372+
other_loop = asyncio.new_event_loop()
373+
other_thread = threading.Thread(target=other_loop.run_forever)
374+
other_thread.start()
375+
376+
try:
377+
asyncio.run_coroutine_threadsafe(
378+
session.add_items([{"role": "user", "content": "from the other loop"}]),
379+
other_loop,
380+
).result(timeout=5)
381+
(held,) = session._connections
382+
383+
# Registering from this loop's worker sweeps while the other worker lives.
384+
await session.add_items([{"role": "user", "content": "from this loop"}])
385+
386+
assert held in session._connections
387+
held.execute("SELECT 1").fetchone()
388+
389+
items = await session.get_items()
390+
assert [item["content"] for item in items] == [
391+
"from the other loop",
392+
"from this loop",
393+
]
394+
finally:
395+
other_loop.call_soon_threadsafe(other_loop.stop)
396+
other_thread.join(timeout=5)
397+
other_loop.close()
398+
session.close()
399+
400+
272401
@pytest.mark.asyncio
273402
async def test_sqlite_session_closed_rejects_empty_add_items():
274403
"""add_items([]) must not bypass the closed check through the empty-list fast path."""

0 commit comments

Comments
 (0)