Skip to content

Commit 221be76

Browse files
MaxFreedomPollardteknium1
authored andcommitted
fix(sessions): briefly wait out a live compression lock instead of killing the turn
append_message refused immediately when another writer held the session's compression lock. The conversation loop turns that into session_persistence_failed and tells the operator to check disk space and permissions, when the store is healthy and merely busy. The two append attempts in the reported incident were 3ms apart, so there was no wait at all before the turn was destroyed (#75083). The wait is deliberately short (_COMPRESSION_BUSY_WAIT_S, 5s) rather than the 60s transcript write patience. The lease is a correctness boundary, not just a busy signal: test_compression_lease_blocks_non_owner_but_allows_owner_flush pins that a late stale turn must not land in a session being compressed. Reusing the full write patience made that append succeed once the lease aged out, which is exactly what the guard exists to prevent. A short budget saves the common case, where compression publishes in a couple of seconds, and still refuses a writer locked out by a long-running or wedged compression. CompressionSessionBusyError could not simply be retried either: it covers two conditions with opposite handling. A compressor discovering its own lease is gone is permanent, and retrying that would spend the whole budget before failing anyway. Split the transient case into a SessionCompressionInProgressError subclass, raised only by append_message, and wait on just that. Existing except CompressionSessionBusyError handlers catch both unchanged. The retry jitter is extracted into _sleep_before_write_retry so the lock path and the compression path share one implementation.
1 parent eeaba3a commit 221be76

2 files changed

Lines changed: 208 additions & 16 deletions

File tree

‎hermes_state.py‎

Lines changed: 79 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1529,6 +1529,22 @@ class CompressionSessionBusyError(RuntimeError):
15291529
"""A non-owner tried to write while compression owns the session."""
15301530

15311531

1532+
class SessionCompressionInProgressError(CompressionSessionBusyError):
1533+
"""A concurrent writer collided with a *live* compression lock.
1534+
1535+
Split out from :class:`CompressionSessionBusyError` because the two
1536+
conditions that class covers need opposite handling. This one is
1537+
transient: a healthy compressor holds the session for a few seconds and
1538+
the lock row carries its own ``expires_at``, so the write can simply wait
1539+
(see ``_execute_write``'s patience loop). The other case, a compressor
1540+
discovering its own lease is gone, is permanent and must fail fast rather
1541+
than spin out the whole patience budget.
1542+
1543+
Subclassing keeps every existing ``except CompressionSessionBusyError``
1544+
handler working unchanged.
1545+
"""
1546+
1547+
15321548
def _connect_tracked_db(path, tracking_path=None, **kwargs):
15331549
"""``sqlite3.connect`` that registers the open fd for lock-safety.
15341550
@@ -1756,6 +1772,15 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin)
17561772
# attempts.
17571773
_WRITE_PATIENCE_S = 20.0
17581774
_TRANSCRIPT_WRITE_PATIENCE_S = 60.0
1775+
# A live compression lock gets its own, much shorter budget than the write
1776+
# lock. Compression publishes in a couple of seconds, so a brief wait saves
1777+
# the overwhelming majority of concurrent turns (#75083). It deliberately
1778+
# stays short: the lease is a correctness boundary, not just a busy signal
1779+
# (see test_compression_lease_blocks_non_owner_but_allows_owner_flush), so
1780+
# a writer that is still locked out after this budget must still be
1781+
# refused rather than allowed to land a stale turn in a session whose
1782+
# compression is genuinely long-running or wedged.
1783+
_COMPRESSION_BUSY_WAIT_S = 5.0
17591784
_WRITE_RETRY_MIN_S = 0.020 # 20ms
17601785
_WRITE_RETRY_MAX_S = 0.150 # 150ms
17611786
_WRITE_RETRY_SLOW_AFTER_S = 2.0
@@ -2346,6 +2371,9 @@ def _execute_write(
23462371
if patience_s is None:
23472372
patience_s = self._WRITE_PATIENCE_S
23482373
deadline = time.monotonic() + patience_s
2374+
# Set on the first compression-busy collision so the short wait is
2375+
# measured from then, not from the start of the write.
2376+
compression_deadline: Optional[float] = None
23492377
while True:
23502378
try:
23512379
with self._lock:
@@ -2366,24 +2394,31 @@ def _execute_write(
23662394
if self._write_count % self._FTS_MERGE_EVERY_N_WRITES == 0:
23672395
self._try_incremental_merge_fts()
23682396
return result
2397+
except SessionCompressionInProgressError:
2398+
# A live foreign compression lock is transient: the compressor
2399+
# publishes in a couple of seconds. Without any wait, a steer
2400+
# that lands mid-compression aborts the user's turn as
2401+
# session_persistence_failed and sends the operator hunting
2402+
# disk space that was never the problem (#75083).
2403+
#
2404+
# The budget is _COMPRESSION_BUSY_WAIT_S, not the write-lock
2405+
# patience: the lease is a correctness boundary, so a writer
2406+
# still locked out after a short wait must be refused rather
2407+
# than left to land a stale turn once a long-running or wedged
2408+
# compression finally lets go.
2409+
if compression_deadline is None:
2410+
compression_deadline = min(
2411+
time.monotonic() + self._COMPRESSION_BUSY_WAIT_S, deadline
2412+
)
2413+
if self._sleep_before_write_retry(
2414+
compression_deadline, self._COMPRESSION_BUSY_WAIT_S
2415+
):
2416+
continue
2417+
raise
23692418
except sqlite3.OperationalError as exc:
23702419
err_msg = str(exc).lower()
23712420
if "locked" in err_msg or "busy" in err_msg:
2372-
now = time.monotonic()
2373-
if now < deadline:
2374-
elapsed = now - (deadline - patience_s)
2375-
if elapsed >= self._WRITE_RETRY_SLOW_AFTER_S:
2376-
jitter = random.uniform(
2377-
self._WRITE_RETRY_SLOW_MIN_S,
2378-
self._WRITE_RETRY_SLOW_MAX_S,
2379-
)
2380-
else:
2381-
jitter = random.uniform(
2382-
self._WRITE_RETRY_MIN_S,
2383-
self._WRITE_RETRY_MAX_S,
2384-
)
2385-
# Never overshoot the deadline by a full slow-jitter.
2386-
time.sleep(min(jitter, max(deadline - now, 0.001)))
2421+
if self._sleep_before_write_retry(deadline, patience_s):
23872422
continue
23882423
# Patience exhausted — say what actually happened so the
23892424
# surfaced error doesn't read as disk/permission damage.
@@ -2410,6 +2445,34 @@ def _execute_write(
24102445
raise
24112446
continue
24122447

2448+
def _sleep_before_write_retry(
2449+
self, deadline: float, patience_s: float
2450+
) -> bool:
2451+
"""Sleep one jitter interval if the patience budget still allows it.
2452+
2453+
Returns True when the caller should retry, False when *deadline* has
2454+
passed and the error should propagate. Jitter stays small for the
2455+
first ``_WRITE_RETRY_SLOW_AFTER_S`` (fast reclaim on millisecond
2456+
contention) and backs off after that, and never overshoots the
2457+
deadline by a full slow-jitter.
2458+
"""
2459+
now = time.monotonic()
2460+
if now >= deadline:
2461+
return False
2462+
elapsed = now - (deadline - patience_s)
2463+
if elapsed >= self._WRITE_RETRY_SLOW_AFTER_S:
2464+
jitter = random.uniform(
2465+
self._WRITE_RETRY_SLOW_MIN_S,
2466+
self._WRITE_RETRY_SLOW_MAX_S,
2467+
)
2468+
else:
2469+
jitter = random.uniform(
2470+
self._WRITE_RETRY_MIN_S,
2471+
self._WRITE_RETRY_MAX_S,
2472+
)
2473+
time.sleep(min(jitter, max(deadline - now, 0.001)))
2474+
return True
2475+
24132476
@staticmethod
24142477
def _is_fts_write_corruption_error(exc: sqlite3.DatabaseError) -> bool:
24152478
"""True for the error class a corrupt FTS index raises on writes.
@@ -5545,7 +5608,7 @@ def _do(conn):
55455608
active_lock is not None
55465609
and active_lock["holder"] != compression_lock_holder
55475610
):
5548-
raise CompressionSessionBusyError(
5611+
raise SessionCompressionInProgressError(
55495612
f"Session {session_id!r} is being compressed by another writer"
55505613
)
55515614
session = conn.execute(
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
"""A live compression lock must delay a concurrent append, not destroy the turn.
2+
3+
``append_message`` refused immediately when another writer held the session's
4+
compression lock. The conversation loop turns that into
5+
``session_persistence_failed`` and tells the operator to check disk space and
6+
permissions, when in fact the store is healthy and busy for a few seconds.
7+
8+
The write lock already gets a patience budget for exactly this reason (#74478).
9+
A compression hold is even more bounded, since the lock row carries its own
10+
``expires_at``, so the same budget applies here.
11+
12+
The sibling condition, a compressor finding its own lease gone, is permanent
13+
and must still fail fast rather than spin out the whole budget.
14+
"""
15+
16+
from __future__ import annotations
17+
18+
import threading
19+
import time
20+
from pathlib import Path
21+
22+
import pytest
23+
24+
from hermes_state import CompressionSessionBusyError, SessionDB
25+
26+
27+
@pytest.fixture
28+
def db(tmp_path: Path) -> SessionDB:
29+
d = SessionDB(tmp_path / "state.db")
30+
d.create_session("sess1", source="test")
31+
return d
32+
33+
34+
def test_append_waits_out_a_live_compression_lock(db: SessionDB) -> None:
35+
"""The classic race: a steer lands while compression owns the session."""
36+
assert db.try_acquire_compression_lock("sess1", "compressor") is True
37+
38+
released = threading.Event()
39+
40+
def _release_soon():
41+
time.sleep(0.3)
42+
db.release_compression_lock("sess1", "compressor")
43+
released.set()
44+
45+
t = threading.Thread(target=_release_soon, daemon=True)
46+
t.start()
47+
try:
48+
started = time.monotonic()
49+
# No compression_lock_holder: this is an ordinary turn writer.
50+
db.append_message("sess1", role="user", content="steered mid-compression")
51+
elapsed = time.monotonic() - started
52+
finally:
53+
t.join(timeout=5)
54+
55+
assert released.is_set(), "test bug: lock was never released"
56+
assert elapsed >= 0.25, "append returned before the lock could clear"
57+
rows = db.get_messages("sess1")
58+
assert any(r["content"] == "steered mid-compression" for r in rows), (
59+
"the message the user sent was lost"
60+
)
61+
62+
63+
def test_append_still_gives_up_when_the_lock_never_clears(
64+
db: SessionDB, monkeypatch
65+
) -> None:
66+
"""The wait is bounded: a lock that never clears is still refused.
67+
68+
The lease is a correctness boundary, so a genuinely long-running or wedged
69+
compression must not end with a stale turn landing in the parent.
70+
"""
71+
monkeypatch.setattr(SessionDB, "_COMPRESSION_BUSY_WAIT_S", 0.5)
72+
assert db.try_acquire_compression_lock("sess1", "compressor") is True
73+
74+
started = time.monotonic()
75+
with pytest.raises(CompressionSessionBusyError):
76+
db.append_message("sess1", role="user", content="never lands")
77+
elapsed = time.monotonic() - started
78+
79+
assert elapsed >= 0.4, "gave up before spending the patience budget"
80+
assert elapsed < 10, "did not give up within a bounded time"
81+
82+
83+
def test_the_lock_owner_is_never_delayed_by_its_own_lock(db: SessionDB) -> None:
84+
assert db.try_acquire_compression_lock("sess1", "compressor") is True
85+
86+
started = time.monotonic()
87+
db.append_message(
88+
"sess1",
89+
role="assistant",
90+
content="written by the compressor",
91+
compression_lock_holder="compressor",
92+
)
93+
assert time.monotonic() - started < 0.2
94+
95+
96+
def test_transient_error_is_a_subclass_of_the_original(db: SessionDB) -> None:
97+
"""Existing `except CompressionSessionBusyError` handlers must still catch."""
98+
from hermes_state import SessionCompressionInProgressError
99+
100+
assert issubclass(SessionCompressionInProgressError, CompressionSessionBusyError)
101+
102+
103+
def test_no_lock_means_no_delay(db: SessionDB) -> None:
104+
started = time.monotonic()
105+
db.append_message("sess1", role="user", content="uncontended")
106+
assert time.monotonic() - started < 0.2
107+
108+
109+
def test_a_lost_compression_lease_still_fails_fast(db: SessionDB) -> None:
110+
"""The other CompressionSessionBusyError case must NOT be retried.
111+
112+
``publish_compression_child`` raises the same base class when the
113+
compressor discovers its own lease is gone. That is permanent, so
114+
retrying would burn the whole patience budget before failing anyway.
115+
Only the transient subclass raised by ``append_message`` is retried.
116+
"""
117+
started = time.monotonic()
118+
with pytest.raises(CompressionSessionBusyError):
119+
db.publish_compression_child(
120+
parent_session_id="sess1",
121+
child_session_id="child1",
122+
source="test",
123+
messages=[{"role": "user", "content": "compacted"}],
124+
compression_lock_holder="not-the-holder",
125+
require_compression_lease=True,
126+
)
127+
assert time.monotonic() - started < 0.5, (
128+
"a lost lease is permanent and must not spend the retry budget"
129+
)

0 commit comments

Comments
 (0)