Skip to content

Commit a750840

Browse files
TheSentinel454Elrondclaude
committed
fix(buzz-acp): hold every REQ's since until its history reaches EOSE
The relay registers a REQ's live subscription before it reads history, so a live event can arrive ahead of a history read that then fails. That event moves last_seen past history that never arrived, and the retry after the CLOSED (or the next reconnect, if the socket drops before EOSE) started from there, skipping it. Only an overflow loss record held the floor before (loganj's review). note_req_sent now records each REQ's since, merged by min with any loss record a replay REQ retires, in unconfirmed_floor_this_conn (renamed from retired_loss_this_conn). A counted EOSE releases it; a CLOSED or a connection close restores it as a loss record through the existing paths. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 60f1eaa commit a750840

3 files changed

Lines changed: 117 additions & 40 deletions

File tree

‎crates/buzz-acp/src/relay.rs‎

Lines changed: 39 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1307,15 +1307,17 @@ struct BgState {
13071307
/// `last_inbound_secs`), capped at the time the close was detected. A
13081308
/// reconnect replays from here minus `SILENT_DROP_REPAIR_WINDOW_SECS`.
13091309
live_until: HashMap<String, u64>,
1310-
/// Loss records (`channel_dropped_since` / `membership_dropped_since`)
1311-
/// that a REQ written on the CURRENT connection retired, keyed by
1312-
/// subscription ID. Writing a replay REQ retires its loss record, but the
1313-
/// replay only finished once that REQ reaches a counted EOSE, which removes
1314-
/// the entry. If the connection closes first, the record is restored, so
1315-
/// the next replay is still held at or before the loss whatever `last_seen`
1316-
/// or `live_until` say. Cleared on every close; it never outlives the
1317-
/// connection, so on a live connection overflow recovery behaves as before.
1318-
retired_loss_this_conn: HashMap<String, u64>,
1310+
/// Replay floor of each subscription whose REQ was written on the CURRENT
1311+
/// connection and has not reached a counted EOSE, keyed by subscription
1312+
/// ID: the earliest of those REQs' `since` and any loss record
1313+
/// (`channel_dropped_since` / `membership_dropped_since`) a replay REQ
1314+
/// retired. The relay registers the live subscription before it reads
1315+
/// history, so live events can move `last_seen` past history that has not
1316+
/// arrived yet. A counted EOSE removes the entry; a CLOSED or a connection
1317+
/// close first restores it as a loss record, so the next REQ is still held
1318+
/// at or before the floor whatever `last_seen` or `live_until` say. Cleared
1319+
/// on every close; it never outlives the connection.
1320+
unconfirmed_floor_this_conn: HashMap<String, u64>,
13191321
/// Close code of the last WebSocket Close frame received, consumed by the
13201322
/// next reconnect to choose the first-attempt delay.
13211323
last_close_code: Option<u16>,
@@ -1377,7 +1379,7 @@ impl BgState {
13771379
backoff_step: 0,
13781380
eose_this_conn: HashSet::new(),
13791381
live_until: HashMap::new(),
1380-
retired_loss_this_conn: HashMap::new(),
1382+
unconfirmed_floor_this_conn: HashMap::new(),
13811383
last_close_code: None,
13821384
last_inbound_secs: None,
13831385
now_secs: unix_now_secs,
@@ -1471,29 +1473,29 @@ impl BgState {
14711473
.is_some_and(|channel_id| self.active_subscriptions.contains_key(&channel_id));
14721474
if known {
14731475
self.eose_this_conn.insert(sub_id.to_owned());
1474-
self.retired_loss_this_conn.remove(sub_id);
1476+
self.unconfirmed_floor_this_conn.remove(sub_id);
14751477
}
14761478
}
14771479

1478-
/// A REQ was written for `sub_id`: it has not reached EOSE yet, and a
1479-
/// second REQ on the same connection supersedes the first for the rest of
1480-
/// that connection (see `superseded_this_conn`). A caller that wrote a
1481-
/// replay REQ retires the subscription's loss record right after this, so
1482-
/// the record is kept in `retired_loss_this_conn` until the replay is known
1483-
/// to have finished.
1484-
fn note_req_sent(&mut self, sub_id: &str) {
1480+
/// A REQ with `since` was written for `sub_id`: it has not reached EOSE
1481+
/// yet, and a second REQ on the same connection supersedes the first for
1482+
/// the rest of that connection (see `superseded_this_conn`). Its floor
1483+
/// (`since`, or an earlier loss record, which a caller that wrote a replay
1484+
/// REQ retires right after this) is kept in `unconfirmed_floor_this_conn`
1485+
/// until the history is known to have arrived.
1486+
fn note_req_sent(&mut self, sub_id: &str, since: Option<u64>) {
14851487
self.eose_this_conn.remove(sub_id);
14861488
let loss = if sub_id == MEMBERSHIP_NOTIF_SUB_ID {
14871489
self.membership_dropped_since
14881490
} else {
14891491
channel_id_from_sub_id(sub_id)
14901492
.and_then(|channel_id| self.channel_dropped_since.get(&channel_id).copied())
14911493
};
1492-
if let Some(loss) = loss {
1493-
self.retired_loss_this_conn
1494+
if let Some(floor) = since.into_iter().chain(loss).min() {
1495+
self.unconfirmed_floor_this_conn
14941496
.entry(sub_id.to_owned())
1495-
.and_modify(|t| *t = (*t).min(loss))
1496-
.or_insert(loss);
1497+
.and_modify(|t| *t = (*t).min(floor))
1498+
.or_insert(floor);
14971499
}
14981500
if !self.req_sent_this_conn.insert(sub_id.to_owned()) {
14991501
self.superseded_this_conn.insert(sub_id.to_owned());
@@ -1503,12 +1505,12 @@ impl BgState {
15031505
/// The current connection closed (detected at `now_secs`): every
15041506
/// subscription that reached EOSE on it was live until its last inbound
15051507
/// frame, capped at `now_secs`. Subscriptions that never reached EOSE keep
1506-
/// whatever cursor they already had, and loss records retired by a replay
1507-
/// that never reached a counted EOSE are restored. `last_inbound_secs` is
1508+
/// whatever cursor they already had, and the floor of each REQ that never
1509+
/// reached a counted EOSE is restored. `last_inbound_secs` is
15081510
/// deliberately kept so a repeated call for the same outage stamps the
15091511
/// same time.
15101512
fn note_connection_closed(&mut self, now_secs: u64) {
1511-
for (sub_id, loss) in std::mem::take(&mut self.retired_loss_this_conn) {
1513+
for (sub_id, loss) in std::mem::take(&mut self.unconfirmed_floor_this_conn) {
15121514
self.restore_loss(&sub_id, loss);
15131515
}
15141516
let live_until = self
@@ -1525,8 +1527,9 @@ impl BgState {
15251527
self.reset_inbound_clock();
15261528
}
15271529

1528-
/// Merge `loss` back into the subscription's loss record (`min` with any
1529-
/// newer loss), so its next replay starts at or before it again.
1530+
/// Merge `loss`, a floor from `unconfirmed_floor_this_conn`, into the
1531+
/// subscription's loss record (`min` with any newer loss), so its next
1532+
/// replay starts at or before it again.
15301533
fn restore_loss(&mut self, sub_id: &str, loss: u64) {
15311534
if sub_id == MEMBERSHIP_NOTIF_SUB_ID {
15321535
self.membership_dropped_since =
@@ -1541,12 +1544,12 @@ impl BgState {
15411544

15421545
/// The relay CLOSED `sub_id` on the current connection, so the subscription
15431546
/// is no longer live: a later connection close must not stamp `live_until`
1544-
/// over the time it was closed, and a loss record its REQ retired is
1545-
/// restored because that replay never reached EOSE. The retry's `since` is
1546-
/// then never later than the closed REQ's.
1547+
/// over the time it was closed, and its REQ's floor is restored because
1548+
/// that history never reached EOSE. The retry's `since` is then never later
1549+
/// than the closed REQ's, even if live events moved `last_seen` past it.
15471550
fn note_sub_closed(&mut self, sub_id: &str) {
15481551
self.eose_this_conn.remove(sub_id);
1549-
if let Some(loss) = self.retired_loss_this_conn.remove(sub_id) {
1552+
if let Some(loss) = self.unconfirmed_floor_this_conn.remove(sub_id) {
15501553
self.restore_loss(sub_id, loss);
15511554
}
15521555
}
@@ -1587,7 +1590,7 @@ impl BgState {
15871590
// cleared: a re-subscribe on this socket reuses the wire ID, and an EOSE
15881591
// for the old REQ must not count for the new one.
15891592
self.live_until.remove(&sub_id);
1590-
self.retired_loss_this_conn.remove(&sub_id);
1593+
self.unconfirmed_floor_this_conn.remove(&sub_id);
15911594
self.closed_failures.remove(&sub_id);
15921595
self.active_filters.remove(channel_id);
15931596
self.rate_limited_pending.remove(channel_id);
@@ -2974,8 +2977,8 @@ async fn resubscribe_after_reconnect(
29742977
if is_fresh_connection {
29752978
// The previous socket is gone. The reconnect entry already closed it
29762979
// out, but a socket that died partway through an earlier resubscribe
2977-
// (`RetryConnection`) did not: restore the loss records its REQs
2978-
// retired before any `since` is computed. Idempotent otherwise.
2980+
// (`RetryConnection`) did not: restore its unconfirmed REQ floors
2981+
// before any `since` is computed. Idempotent otherwise.
29792982
state.note_connection_closed((state.now_secs)());
29802983
// These queues are derived from active subscription intent and rebuilt
29812984
// below. The rate-limit gate is deliberately preserved: the relay's
@@ -3683,7 +3686,7 @@ async fn send_subscribe(
36833686
Ok(text) => {
36843687
match ws_send_timeout(ws, Message::Text(text.into()), WS_SEND_TIMEOUT_SECS).await {
36853688
Ok(()) => {
3686-
state.note_req_sent(&sub_id);
3689+
state.note_req_sent(&sub_id, since);
36873690
debug!(
36883691
"subscribed to channel {channel_id}{}",
36893692
if since.is_some() {
@@ -3739,7 +3742,7 @@ async fn send_membership_subscribe(
37393742
Ok(text) => {
37403743
match ws_send_timeout(ws, Message::Text(text.into()), WS_SEND_TIMEOUT_SECS).await {
37413744
Ok(()) => {
3742-
state.note_req_sent(MEMBERSHIP_NOTIF_SUB_ID);
3745+
state.note_req_sent(MEMBERSHIP_NOTIF_SUB_ID, since);
37433746
debug!("subscribed to membership notifications (since={since_ts})");
37443747
true
37453748
}

‎crates/buzz-acp/src/relay/closed_retry_tests.rs‎

Lines changed: 77 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
//! and EOSE never follows. The retry must back off, and its `since` must still
66
//! cover the history that failed to arrive.
77
use super::resume_tests::{
8-
deliver_eose, first_replay_req, lose_connection, mock_relay, next_req, open_replay, LossArm,
9-
MockCtl,
8+
deliver_eose, deliver_event, first_replay_req, lose_connection, mock_relay, next_req,
9+
open_replay, tagged_event, LossArm, MockCtl,
1010
};
1111
use super::tests::{seed_test_subscription, test_channel_filter, test_ws_pair};
1212
use super::*;
@@ -249,9 +249,74 @@ async fn closed_gap_replay_restores_its_loss_record_for_the_retry() {
249249
}
250250
}
251251

252+
/// The relay registers the live subscription before it reads history, so a
253+
/// live event can arrive ahead of a history read that then fails, moving
254+
/// `last_seen` past history that never arrived. With no loss record in play,
255+
/// the REQ's own `since` must still hold the same-socket retry (after a CLOSED)
256+
/// and the next reconnect (socket dropped before EOSE) at or before it. A
257+
/// counted EOSE releases that floor.
258+
#[tokio::test]
259+
async fn live_event_before_a_failed_history_read_keeps_the_reqs_floor() {
260+
const BASE: u64 = 1_000;
261+
enum End {
262+
Closed(&'static str),
263+
Dropped,
264+
EoseThenDropped,
265+
}
266+
for arm in LossArm::both() {
267+
for end in [
268+
End::Closed(DB_ERROR),
269+
End::Closed("rate-limited: retry in 30s"),
270+
End::Dropped,
271+
End::EoseThenDropped,
272+
] {
273+
let mut state = BgState::new();
274+
match arm {
275+
LossArm::Channel(channel) => seed_test_subscription(&mut state, channel),
276+
LossArm::Membership => {
277+
state.membership_sub_active = true;
278+
state.startup_watermark = Some(BASE);
279+
}
280+
}
281+
let sub = arm.sub();
282+
let (mut client, _server, first) = open_replay(&mut state, &sub).await;
283+
assert!(first <= BASE, "{sub}: first REQ since={first}");
284+
let (tx, _rx) = mpsc::channel(8);
285+
deliver_event(&mut client, &mut state, &tx, &sub, &arm.event(2_000)).await;
286+
assert_eq!(
287+
arm.since(&state),
288+
Some(2_000),
289+
"{sub}: the live event moved the cursor"
290+
);
291+
292+
let since = match end {
293+
End::Closed(message) => {
294+
deliver_closed(&mut client, &mut state, &sub, message).await;
295+
arm.since(&state).unwrap()
296+
}
297+
End::Dropped => {
298+
lose_connection(&mut state).await;
299+
first_replay_req(&mut state, &sub).await
300+
}
301+
End::EoseThenDropped => {
302+
deliver_eose(&mut client, &mut state, &sub).await;
303+
lose_connection(&mut state).await;
304+
assert_eq!(arm.dropped(&state), None, "{sub}: EOSE released the floor");
305+
continue;
306+
}
307+
};
308+
assert!(
309+
since <= BASE,
310+
"{sub}: must still replay from {BASE}, got since={since}"
311+
);
312+
}
313+
}
314+
}
315+
252316
/// Through the real socket-owner loop: a channel REQ answered with an error
253317
/// CLOSED is retried after the first backoff step (2s ±20%), not before, with
254-
/// the same `since`. The loop has no other traffic, so only the parked-retry
318+
/// the same `since`, even though a live event arrived before the CLOSED.
319+
/// After the CLOSED the loop has no other traffic, so only the parked-retry
255320
/// wake can send it in time.
256321
#[tokio::test]
257322
async fn socket_owner_retries_a_closed_req_after_the_backoff() {
@@ -289,6 +354,15 @@ async fn socket_owner_retries_a_closed_req_after_the_backoff() {
289354
.unwrap();
290355
let first = next_req(&mut conn).await;
291356
assert_eq!(first[1], channel_sub_id(channel));
357+
// The relay registers the live subscription before it reads history, so a
358+
// live event can arrive ahead of the failed read.
359+
conn.ctl
360+
.send(MockCtl::Send(json!([
361+
"EVENT",
362+
first[1],
363+
tagged_event(channel, 2_000)
364+
])))
365+
.unwrap();
292366
conn.ctl
293367
.send(MockCtl::Send(json!(["CLOSED", first[1], DB_ERROR])))
294368
.unwrap();

‎crates/buzz-acp/src/relay/resume_tests.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1125,7 +1125,7 @@ impl LossArm {
11251125
}
11261126
}
11271127

1128-
fn event(self, created_at: u64) -> Event {
1128+
pub(super) fn event(self, created_at: u64) -> Event {
11291129
match self {
11301130
Self::Channel(channel) => tagged_event(channel, created_at),
11311131
Self::Membership => tagged_event(Uuid::nil(), created_at),

0 commit comments

Comments
 (0)