Skip to content

Commit e9cff2e

Browse files
TheSentinel454Elrondclaude
committed
fix(buzz-acp): lower live_until on REQ write instead of a new floor map
The unconfirmed-REQ floor also held back overflow recovery on a live connection: a superseded REQ's EOSE is never counted, so its since stuck until the next connection and every later recovery REQ replayed from the first one. That changed recovery behaviour the PR does not own (two recovery_wake_tests caught it). Fix the review schedule at its source instead: writing a REQ lowers the subscription's retained live_until so it resumes no later than that REQ's since. A gap replay below live_until keeps its floor when the socket drops before EOSE; a counted EOSE plus close restamps it. No new state. Adds a regression for overflow recovery below a retained live_until. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 8157e07 commit e9cff2e

2 files changed

Lines changed: 65 additions & 51 deletions

File tree

‎crates/buzz-acp/src/relay.rs‎

Lines changed: 21 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,15 +1280,9 @@ struct BgState {
12801280
/// Unix seconds up to which a subscription that reached EOSE was last known
12811281
/// to be live: the newest non-Close frame read on that connection (see
12821282
/// `last_inbound_secs`), capped at the time the close was detected. A
1283-
/// reconnect replays from here minus `RESUME_LIVE_SKEW_SECS`.
1283+
/// reconnect replays from here minus `RESUME_LIVE_SKEW_SECS`. Lowered when
1284+
/// a REQ is written with an older `since` (see [`BgState::note_req_sent`]).
12841285
live_until: HashMap<String, u64>,
1285-
/// `since` of the newest REQ written for a subscription that has not yet
1286-
/// reached EOSE. Until it does, that REQ's replay may be incomplete, so the
1287-
/// next replay never starts after it. This keeps an overflow-recovery or
1288-
/// reconnect replay's floor when the socket drops before its EOSE, even
1289-
/// though `channel_dropped_since` / `membership_dropped_since` were
1290-
/// retired when the REQ was written.
1291-
unconfirmed_req_since: HashMap<String, u64>,
12921286
/// Close code of the last WebSocket Close frame received, consumed by the
12931287
/// next reconnect to choose the first-attempt delay.
12941288
last_close_code: Option<u16>,
@@ -1349,7 +1343,6 @@ impl BgState {
13491343
backoff_step: 0,
13501344
eose_this_conn: HashSet::new(),
13511345
live_until: HashMap::new(),
1352-
unconfirmed_req_since: HashMap::new(),
13531346
last_close_code: None,
13541347
last_inbound_secs: None,
13551348
now_secs: unix_now_secs,
@@ -1392,7 +1385,7 @@ impl BgState {
13921385
.copied()
13931386
.or(self.startup_watermark),
13941387
self.last_seen.get(channel_id).copied(),
1395-
self.replay_floor(&sub_id, self.channel_dropped_since.get(channel_id).copied()),
1388+
self.channel_dropped_since.get(channel_id).copied(),
13961389
self.live_until.get(&sub_id).copied(),
13971390
(self.now_secs)(),
13981391
)
@@ -1403,20 +1396,12 @@ impl BgState {
14031396
resume_since(
14041397
self.startup_watermark,
14051398
self.membership_last_seen,
1406-
self.replay_floor(MEMBERSHIP_NOTIF_SUB_ID, self.membership_dropped_since),
1399+
self.membership_dropped_since,
14071400
self.live_until.get(MEMBERSHIP_NOTIF_SUB_ID).copied(),
14081401
(self.now_secs)(),
14091402
)
14101403
}
14111404

1412-
/// The point a subscription's next replay must not start after: the older
1413-
/// of its recorded backpressure loss and the `since` of a REQ that has not
1414-
/// reached EOSE.
1415-
fn replay_floor(&self, sub_id: &str, dropped: Option<u64>) -> Option<u64> {
1416-
let unconfirmed = self.unconfirmed_req_since.get(sub_id).copied();
1417-
dropped.into_iter().chain(unconfirmed).min()
1418-
}
1419-
14201405
/// Unix seconds to stamp an inbound frame with: the connection's anchor
14211406
/// wall time plus monotonic time elapsed since. The monotonic clock does
14221407
/// not advance while the machine is suspended (macOS, Linux), so a frame
@@ -1449,21 +1434,23 @@ impl BgState {
14491434
.is_some_and(|channel_id| self.active_subscriptions.contains_key(&channel_id));
14501435
if known {
14511436
self.eose_this_conn.insert(sub_id.to_owned());
1452-
self.unconfirmed_req_since.remove(sub_id);
14531437
}
14541438
}
14551439

14561440
/// A REQ with `since` was written for `sub_id`: it has not reached EOSE
1457-
/// yet, so its `since` floors the next replay (see `unconfirmed_req_since`),
1458-
/// and a second REQ on the same connection supersedes the first for the
1459-
/// rest of that connection (see `superseded_this_conn`).
1441+
/// yet, and a second REQ on the same connection supersedes the first for
1442+
/// the rest of that connection (see `superseded_this_conn`).
1443+
///
1444+
/// Its replay may never finish (the socket can drop first), so retained
1445+
/// live coverage is lowered to resume no later than this `since`. A gap
1446+
/// replay below `live_until` (overflow recovery, or a reconnect whose
1447+
/// `since` was held back by `channel_dropped_since`) therefore keeps its
1448+
/// floor even though writing the REQ retired the loss record. A counted
1449+
/// EOSE followed by a close restamps `live_until` as usual.
14601450
fn note_req_sent(&mut self, sub_id: &str, since: Option<u64>) {
14611451
self.eose_this_conn.remove(sub_id);
1462-
if let Some(since) = since {
1463-
self.unconfirmed_req_since
1464-
.entry(sub_id.to_owned())
1465-
.and_modify(|floor| *floor = (*floor).min(since))
1466-
.or_insert(since);
1452+
if let (Some(since), Some(live_until)) = (since, self.live_until.get_mut(sub_id)) {
1453+
*live_until = (*live_until).min(since.saturating_add(RESUME_LIVE_SKEW_SECS));
14671454
}
14681455
if !self.req_sent_this_conn.insert(sub_id.to_owned()) {
14691456
self.superseded_this_conn.insert(sub_id.to_owned());
@@ -1506,7 +1493,6 @@ impl BgState {
15061493
// cleared: a re-subscribe on this socket reuses the wire ID, and an EOSE
15071494
// for the old REQ must not count for the new one.
15081495
self.live_until.remove(&sub_id);
1509-
self.unconfirmed_req_since.remove(&sub_id);
15101496
self.active_filters.remove(channel_id);
15111497
self.rate_limited_pending.remove(channel_id);
15121498
self.resubscribe_retry.remove(channel_id);
@@ -3809,20 +3795,20 @@ fn first_reconnect_delay(close_code: Option<u16>, sample: f64) -> Duration {
38093795
///
38103796
/// - `base`: original replay floor (`subscribe_since` / startup watermark).
38113797
/// - `last_seen`: newest delivered event `created_at`.
3812-
/// - `floor`: [`BgState::replay_floor`], the older of the oldest event lost to
3813-
/// backpressure and the `since` of a REQ that has not reached EOSE; the
3798+
/// - `dropped`: oldest event lost to backpressure; a recorded gap, so the
38143799
/// result never advances past it.
38153800
/// - `live_until`: see [`BgState::live_until`].
38163801
/// - `now`: wall-clock unix seconds, for [`SILENT_DROP_REPAIR_WINDOW_SECS`].
38173802
///
3818-
/// Without `live_until` this is the pre-existing `min(last_seen, floor)`
3803+
/// Without `live_until` this is the pre-existing `min(last_seen, dropped)`
38193804
/// else `base`. With `live_until`, the delivered-progress cursor becomes
38203805
/// `max(base, last_seen, min(live_until - RESUME_LIVE_SKEW_SECS,
3821-
/// now - SILENT_DROP_REPAIR_WINDOW_SECS))`, then the floor is applied on top.
3806+
/// now - SILENT_DROP_REPAIR_WINDOW_SECS))`, then the dropped floor is applied
3807+
/// on top.
38223808
fn resume_since(
38233809
base: Option<u64>,
38243810
last_seen: Option<u64>,
3825-
floor: Option<u64>,
3811+
dropped: Option<u64>,
38263812
live_until: Option<u64>,
38273813
now: u64,
38283814
) -> Option<u64> {
@@ -3840,7 +3826,7 @@ fn resume_since(
38403826
}
38413827
None => last_seen,
38423828
};
3843-
match (cursor, floor) {
3829+
match (cursor, dropped) {
38443830
(Some(c), Some(d)) => Some(c.min(d)),
38453831
(Some(c), None) => Some(c),
38463832
(None, Some(d)) => Some(d),

‎crates/buzz-acp/src/relay/resume_tests.rs‎

Lines changed: 44 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -866,8 +866,8 @@ async fn replay_then_lose_before_eose(state: &mut BgState, sub: &str) -> u64 {
866866
/// The review's schedule: a subscription with a retained `live_until` loses an
867867
/// event to backpressure, the reconnect replay REQ correctly goes out below it
868868
/// (retiring the loss record), and that socket drops before the replay's EOSE.
869-
/// The next replay must still cover the lost event. Once a replay does reach
870-
/// EOSE, the cursor moves past the gap again.
869+
/// The retained `live_until` must not skip the lost event on the next replay.
870+
/// Once a replay does reach EOSE, the cursor moves past the gap again.
871871
async fn unfinished_gap_replay_keeps_its_floor(
872872
state: &mut BgState,
873873
sub: &str,
@@ -942,27 +942,55 @@ async fn unfinished_membership_gap_replay_keeps_its_floor() {
942942
.await;
943943
}
944944

945+
/// Overflow recovery on a connection whose subscription already has a retained
946+
/// `live_until`: the recovery REQ retires the loss record when written, and the
947+
/// socket drops before its replay finishes. The next replay must still cover
948+
/// the lost event rather than resume from the retained live coverage.
945949
#[tokio::test]
946-
async fn removing_a_channel_clears_its_unconfirmed_replay_floor() {
947-
let (mut client, _server) = test_ws_pair().await;
950+
async fn overflow_recovery_below_a_retained_live_until_keeps_its_floor() {
948951
let mut state = BgState::new();
949952
let channel = Uuid::new_v4();
950-
seed_test_subscription(&mut state, channel);
951-
let since = state.channel_since(&channel);
952-
assert!(
953-
send_subscribe(
953+
seed_test_subscription(&mut state, channel); // base 1_000
954+
let sub = channel_sub_id(channel);
955+
let (mut earlier, _earlier_server) = test_ws_pair().await;
956+
deliver_eose(&mut earlier, &mut state, &sub).await;
957+
lose_connection(&mut state).await;
958+
assert!(state.live_until.contains_key(&sub));
959+
960+
// The reconnect replay reaches EOSE on the new socket.
961+
let (mut client, mut server) = test_ws_pair().await;
962+
let (_cmd_tx, mut cmd_rx) = mpsc::channel(1);
963+
let result =
964+
resubscribe_after_reconnect(&mut client, &mut cmd_rx, &mut state, "agent", true).await;
965+
assert!(matches!(result, ResubscribeResult::Ok));
966+
assert_eq!(next_test_frame(&mut server).await[1], sub);
967+
deliver_eose(&mut client, &mut state, &sub).await;
968+
969+
// Live events overflow the queue; the third is recorded as lost.
970+
let (tx, mut rx) = mpsc::channel(2);
971+
for ts in [1_010, 1_020, 1_100] {
972+
deliver_event(
954973
&mut client,
955974
&mut state,
956-
channel,
957-
"agent",
958-
since,
959-
&test_channel_filter()
975+
&tx,
976+
&sub,
977+
&tagged_event(channel, ts),
960978
)
961-
.await
979+
.await;
980+
}
981+
assert_eq!(state.channel_dropped_since[&channel], 1_100);
982+
while rx.try_recv().is_ok() {}
983+
984+
recovery::recover_one(&mut client, &mut state, &tx, "agent").await;
985+
assert_eq!(next_test_frame(&mut server).await[0], "REQ");
986+
assert!(!state.channel_dropped_since.contains_key(&channel));
987+
988+
lose_connection(&mut state).await;
989+
let since = first_replay_req(&mut state, &sub).await;
990+
assert!(
991+
since <= 1_100,
992+
"replay must still cover the dropped event at 1100, got since={since}"
962993
);
963-
assert_eq!(state.unconfirmed_req_since[&channel_sub_id(channel)], 1_000);
964-
state.clear_channel_state(&channel);
965-
assert!(state.unconfirmed_req_since.is_empty());
966994
}
967995

968996
// -- Silent socket death, through the real socket-owner loop ----------------

0 commit comments

Comments
 (0)