Skip to content

Commit d1ba7ee

Browse files
TheSentinel454Elrondclaude
committed
fix(relay): count lost ephemeral fan-out events without closing
An ephemeral event (presence, typing, observer frames) is never stored, so a reconnect cannot replay it. Fencing and closing its recipients recovered nothing, and a global presence event reaches most of a community's connections, so one stale presence frame closed nearly all of them. A lost ephemeral event is now only counted, with reason="ephemeral"; it is still never delivered past its deadline. The loss tests move to a stored kind. New tests pin that an ephemeral loss neither fences nor closes while a stored loss on the same queue still does, and that the fence also withholds relay-local fan-out. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 03cad88 commit d1ba7ee

1 file changed

Lines changed: 128 additions & 35 deletions

File tree

‎crates/buzz-relay/src/handlers/multinode_fanout.rs‎

Lines changed: 128 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
//! broadcast, it cannot tell which events it skipped, so it does the same for
1616
//! every subscribed connection.
1717
//!
18+
//! An ephemeral event (presence, typing, observer frames) is never stored, so
19+
//! no reconnect can replay it. Losing one is only counted: closing its
20+
//! recipients would recover nothing, and a global presence event reaches most
21+
//! of a community's connections.
22+
//!
1823
//! A client resumes from the newest `created_at` it received, minus a few
1924
//! seconds of slack. Recovery depends on that cursor never getting further
2025
//! ahead of a lost event than the slack:
@@ -44,6 +49,7 @@ use std::hash::{DefaultHasher, Hash, Hasher};
4449
use std::sync::{Arc, Mutex, PoisonError};
4550
use std::time::Duration;
4651

52+
use buzz_core::kind::{event_kind_u32, is_ephemeral};
4753
use buzz_core::CommunityId;
4854
use buzz_pubsub::ChannelEvent;
4955
use tokio::sync::mpsc::error::TrySendError;
@@ -139,14 +145,20 @@ async fn deliver_or_lose(state: &Arc<AppState>, closer: &LagCloser, queued: Queu
139145
/// One known event will not be delivered: resync the syncs and fence and
140146
/// close the connections it would have reached. Matching skips the access
141147
/// gate, so a recipient that would have been filtered out may be closed too;
142-
/// that costs a reconnect, never a leak.
148+
/// that costs a reconnect, never a leak. A lost ephemeral event is only
149+
/// counted, since no reconnect can replay it.
143150
fn lose(
144151
state: &AppState,
145152
closer: &LagCloser,
146153
community_id: CommunityId,
147154
stored: &StoredEvent,
148155
reason: &'static str,
149156
) {
157+
if is_ephemeral(event_kind_u32(&stored.event)) {
158+
metrics::counter!("buzz_multinode_fanout_dropped_total", "reason" => "ephemeral")
159+
.increment(1);
160+
return;
161+
}
150162
metrics::counter!("buzz_multinode_fanout_dropped_total", "reason" => reason).increment(1);
151163
let matches = state.sub_registry.fan_out_scoped(community_id, stored);
152164
state.artifact_syncs.resync_matches(&matches, LAG_RESYNC);
@@ -301,15 +313,22 @@ mod tests {
301313
use uuid::Uuid;
302314

303315
use super::*;
316+
use crate::handlers::event::fan_out_event_to_local_subscribers;
304317

305318
fn community() -> CommunityId {
306319
CommunityId::from_uuid(Uuid::nil())
307320
}
308321

309-
/// A connection with one presence subscription, its frame receiver, and
322+
/// A stored kind: a lost event of it can be replayed, so its loss closes.
323+
const STORED: u16 = 1;
324+
/// An ephemeral kind: a lost event of it cannot be replayed.
325+
const PRESENCE: u16 = KIND_PRESENCE_UPDATE as u16;
326+
327+
/// A connection with one subscription to `kind`, its frame receiver, and
310328
/// its cancellation token.
311-
fn presence_subscriber(
329+
fn subscriber(
312330
state: &AppState,
331+
kind: u16,
313332
) -> (ConnId, mpsc::Receiver<Message>, CancellationToken) {
314333
let conn_id = Uuid::new_v4();
315334
let (tx, rx) = mpsc::channel(64);
@@ -329,20 +348,20 @@ mod tests {
329348
);
330349
state.sub_registry.register(
331350
conn_id,
332-
"presence".to_string(),
333-
vec![Filter::new().kind(Kind::Custom(KIND_PRESENCE_UPDATE as u16))],
351+
"sub".to_string(),
352+
vec![Filter::new().kind(Kind::Custom(kind))],
334353
None,
335354
);
336355
(conn_id, rx, cancel)
337356
}
338357

339-
fn presence(n: usize) -> ChannelEvent {
358+
fn event(kind: u16, n: usize) -> ChannelEvent {
340359
ChannelEvent {
341360
community_id: community(),
342361
topic: EventTopic::Global,
343-
event: EventBuilder::new(Kind::Custom(KIND_PRESENCE_UPDATE as u16), n.to_string())
362+
event: EventBuilder::new(Kind::Custom(kind), n.to_string())
344363
.sign_with_keys(&Keys::generate())
345-
.expect("sign presence"),
364+
.expect("sign event"),
346365
received_at: Instant::now(),
347366
}
348367
}
@@ -355,11 +374,11 @@ mod tests {
355374
#[tokio::test]
356375
async fn delivers_one_topic_in_order_through_workers() {
357376
let state = crate::state::tests::test_state().await;
358-
let (_conn, mut frames, cancel) = presence_subscriber(&state);
377+
let (_conn, mut frames, cancel) = subscriber(&state, STORED);
359378
let (tx, rx) = broadcast::channel(64);
360379
let handle = spawn(Arc::clone(&state), rx);
361380

362-
let sent: Vec<_> = (0..20).map(presence).collect();
381+
let sent: Vec<_> = (0..20).map(|n| event(STORED, n)).collect();
363382
for event in &sent {
364383
tx.send(event.clone()).expect("broadcast");
365384
}
@@ -386,20 +405,20 @@ mod tests {
386405
#[tokio::test]
387406
async fn full_worker_queue_closes_the_dropped_events_recipients() {
388407
let state = crate::state::tests::test_state().await;
389-
let (reached, _frames, _cancel) = presence_subscriber(&state);
408+
let (reached, _frames, _cancel) = subscriber(&state, STORED);
390409
let other = Uuid::new_v4();
391410
state.sub_registry.register(
392411
other,
393412
"other-kind".to_string(),
394-
vec![Filter::new().kind(Kind::Custom(1))],
413+
vec![Filter::new().kind(Kind::Custom(7))],
395414
None,
396415
);
397416
// Hold the only queue's receiver without draining it.
398417
let (dispatcher, _queues) = Dispatcher::new(1, 1);
399418

400-
dispatcher.dispatch(&state, presence(0));
419+
dispatcher.dispatch(&state, event(STORED, 0));
401420
assert!(pending(&dispatcher).is_empty(), "the first event fits");
402-
dispatcher.dispatch(&state, presence(1));
421+
dispatcher.dispatch(&state, event(STORED, 1));
403422
assert_eq!(pending(&dispatcher), HashSet::from([reached]));
404423
}
405424

@@ -408,11 +427,11 @@ mod tests {
408427
#[tokio::test]
409428
async fn local_echo_on_a_full_queue_closes_nothing() {
410429
let state = crate::state::tests::test_state().await;
411-
let _subscriber = presence_subscriber(&state);
430+
let _subscriber = subscriber(&state, STORED);
412431
let (dispatcher, _queues) = Dispatcher::new(1, 1);
413-
dispatcher.dispatch(&state, presence(0));
432+
dispatcher.dispatch(&state, event(STORED, 0));
414433

415-
let echo = presence(1);
434+
let echo = event(STORED, 1);
416435
state.mark_local_event(echo.community_id, &echo.event.id);
417436
dispatcher.dispatch(&state, echo);
418437
assert!(pending(&dispatcher).is_empty());
@@ -422,8 +441,8 @@ mod tests {
422441
#[tokio::test]
423442
async fn dispatcher_lag_closes_every_subscribed_connection() {
424443
let state = crate::state::tests::test_state().await;
425-
let (first, _f1, _c1) = presence_subscriber(&state);
426-
let (second, _f2, _c2) = presence_subscriber(&state);
444+
let (first, _f1, _c1) = subscriber(&state, STORED);
445+
let (second, _f2, _c2) = subscriber(&state, STORED);
427446
let (dispatcher, _queues) = Dispatcher::new(1, 1);
428447

429448
dispatcher.lagged(&state, 7);
@@ -435,7 +454,7 @@ mod tests {
435454
async fn closer_cancels_queued_connections_in_paced_batches() {
436455
let state = crate::state::tests::test_state().await;
437456
let subscribers: Vec<_> = (0..CLOSE_BATCH + 1)
438-
.map(|_| presence_subscriber(&state))
457+
.map(|_| subscriber(&state, STORED))
439458
.collect();
440459
let closer = Arc::new(LagCloser::default());
441460
tokio::spawn(Arc::clone(&closer).run(Arc::clone(&state)));
@@ -470,17 +489,17 @@ mod tests {
470489
#[tokio::test]
471490
async fn dropped_event_fences_its_recipients_before_their_close() {
472491
let state = crate::state::tests::test_state().await;
473-
let (_conn, mut frames, cancel) = presence_subscriber(&state);
492+
let (_conn, mut frames, cancel) = subscriber(&state, STORED);
474493
let (dispatcher, _queues) = Dispatcher::new(1, 1);
475494
assert!(
476-
worker_delivers(&state, &dispatcher.closer, &mut frames, presence(0)).await,
495+
worker_delivers(&state, &dispatcher.closer, &mut frames, event(STORED, 0)).await,
477496
"delivers before any loss"
478497
);
479-
dispatcher.dispatch(&state, presence(1));
480-
dispatcher.dispatch(&state, presence(2)); // dropped
498+
dispatcher.dispatch(&state, event(STORED, 1));
499+
dispatcher.dispatch(&state, event(STORED, 2)); // dropped
481500

482501
assert!(
483-
!worker_delivers(&state, &dispatcher.closer, &mut frames, presence(3)).await,
502+
!worker_delivers(&state, &dispatcher.closer, &mut frames, event(STORED, 3)).await,
484503
"an event after the loss must not reach the fenced connection"
485504
);
486505
assert!(
@@ -493,11 +512,11 @@ mod tests {
493512
#[tokio::test]
494513
async fn dispatcher_lag_fences_every_subscribed_connection() {
495514
let state = crate::state::tests::test_state().await;
496-
let (_conn, mut frames, cancel) = presence_subscriber(&state);
515+
let (_conn, mut frames, cancel) = subscriber(&state, STORED);
497516
let (dispatcher, _queues) = Dispatcher::new(1, 1);
498517
dispatcher.lagged(&state, 3);
499518

500-
assert!(!worker_delivers(&state, &dispatcher.closer, &mut frames, presence(0)).await);
519+
assert!(!worker_delivers(&state, &dispatcher.closer, &mut frames, event(STORED, 0)).await);
501520
assert!(!cancel.is_cancelled());
502521
}
503522

@@ -508,24 +527,24 @@ mod tests {
508527
#[tokio::test(start_paused = true)]
509528
async fn backlogged_event_is_delivered_by_its_deadline_or_lost() {
510529
let state = crate::state::tests::test_state().await;
511-
let (conn, mut frames, cancel) = presence_subscriber(&state);
530+
let (conn, mut frames, cancel) = subscriber(&state, STORED);
512531
let (dispatcher, mut queues) = Dispatcher::new(1, 4);
513532
let queue = &mut queues[0];
514533

515-
dispatcher.dispatch(&state, presence(0));
534+
dispatcher.dispatch(&state, event(STORED, 0));
516535
tokio::time::advance(MAX_QUEUE_DELAY - Duration::from_millis(1)).await;
517536
let in_time = queue.try_recv().expect("queued");
518537
deliver_or_lose(&state, &dispatcher.closer, in_time).await;
519538
assert!(frames.try_recv().is_ok(), "delivered inside its deadline");
520539

521-
dispatcher.dispatch(&state, presence(1)); // R
540+
dispatcher.dispatch(&state, event(STORED, 1)); // R
522541
tokio::time::advance(MAX_QUEUE_DELAY).await;
523542
let late = queue.try_recv().expect("queued");
524543
deliver_or_lose(&state, &dispatcher.closer, late).await;
525544
assert!(frames.try_recv().is_err(), "R is not delivered late");
526545
assert_eq!(pending(&dispatcher), HashSet::from([conn]));
527546
assert!(
528-
!worker_delivers(&state, &dispatcher.closer, &mut frames, presence(2)).await,
547+
!worker_delivers(&state, &dispatcher.closer, &mut frames, event(STORED, 2)).await,
529548
"R's recipient is fenced until it closes"
530549
);
531550
assert!(!cancel.is_cancelled(), "the close waits for the closer");
@@ -536,10 +555,10 @@ mod tests {
536555
#[tokio::test(start_paused = true)]
537556
async fn event_that_waited_before_dispatch_is_lost_not_delivered() {
538557
let state = crate::state::tests::test_state().await;
539-
let (conn, mut frames, _cancel) = presence_subscriber(&state);
558+
let (conn, mut frames, _cancel) = subscriber(&state, STORED);
540559
let (dispatcher, mut queues) = Dispatcher::new(1, 4);
541560

542-
let received = presence(0);
561+
let received = event(STORED, 0);
543562
tokio::time::advance(MAX_QUEUE_DELAY).await;
544563
dispatcher.dispatch(&state, received);
545564
let queued = queues[0].try_recv().expect("queued");
@@ -563,7 +582,7 @@ mod tests {
563582
)
564583
.await;
565584
let channel_id = Uuid::new_v4();
566-
let (conn, mut frames, _cancel) = presence_subscriber(&state);
585+
let (conn, mut frames, _cancel) = subscriber(&state, STORED);
567586
state.sub_registry.register(
568587
conn,
569588
"channel".to_string(),
@@ -599,7 +618,7 @@ mod tests {
599618
async fn each_close_batch_is_counted_as_it_closes() {
600619
let state = crate::state::tests::test_state().await;
601620
let subscribers: Vec<_> = (0..CLOSE_BATCH + 1)
602-
.map(|_| presence_subscriber(&state))
621+
.map(|_| subscriber(&state, STORED))
603622
.collect();
604623
let closer = LagCloser::default();
605624
closer.queue(&state, subscribers.iter().map(|(conn_id, _, _)| *conn_id));
@@ -621,4 +640,78 @@ mod tests {
621640
.sum();
622641
assert_eq!(counted, CLOSE_BATCH as u64);
623642
}
643+
644+
/// A lost ephemeral event cannot be replayed, so it is only counted: its
645+
/// recipients are neither fenced nor closed. A stored event lost on the
646+
/// same full queue still closes them.
647+
#[tokio::test(start_paused = true)]
648+
async fn lost_ephemeral_event_is_counted_and_closes_nothing() {
649+
let state = crate::state::tests::test_state().await;
650+
let (conn, mut frames, cancel) = subscriber(&state, PRESENCE);
651+
state.sub_registry.register(
652+
conn,
653+
"stored".to_string(),
654+
vec![Filter::new().kind(Kind::Custom(STORED))],
655+
None,
656+
);
657+
let (dispatcher, _queues) = Dispatcher::new(1, 1);
658+
dispatcher.dispatch(&state, event(PRESENCE, 0));
659+
660+
let recorder = metrics_util::debugging::DebuggingRecorder::new();
661+
let snapshotter = recorder.snapshotter();
662+
metrics::with_local_recorder(&recorder, || {
663+
dispatcher.dispatch(&state, event(PRESENCE, 1)); // queue full
664+
});
665+
let reasons: Vec<String> = snapshotter
666+
.snapshot()
667+
.into_vec()
668+
.into_iter()
669+
.filter(|(key, ..)| key.key().name() == "buzz_multinode_fanout_dropped_total")
670+
.flat_map(|(key, ..)| {
671+
key.key()
672+
.labels()
673+
.map(|label| label.value().to_string())
674+
.collect::<Vec<_>>()
675+
})
676+
.collect();
677+
assert_eq!(reasons, ["ephemeral"], "counted as an ephemeral drop");
678+
assert!(
679+
pending(&dispatcher).is_empty(),
680+
"a full queue closes nothing"
681+
);
682+
683+
let stale = event(PRESENCE, 2);
684+
tokio::time::advance(MAX_QUEUE_DELAY).await;
685+
deliver_or_lose(&state, &dispatcher.closer, Queued::new(stale)).await;
686+
assert!(frames.try_recv().is_err(), "still never delivered late");
687+
assert!(
688+
pending(&dispatcher).is_empty(),
689+
"a stale one closes nothing"
690+
);
691+
692+
assert!(
693+
worker_delivers(&state, &dispatcher.closer, &mut frames, event(PRESENCE, 3)).await,
694+
"nor fences the connection"
695+
);
696+
dispatcher.dispatch(&state, event(STORED, 4)); // queue full
697+
assert_eq!(pending(&dispatcher), HashSet::from([conn]));
698+
assert!(!cancel.is_cancelled());
699+
}
700+
701+
/// The fence holds on relay-local fan-out too, not only on the workers.
702+
#[tokio::test]
703+
async fn fence_withholds_local_fan_out() {
704+
let state = crate::state::tests::test_state().await;
705+
let (conn, mut frames, _cancel) = subscriber(&state, STORED);
706+
let local = |n| pubsub_stored_event(event(STORED, n));
707+
708+
fan_out_event_to_local_subscribers(&state, community(), &local(0)).await;
709+
assert!(frames.try_recv().is_ok(), "delivers before the fence");
710+
state.conn_manager.fence_live_delivery(conn);
711+
fan_out_event_to_local_subscribers(&state, community(), &local(1)).await;
712+
assert!(
713+
frames.try_recv().is_err(),
714+
"a fenced connection gets nothing"
715+
);
716+
}
624717
}

0 commit comments

Comments
 (0)