1515//! broadcast, it cannot tell which events it skipped, so it does the same for
1616//! every subscribed connection.
1717//!
18+ //! An ephemeral event (presence, typing, observer frames) is never stored, so
19+ //! no reconnect can replay it. Losing one is only counted: closing its
20+ //! recipients would recover nothing, and a global presence event reaches most
21+ //! of a community's connections.
22+ //!
1823//! A client resumes from the newest `created_at` it received, minus a few
1924//! seconds of slack. Recovery depends on that cursor never getting further
2025//! ahead of a lost event than the slack:
@@ -44,6 +49,7 @@ use std::hash::{DefaultHasher, Hash, Hasher};
4449use std:: sync:: { Arc , Mutex , PoisonError } ;
4550use std:: time:: Duration ;
4651
52+ use buzz_core:: kind:: { event_kind_u32, is_ephemeral} ;
4753use buzz_core:: CommunityId ;
4854use buzz_pubsub:: ChannelEvent ;
4955use tokio:: sync:: mpsc:: error:: TrySendError ;
@@ -139,14 +145,20 @@ async fn deliver_or_lose(state: &Arc<AppState>, closer: &LagCloser, queued: Queu
139145/// One known event will not be delivered: resync the syncs and fence and
140146/// close the connections it would have reached. Matching skips the access
141147/// gate, so a recipient that would have been filtered out may be closed too;
142- /// that costs a reconnect, never a leak.
148+ /// that costs a reconnect, never a leak. A lost ephemeral event is only
149+ /// counted, since no reconnect can replay it.
143150fn lose (
144151 state : & AppState ,
145152 closer : & LagCloser ,
146153 community_id : CommunityId ,
147154 stored : & StoredEvent ,
148155 reason : & ' static str ,
149156) {
157+ if is_ephemeral ( event_kind_u32 ( & stored. event ) ) {
158+ metrics:: counter!( "buzz_multinode_fanout_dropped_total" , "reason" => "ephemeral" )
159+ . increment ( 1 ) ;
160+ return ;
161+ }
150162 metrics:: counter!( "buzz_multinode_fanout_dropped_total" , "reason" => reason) . increment ( 1 ) ;
151163 let matches = state. sub_registry . fan_out_scoped ( community_id, stored) ;
152164 state. artifact_syncs . resync_matches ( & matches, LAG_RESYNC ) ;
@@ -301,15 +313,22 @@ mod tests {
301313 use uuid:: Uuid ;
302314
303315 use super :: * ;
316+ use crate :: handlers:: event:: fan_out_event_to_local_subscribers;
304317
305318 fn community ( ) -> CommunityId {
306319 CommunityId :: from_uuid ( Uuid :: nil ( ) )
307320 }
308321
309- /// A connection with one presence subscription, its frame receiver, and
322+ /// A stored kind: a lost event of it can be replayed, so its loss closes.
323+ const STORED : u16 = 1 ;
324+ /// An ephemeral kind: a lost event of it cannot be replayed.
325+ const PRESENCE : u16 = KIND_PRESENCE_UPDATE as u16 ;
326+
327+ /// A connection with one subscription to `kind`, its frame receiver, and
310328 /// its cancellation token.
311- fn presence_subscriber (
329+ fn subscriber (
312330 state : & AppState ,
331+ kind : u16 ,
313332 ) -> ( ConnId , mpsc:: Receiver < Message > , CancellationToken ) {
314333 let conn_id = Uuid :: new_v4 ( ) ;
315334 let ( tx, rx) = mpsc:: channel ( 64 ) ;
@@ -329,20 +348,20 @@ mod tests {
329348 ) ;
330349 state. sub_registry . register (
331350 conn_id,
332- "presence " . to_string ( ) ,
333- vec ! [ Filter :: new( ) . kind( Kind :: Custom ( KIND_PRESENCE_UPDATE as u16 ) ) ] ,
351+ "sub " . to_string ( ) ,
352+ vec ! [ Filter :: new( ) . kind( Kind :: Custom ( kind ) ) ] ,
334353 None ,
335354 ) ;
336355 ( conn_id, rx, cancel)
337356 }
338357
339- fn presence ( n : usize ) -> ChannelEvent {
358+ fn event ( kind : u16 , n : usize ) -> ChannelEvent {
340359 ChannelEvent {
341360 community_id : community ( ) ,
342361 topic : EventTopic :: Global ,
343- event : EventBuilder :: new ( Kind :: Custom ( KIND_PRESENCE_UPDATE as u16 ) , n. to_string ( ) )
362+ event : EventBuilder :: new ( Kind :: Custom ( kind ) , n. to_string ( ) )
344363 . sign_with_keys ( & Keys :: generate ( ) )
345- . expect ( "sign presence " ) ,
364+ . expect ( "sign event " ) ,
346365 received_at : Instant :: now ( ) ,
347366 }
348367 }
@@ -355,11 +374,11 @@ mod tests {
355374 #[ tokio:: test]
356375 async fn delivers_one_topic_in_order_through_workers ( ) {
357376 let state = crate :: state:: tests:: test_state ( ) . await ;
358- let ( _conn, mut frames, cancel) = presence_subscriber ( & state) ;
377+ let ( _conn, mut frames, cancel) = subscriber ( & state, STORED ) ;
359378 let ( tx, rx) = broadcast:: channel ( 64 ) ;
360379 let handle = spawn ( Arc :: clone ( & state) , rx) ;
361380
362- let sent: Vec < _ > = ( 0 ..20 ) . map ( presence ) . collect ( ) ;
381+ let sent: Vec < _ > = ( 0 ..20 ) . map ( |n| event ( STORED , n ) ) . collect ( ) ;
363382 for event in & sent {
364383 tx. send ( event. clone ( ) ) . expect ( "broadcast" ) ;
365384 }
@@ -386,20 +405,20 @@ mod tests {
386405 #[ tokio:: test]
387406 async fn full_worker_queue_closes_the_dropped_events_recipients ( ) {
388407 let state = crate :: state:: tests:: test_state ( ) . await ;
389- let ( reached, _frames, _cancel) = presence_subscriber ( & state) ;
408+ let ( reached, _frames, _cancel) = subscriber ( & state, STORED ) ;
390409 let other = Uuid :: new_v4 ( ) ;
391410 state. sub_registry . register (
392411 other,
393412 "other-kind" . to_string ( ) ,
394- vec ! [ Filter :: new( ) . kind( Kind :: Custom ( 1 ) ) ] ,
413+ vec ! [ Filter :: new( ) . kind( Kind :: Custom ( 7 ) ) ] ,
395414 None ,
396415 ) ;
397416 // Hold the only queue's receiver without draining it.
398417 let ( dispatcher, _queues) = Dispatcher :: new ( 1 , 1 ) ;
399418
400- dispatcher. dispatch ( & state, presence ( 0 ) ) ;
419+ dispatcher. dispatch ( & state, event ( STORED , 0 ) ) ;
401420 assert ! ( pending( & dispatcher) . is_empty( ) , "the first event fits" ) ;
402- dispatcher. dispatch ( & state, presence ( 1 ) ) ;
421+ dispatcher. dispatch ( & state, event ( STORED , 1 ) ) ;
403422 assert_eq ! ( pending( & dispatcher) , HashSet :: from( [ reached] ) ) ;
404423 }
405424
@@ -408,11 +427,11 @@ mod tests {
408427 #[ tokio:: test]
409428 async fn local_echo_on_a_full_queue_closes_nothing ( ) {
410429 let state = crate :: state:: tests:: test_state ( ) . await ;
411- let _subscriber = presence_subscriber ( & state) ;
430+ let _subscriber = subscriber ( & state, STORED ) ;
412431 let ( dispatcher, _queues) = Dispatcher :: new ( 1 , 1 ) ;
413- dispatcher. dispatch ( & state, presence ( 0 ) ) ;
432+ dispatcher. dispatch ( & state, event ( STORED , 0 ) ) ;
414433
415- let echo = presence ( 1 ) ;
434+ let echo = event ( STORED , 1 ) ;
416435 state. mark_local_event ( echo. community_id , & echo. event . id ) ;
417436 dispatcher. dispatch ( & state, echo) ;
418437 assert ! ( pending( & dispatcher) . is_empty( ) ) ;
@@ -422,8 +441,8 @@ mod tests {
422441 #[ tokio:: test]
423442 async fn dispatcher_lag_closes_every_subscribed_connection ( ) {
424443 let state = crate :: state:: tests:: test_state ( ) . await ;
425- let ( first, _f1, _c1) = presence_subscriber ( & state) ;
426- let ( second, _f2, _c2) = presence_subscriber ( & state) ;
444+ let ( first, _f1, _c1) = subscriber ( & state, STORED ) ;
445+ let ( second, _f2, _c2) = subscriber ( & state, STORED ) ;
427446 let ( dispatcher, _queues) = Dispatcher :: new ( 1 , 1 ) ;
428447
429448 dispatcher. lagged ( & state, 7 ) ;
@@ -435,7 +454,7 @@ mod tests {
435454 async fn closer_cancels_queued_connections_in_paced_batches ( ) {
436455 let state = crate :: state:: tests:: test_state ( ) . await ;
437456 let subscribers: Vec < _ > = ( 0 ..CLOSE_BATCH + 1 )
438- . map ( |_| presence_subscriber ( & state) )
457+ . map ( |_| subscriber ( & state, STORED ) )
439458 . collect ( ) ;
440459 let closer = Arc :: new ( LagCloser :: default ( ) ) ;
441460 tokio:: spawn ( Arc :: clone ( & closer) . run ( Arc :: clone ( & state) ) ) ;
@@ -470,17 +489,17 @@ mod tests {
470489 #[ tokio:: test]
471490 async fn dropped_event_fences_its_recipients_before_their_close ( ) {
472491 let state = crate :: state:: tests:: test_state ( ) . await ;
473- let ( _conn, mut frames, cancel) = presence_subscriber ( & state) ;
492+ let ( _conn, mut frames, cancel) = subscriber ( & state, STORED ) ;
474493 let ( dispatcher, _queues) = Dispatcher :: new ( 1 , 1 ) ;
475494 assert ! (
476- worker_delivers( & state, & dispatcher. closer, & mut frames, presence ( 0 ) ) . await ,
495+ worker_delivers( & state, & dispatcher. closer, & mut frames, event ( STORED , 0 ) ) . await ,
477496 "delivers before any loss"
478497 ) ;
479- dispatcher. dispatch ( & state, presence ( 1 ) ) ;
480- dispatcher. dispatch ( & state, presence ( 2 ) ) ; // dropped
498+ dispatcher. dispatch ( & state, event ( STORED , 1 ) ) ;
499+ dispatcher. dispatch ( & state, event ( STORED , 2 ) ) ; // dropped
481500
482501 assert ! (
483- !worker_delivers( & state, & dispatcher. closer, & mut frames, presence ( 3 ) ) . await ,
502+ !worker_delivers( & state, & dispatcher. closer, & mut frames, event ( STORED , 3 ) ) . await ,
484503 "an event after the loss must not reach the fenced connection"
485504 ) ;
486505 assert ! (
@@ -493,11 +512,11 @@ mod tests {
493512 #[ tokio:: test]
494513 async fn dispatcher_lag_fences_every_subscribed_connection ( ) {
495514 let state = crate :: state:: tests:: test_state ( ) . await ;
496- let ( _conn, mut frames, cancel) = presence_subscriber ( & state) ;
515+ let ( _conn, mut frames, cancel) = subscriber ( & state, STORED ) ;
497516 let ( dispatcher, _queues) = Dispatcher :: new ( 1 , 1 ) ;
498517 dispatcher. lagged ( & state, 3 ) ;
499518
500- assert ! ( !worker_delivers( & state, & dispatcher. closer, & mut frames, presence ( 0 ) ) . await ) ;
519+ assert ! ( !worker_delivers( & state, & dispatcher. closer, & mut frames, event ( STORED , 0 ) ) . await ) ;
501520 assert ! ( !cancel. is_cancelled( ) ) ;
502521 }
503522
@@ -508,24 +527,24 @@ mod tests {
508527 #[ tokio:: test( start_paused = true ) ]
509528 async fn backlogged_event_is_delivered_by_its_deadline_or_lost ( ) {
510529 let state = crate :: state:: tests:: test_state ( ) . await ;
511- let ( conn, mut frames, cancel) = presence_subscriber ( & state) ;
530+ let ( conn, mut frames, cancel) = subscriber ( & state, STORED ) ;
512531 let ( dispatcher, mut queues) = Dispatcher :: new ( 1 , 4 ) ;
513532 let queue = & mut queues[ 0 ] ;
514533
515- dispatcher. dispatch ( & state, presence ( 0 ) ) ;
534+ dispatcher. dispatch ( & state, event ( STORED , 0 ) ) ;
516535 tokio:: time:: advance ( MAX_QUEUE_DELAY - Duration :: from_millis ( 1 ) ) . await ;
517536 let in_time = queue. try_recv ( ) . expect ( "queued" ) ;
518537 deliver_or_lose ( & state, & dispatcher. closer , in_time) . await ;
519538 assert ! ( frames. try_recv( ) . is_ok( ) , "delivered inside its deadline" ) ;
520539
521- dispatcher. dispatch ( & state, presence ( 1 ) ) ; // R
540+ dispatcher. dispatch ( & state, event ( STORED , 1 ) ) ; // R
522541 tokio:: time:: advance ( MAX_QUEUE_DELAY ) . await ;
523542 let late = queue. try_recv ( ) . expect ( "queued" ) ;
524543 deliver_or_lose ( & state, & dispatcher. closer , late) . await ;
525544 assert ! ( frames. try_recv( ) . is_err( ) , "R is not delivered late" ) ;
526545 assert_eq ! ( pending( & dispatcher) , HashSet :: from( [ conn] ) ) ;
527546 assert ! (
528- !worker_delivers( & state, & dispatcher. closer, & mut frames, presence ( 2 ) ) . await ,
547+ !worker_delivers( & state, & dispatcher. closer, & mut frames, event ( STORED , 2 ) ) . await ,
529548 "R's recipient is fenced until it closes"
530549 ) ;
531550 assert ! ( !cancel. is_cancelled( ) , "the close waits for the closer" ) ;
@@ -536,10 +555,10 @@ mod tests {
536555 #[ tokio:: test( start_paused = true ) ]
537556 async fn event_that_waited_before_dispatch_is_lost_not_delivered ( ) {
538557 let state = crate :: state:: tests:: test_state ( ) . await ;
539- let ( conn, mut frames, _cancel) = presence_subscriber ( & state) ;
558+ let ( conn, mut frames, _cancel) = subscriber ( & state, STORED ) ;
540559 let ( dispatcher, mut queues) = Dispatcher :: new ( 1 , 4 ) ;
541560
542- let received = presence ( 0 ) ;
561+ let received = event ( STORED , 0 ) ;
543562 tokio:: time:: advance ( MAX_QUEUE_DELAY ) . await ;
544563 dispatcher. dispatch ( & state, received) ;
545564 let queued = queues[ 0 ] . try_recv ( ) . expect ( "queued" ) ;
@@ -563,7 +582,7 @@ mod tests {
563582 )
564583 . await ;
565584 let channel_id = Uuid :: new_v4 ( ) ;
566- let ( conn, mut frames, _cancel) = presence_subscriber ( & state) ;
585+ let ( conn, mut frames, _cancel) = subscriber ( & state, STORED ) ;
567586 state. sub_registry . register (
568587 conn,
569588 "channel" . to_string ( ) ,
@@ -599,7 +618,7 @@ mod tests {
599618 async fn each_close_batch_is_counted_as_it_closes ( ) {
600619 let state = crate :: state:: tests:: test_state ( ) . await ;
601620 let subscribers: Vec < _ > = ( 0 ..CLOSE_BATCH + 1 )
602- . map ( |_| presence_subscriber ( & state) )
621+ . map ( |_| subscriber ( & state, STORED ) )
603622 . collect ( ) ;
604623 let closer = LagCloser :: default ( ) ;
605624 closer. queue ( & state, subscribers. iter ( ) . map ( |( conn_id, _, _) | * conn_id) ) ;
@@ -621,4 +640,78 @@ mod tests {
621640 . sum ( ) ;
622641 assert_eq ! ( counted, CLOSE_BATCH as u64 ) ;
623642 }
643+
644+ /// A lost ephemeral event cannot be replayed, so it is only counted: its
645+ /// recipients are neither fenced nor closed. A stored event lost on the
646+ /// same full queue still closes them.
647+ #[ tokio:: test( start_paused = true ) ]
648+ async fn lost_ephemeral_event_is_counted_and_closes_nothing ( ) {
649+ let state = crate :: state:: tests:: test_state ( ) . await ;
650+ let ( conn, mut frames, cancel) = subscriber ( & state, PRESENCE ) ;
651+ state. sub_registry . register (
652+ conn,
653+ "stored" . to_string ( ) ,
654+ vec ! [ Filter :: new( ) . kind( Kind :: Custom ( STORED ) ) ] ,
655+ None ,
656+ ) ;
657+ let ( dispatcher, _queues) = Dispatcher :: new ( 1 , 1 ) ;
658+ dispatcher. dispatch ( & state, event ( PRESENCE , 0 ) ) ;
659+
660+ let recorder = metrics_util:: debugging:: DebuggingRecorder :: new ( ) ;
661+ let snapshotter = recorder. snapshotter ( ) ;
662+ metrics:: with_local_recorder ( & recorder, || {
663+ dispatcher. dispatch ( & state, event ( PRESENCE , 1 ) ) ; // queue full
664+ } ) ;
665+ let reasons: Vec < String > = snapshotter
666+ . snapshot ( )
667+ . into_vec ( )
668+ . into_iter ( )
669+ . filter ( |( key, ..) | key. key ( ) . name ( ) == "buzz_multinode_fanout_dropped_total" )
670+ . flat_map ( |( key, ..) | {
671+ key. key ( )
672+ . labels ( )
673+ . map ( |label| label. value ( ) . to_string ( ) )
674+ . collect :: < Vec < _ > > ( )
675+ } )
676+ . collect ( ) ;
677+ assert_eq ! ( reasons, [ "ephemeral" ] , "counted as an ephemeral drop" ) ;
678+ assert ! (
679+ pending( & dispatcher) . is_empty( ) ,
680+ "a full queue closes nothing"
681+ ) ;
682+
683+ let stale = event ( PRESENCE , 2 ) ;
684+ tokio:: time:: advance ( MAX_QUEUE_DELAY ) . await ;
685+ deliver_or_lose ( & state, & dispatcher. closer , Queued :: new ( stale) ) . await ;
686+ assert ! ( frames. try_recv( ) . is_err( ) , "still never delivered late" ) ;
687+ assert ! (
688+ pending( & dispatcher) . is_empty( ) ,
689+ "a stale one closes nothing"
690+ ) ;
691+
692+ assert ! (
693+ worker_delivers( & state, & dispatcher. closer, & mut frames, event( PRESENCE , 3 ) ) . await ,
694+ "nor fences the connection"
695+ ) ;
696+ dispatcher. dispatch ( & state, event ( STORED , 4 ) ) ; // queue full
697+ assert_eq ! ( pending( & dispatcher) , HashSet :: from( [ conn] ) ) ;
698+ assert ! ( !cancel. is_cancelled( ) ) ;
699+ }
700+
701+ /// The fence holds on relay-local fan-out too, not only on the workers.
702+ #[ tokio:: test]
703+ async fn fence_withholds_local_fan_out ( ) {
704+ let state = crate :: state:: tests:: test_state ( ) . await ;
705+ let ( conn, mut frames, _cancel) = subscriber ( & state, STORED ) ;
706+ let local = |n| pubsub_stored_event ( event ( STORED , n) ) ;
707+
708+ fan_out_event_to_local_subscribers ( & state, community ( ) , & local ( 0 ) ) . await ;
709+ assert ! ( frames. try_recv( ) . is_ok( ) , "delivers before the fence" ) ;
710+ state. conn_manager . fence_live_delivery ( conn) ;
711+ fan_out_event_to_local_subscribers ( & state, community ( ) , & local ( 1 ) ) . await ;
712+ assert ! (
713+ frames. try_recv( ) . is_err( ) ,
714+ "a fenced connection gets nothing"
715+ ) ;
716+ }
624717}
0 commit comments