Skip to content

Commit 36d3d2e

Browse files
tlongwell-blocknpub1mprnacetjua2xx3p5eddmhxyk6wv929ymm5py8kd2xfxurxahspqqlgytaTyler Longwell
authored
Fix presence fan-out across relay pods (#1227)
Signed-off-by: Tyler Longwell <[email protected]> Co-authored-by: npub1mprnacetjua2xx3p5eddmhxyk6wv929ymm5py8kd2xfxurxahspqqlgyta <d8473ee32b973aa31a21a65adddcc4b69cc2a8a4dee8121ecd51926e0cddbc02@sprout-oss.stage.blox.sqprod.co> Co-authored-by: Tyler Longwell <[email protected]>
1 parent 142a5c9 commit 36d3d2e

2 files changed

Lines changed: 246 additions & 79 deletions

File tree

‎crates/buzz-relay/src/handlers/event.rs‎

Lines changed: 244 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,57 @@ pub async fn filter_fanout_by_access(
112112
allowed
113113
}
114114

115+
/// Fan out one event received from Redis pub/sub to this relay's local subscribers.
116+
pub async fn fan_out_pubsub_event(state: &Arc<AppState>, channel_event: buzz_pubsub::ChannelEvent) {
117+
// Nil UUID is the sentinel for channel-less global events (see
118+
// `handle_ephemeral_event`'s global branch). Convert back to None so
119+
// `fan_out()` uses the global subscriber index.
120+
let channel_id = if channel_event.channel_id.is_nil() {
121+
None
122+
} else {
123+
Some(channel_event.channel_id)
124+
};
125+
let stored = StoredEvent::new(channel_event.event, channel_id);
126+
127+
// Skip events that were already fanned out in-process (local echo). The
128+
// cache has TTL-based eviction (60s) so entries are bounded regardless of
129+
// subscriber health.
130+
let event_id_bytes = stored.event.id.to_bytes();
131+
if state.local_event_ids.get(&event_id_bytes).is_some() {
132+
state.local_event_ids.invalidate(&event_id_bytes);
133+
return;
134+
}
135+
136+
let matches = state.sub_registry.fan_out(&stored);
137+
let matches = filter_fanout_by_access(state, &stored, matches).await;
138+
metrics::counter!("buzz_multinode_fanout_total").increment(1);
139+
if matches.is_empty() {
140+
return;
141+
}
142+
143+
let event_json = match serde_json::to_string(&stored.event) {
144+
Ok(json) => json,
145+
Err(e) => {
146+
tracing::error!("Failed to serialize event for multi-node fan-out: {e}");
147+
return;
148+
}
149+
};
150+
let mut drop_count = 0u32;
151+
for (conn_id, sub_id) in &matches {
152+
let msg = format!(r#"["EVENT","{}",{}]"#, sub_id, event_json);
153+
if !state.conn_manager.send_to(*conn_id, msg) {
154+
drop_count += 1;
155+
}
156+
}
157+
if drop_count > 0 {
158+
tracing::warn!(
159+
event_id = %stored.event.id.to_hex(),
160+
drop_count,
161+
"multi-node fan-out: {drop_count} connection(s) dropped"
162+
);
163+
}
164+
}
165+
115166
/// Publish a stored event to subscribers and kick off async side effects.
116167
pub(crate) async fn dispatch_persistent_event(
117168
state: &Arc<AppState>,
@@ -474,28 +525,9 @@ async fn handle_ephemeral_event(
474525
let _ = state.pubsub.set_presence(&auth_pubkey, &status).await;
475526
}
476527

477-
let stored_event = StoredEvent::new(event.clone(), None);
478-
let matches = state.sub_registry.fan_out(&stored_event);
479-
metrics::histogram!("buzz_fanout_recipients").record(matches.len() as f64);
480-
let event_json = serde_json::to_string(&event)
481-
.expect("nostr::Event serialization is infallible for well-formed events");
482-
let mut drop_count = 0u32;
483-
for (target_conn_id, sub_id) in &matches {
484-
let msg = format!(r#"["EVENT","{}",{}]"#, sub_id, event_json);
485-
if !state.conn_manager.send_to(*target_conn_id, msg) {
486-
drop_count += 1;
487-
}
488-
}
489-
if drop_count > 0 {
490-
tracing::warn!(
491-
event_id = %event_id_hex,
492-
drop_count,
493-
"fan-out: {drop_count} connection(s) cancelled due to full/closed buffers"
494-
);
495-
}
496-
497-
conn.send(RelayMessage::ok(event_id_hex, true, ""));
498-
return;
528+
// Presence is a channel-less ephemeral event. After updating Redis
529+
// presence state, let it fall through to the shared global ephemeral
530+
// publish/fan-out path below so other relay nodes receive the live delta.
499531
}
500532

501533
// Mesh status report (kind:24620). An authenticated relay member reports its
@@ -1001,6 +1033,188 @@ mod tests {
10011033
assert!(err.contains("NIP-44"));
10021034
}
10031035

1036+
mod pubsub_fanout {
1037+
use std::collections::HashMap;
1038+
use std::sync::atomic::AtomicU8;
1039+
use std::sync::Arc;
1040+
1041+
use axum::extract::ws::Message;
1042+
use buzz_core::kind::KIND_PRESENCE_UPDATE;
1043+
use buzz_pubsub::ChannelEvent;
1044+
use nostr::{EventBuilder, Filter, Keys, Kind};
1045+
use tokio::sync::{mpsc, Mutex};
1046+
use tokio_util::sync::CancellationToken;
1047+
use uuid::Uuid;
1048+
1049+
use crate::handlers::event::fan_out_pubsub_event;
1050+
use crate::state::AppState;
1051+
1052+
async fn test_state() -> Arc<AppState> {
1053+
super::fanout_access::test_state().await
1054+
}
1055+
1056+
fn register_presence_sub(
1057+
state: &AppState,
1058+
sub_id: &str,
1059+
) -> (Uuid, mpsc::Receiver<Message>) {
1060+
let conn_id = Uuid::new_v4();
1061+
let (tx, rx) = mpsc::channel(10);
1062+
state.conn_manager.register(
1063+
conn_id,
1064+
tx,
1065+
CancellationToken::new(),
1066+
Arc::new(AtomicU8::new(0)),
1067+
Arc::new(Mutex::new(HashMap::new())),
1068+
);
1069+
state.sub_registry.register(
1070+
conn_id,
1071+
sub_id.to_string(),
1072+
vec![Filter::new().kind(Kind::Custom(KIND_PRESENCE_UPDATE as u16))],
1073+
None,
1074+
);
1075+
(conn_id, rx)
1076+
}
1077+
1078+
fn presence_event(status: &str) -> nostr::Event {
1079+
EventBuilder::new(Kind::Custom(KIND_PRESENCE_UPDATE as u16), status)
1080+
.sign_with_keys(&Keys::generate())
1081+
.expect("sign presence")
1082+
}
1083+
1084+
fn event_from_ws_message(msg: Message) -> nostr::Event {
1085+
let Message::Text(text) = msg else {
1086+
panic!("expected text ws message");
1087+
};
1088+
let v: serde_json::Value = serde_json::from_str(&text).expect("EVENT frame JSON");
1089+
assert_eq!(v[0], "EVENT");
1090+
serde_json::from_value(v[2].clone()).expect("nostr event")
1091+
}
1092+
1093+
#[tokio::test]
1094+
async fn global_presence_pubsub_event_fans_out_to_local_subscribers() {
1095+
let state = test_state().await;
1096+
let (_conn_id, mut rx) = register_presence_sub(&state, "presence");
1097+
let event = presence_event("online");
1098+
let event_id = event.id;
1099+
1100+
fan_out_pubsub_event(
1101+
&state,
1102+
ChannelEvent {
1103+
channel_id: Uuid::nil(),
1104+
event,
1105+
},
1106+
)
1107+
.await;
1108+
1109+
let delivered = event_from_ws_message(rx.try_recv().expect("presence delivered"));
1110+
assert_eq!(delivered.id, event_id);
1111+
assert!(rx.try_recv().is_err(), "presence is delivered once");
1112+
}
1113+
1114+
#[tokio::test]
1115+
async fn local_echo_presence_pubsub_event_is_not_delivered_twice() {
1116+
let state = test_state().await;
1117+
let (_conn_id, mut rx) = register_presence_sub(&state, "presence");
1118+
let event = presence_event("online");
1119+
1120+
state.mark_local_event(&event.id);
1121+
fan_out_pubsub_event(
1122+
&state,
1123+
ChannelEvent {
1124+
channel_id: Uuid::nil(),
1125+
event,
1126+
},
1127+
)
1128+
.await;
1129+
1130+
assert!(
1131+
rx.try_recv().is_err(),
1132+
"Redis echo of locally fanned-out presence must be suppressed"
1133+
);
1134+
}
1135+
1136+
async fn redis_url_if_available() -> Option<String> {
1137+
let redis_url =
1138+
std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1:6379".to_string());
1139+
let pool = deadpool_redis::Config::from_url(&redis_url)
1140+
.create_pool(Some(deadpool_redis::Runtime::Tokio1))
1141+
.ok()?;
1142+
let mut conn = pool.get().await.ok()?;
1143+
redis::cmd("PING")
1144+
.query_async::<String>(&mut conn)
1145+
.await
1146+
.ok()?;
1147+
Some(redis_url)
1148+
}
1149+
1150+
fn spawn_pubsub_fanout_loop(state: Arc<AppState>) -> tokio::task::JoinHandle<()> {
1151+
let mut rx = state.pubsub.subscribe_local();
1152+
tokio::spawn(async move {
1153+
while let Ok(channel_event) = rx.recv().await {
1154+
fan_out_pubsub_event(&state, channel_event).await;
1155+
}
1156+
})
1157+
}
1158+
1159+
#[tokio::test]
1160+
async fn redis_presence_publish_reaches_second_relay_and_suppresses_origin_echo() {
1161+
let Some(redis_url) = redis_url_if_available().await else {
1162+
eprintln!("skipping Redis round-trip presence fan-out test: Redis unavailable");
1163+
return;
1164+
};
1165+
1166+
let origin = super::fanout_access::test_state_with_redis_url(&redis_url).await;
1167+
let receiver = super::fanout_access::test_state_with_redis_url(&redis_url).await;
1168+
1169+
let origin_subscriber = tokio::spawn(origin.pubsub.clone().run_subscriber());
1170+
let receiver_subscriber = tokio::spawn(receiver.pubsub.clone().run_subscriber());
1171+
let origin_fanout = spawn_pubsub_fanout_loop(origin.clone());
1172+
let receiver_fanout = spawn_pubsub_fanout_loop(receiver.clone());
1173+
1174+
let (_origin_conn, mut origin_rx) = register_presence_sub(&origin, "origin-presence");
1175+
let (_receiver_conn, mut receiver_rx) =
1176+
register_presence_sub(&receiver, "receiver-presence");
1177+
1178+
// Match buzz-pubsub's own Redis round-trip test: give PSUBSCRIBE a
1179+
// bounded moment to attach before publishing the single test event.
1180+
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1181+
1182+
let event = presence_event("online");
1183+
let event_id = event.id;
1184+
origin.mark_local_event(&event.id);
1185+
origin
1186+
.pubsub
1187+
.publish_event(Uuid::nil(), &event)
1188+
.await
1189+
.expect("publish presence through Redis");
1190+
1191+
let delivered =
1192+
tokio::time::timeout(std::time::Duration::from_secs(2), receiver_rx.recv())
1193+
.await
1194+
.expect("presence reached second relay")
1195+
.expect("receiver connection still open");
1196+
let delivered = event_from_ws_message(delivered);
1197+
assert_eq!(delivered.id, event_id);
1198+
assert!(
1199+
tokio::time::timeout(std::time::Duration::from_millis(100), receiver_rx.recv())
1200+
.await
1201+
.is_err(),
1202+
"second relay receives the presence event exactly once"
1203+
);
1204+
assert!(
1205+
tokio::time::timeout(std::time::Duration::from_millis(250), origin_rx.recv())
1206+
.await
1207+
.is_err(),
1208+
"origin relay suppresses the Redis echo after local fan-out"
1209+
);
1210+
1211+
origin_subscriber.abort();
1212+
receiver_subscriber.abort();
1213+
origin_fanout.abort();
1214+
receiver_fanout.abort();
1215+
}
1216+
}
1217+
10041218
mod fanout_access {
10051219
use std::collections::HashMap;
10061220
use std::sync::atomic::AtomicU8;
@@ -1015,15 +1229,16 @@ mod tests {
10151229
use crate::handlers::event::filter_fanout_by_access;
10161230
use crate::state::AppState;
10171231

1018-
fn test_config() -> crate::config::Config {
1232+
pub(super) fn test_config() -> crate::config::Config {
10191233
let mut config = crate::config::Config::from_env().expect("default config loads");
10201234
config.require_relay_membership = false;
10211235
config.redis_url = "redis://127.0.0.1:1".to_string();
10221236
config
10231237
}
10241238

1025-
async fn test_state() -> Arc<AppState> {
1026-
let config = test_config();
1239+
pub(super) async fn test_state_with_redis_url(redis_url: &str) -> Arc<AppState> {
1240+
let mut config = test_config();
1241+
config.redis_url = redis_url.to_string();
10271242
let pool = sqlx::PgPool::connect_lazy(&config.database_url).expect("lazy pg pool");
10281243
let db = buzz_db::Db::from_pool(pool.clone());
10291244
let redis_pool = deadpool_redis::Config::from_url(&config.redis_url)
@@ -1062,6 +1277,10 @@ mod tests {
10621277
Arc::new(state)
10631278
}
10641279

1280+
pub(super) async fn test_state() -> Arc<AppState> {
1281+
test_state_with_redis_url("redis://127.0.0.1:1").await
1282+
}
1283+
10651284
fn register_conn(state: &AppState, pubkey: Option<Vec<u8>>) -> Uuid {
10661285
let conn_id = Uuid::new_v4();
10671286
let (tx, _rx) = mpsc::channel(1);

‎crates/buzz-relay/src/main.rs‎

Lines changed: 2 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -494,63 +494,11 @@ async fn main() -> anyhow::Result<()> {
494494
loop {
495495
match rx.recv().await {
496496
Ok(channel_event) => {
497-
// Nil UUID is the sentinel for channel-less global events
498-
// (see event.rs `else` branch). Convert back to None so
499-
// fan_out() uses the global subscriber index instead of
500-
// looking up subscribers under Some(Uuid::nil()), which
501-
// would find nothing and silently drop every cross-node
502-
// global event.
503-
let channel_id = if channel_event.channel_id.is_nil() {
504-
None
505-
} else {
506-
Some(channel_event.channel_id)
507-
};
508-
let stored = buzz_core::StoredEvent::new(channel_event.event, channel_id);
509-
510-
// Skip events that were already fanned out in-process (local echo).
511-
// The cache has TTL-based eviction (60s) so entries are bounded
512-
// regardless of subscriber health.
513-
let event_id_bytes = stored.event.id.to_bytes();
514-
if state_for_sub.local_event_ids.get(&event_id_bytes).is_some() {
515-
state_for_sub.local_event_ids.invalidate(&event_id_bytes);
516-
continue;
517-
}
518-
519-
let matches = state_for_sub.sub_registry.fan_out(&stored);
520-
let matches = buzz_relay::handlers::event::filter_fanout_by_access(
497+
buzz_relay::handlers::event::fan_out_pubsub_event(
521498
&state_for_sub,
522-
&stored,
523-
matches,
499+
channel_event,
524500
)
525501
.await;
526-
metrics::counter!("buzz_multinode_fanout_total").increment(1);
527-
if matches.is_empty() {
528-
continue;
529-
}
530-
531-
let event_json = match serde_json::to_string(&stored.event) {
532-
Ok(json) => json,
533-
Err(e) => {
534-
tracing::error!(
535-
"Failed to serialize event for multi-node fan-out: {e}"
536-
);
537-
continue;
538-
}
539-
};
540-
let mut drop_count = 0u32;
541-
for (conn_id, sub_id) in &matches {
542-
let msg = format!(r#"["EVENT","{}",{}]"#, sub_id, event_json);
543-
if !state_for_sub.conn_manager.send_to(*conn_id, msg) {
544-
drop_count += 1;
545-
}
546-
}
547-
if drop_count > 0 {
548-
tracing::warn!(
549-
event_id = %stored.event.id.to_hex(),
550-
drop_count,
551-
"multi-node fan-out: {drop_count} connection(s) dropped"
552-
);
553-
}
554502
}
555503
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
556504
metrics::counter!("buzz_multinode_fanout_lag_total").increment(n);

0 commit comments

Comments
 (0)