Skip to content

Commit 77c4e96

Browse files
author
Elrond
committed
feat(relay): sweep NIP-43 snapshots on one leader replica only
Reuse the usage-metrics session advisory lock lease (renamed to the generic SessionLeader / try_lock_session_leader) so only the replica holding NIP43_RECONCILE_LOCK_KEY runs the sweep. Standbys retry the lock once per interval; Postgres releases it when the leader's session ends. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz>
1 parent e5582f7 commit 77c4e96

5 files changed

Lines changed: 141 additions & 36 deletions

File tree

‎crates/buzz-db/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,6 @@ pub use event::{EventQuery, DEFAULT_MAX_PAGE_LIMIT};
8585
pub use nip_fi_binding::{NipFiBindingRecord, NipFiUnbind, RowCapExceeded};
8686
pub use reaction::ReactionEventInsertOutcome;
8787
pub use reminder::DueReminder;
88-
pub use usage::UsageMetricsLeader;
88+
pub use usage::SessionLeader;
8989

9090
use buzz_core::CommunityId;

‎crates/buzz-db/src/store/usage.rs‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -195,16 +195,16 @@ where
195195
.await?)
196196
}
197197

198-
/// Owns the detached Postgres session holding the relay usage-metrics advisory lock.
198+
/// Owns the detached Postgres session holding a deployment-wide leader advisory lock.
199199
///
200200
/// The connection deliberately does not return to the main pool: session advisory
201-
/// locks must remain bound to this exact physical connection, and the poller
202-
/// pings it before each leader-only collection tick.
203-
pub struct UsageMetricsLeader {
201+
/// locks must remain bound to this exact physical connection, and each leader-only
202+
/// loop (usage metrics, NIP-43 snapshot reconciliation) pings it before working.
203+
pub struct SessionLeader {
204204
connection: PgConnection,
205205
}
206206

207-
impl UsageMetricsLeader {
207+
impl SessionLeader {
208208
/// Returns whether the lock-owning session is still reachable.
209209
///
210210
/// Bounded to 5 seconds — a blackholed connection (no RST) would otherwise
@@ -604,17 +604,15 @@ fn into_community_hosts(rows: Vec<(Uuid, String)>) -> Vec<CommunityHost> {
604604
}
605605

606606
impl Db {
607-
/// Try to acquire the detached session advisory lock for relay usage metrics.
607+
/// Try to acquire the detached session advisory lock `lock_key`, making this
608+
/// replica the leader for the job that key names.
608609
///
609610
/// The returned guard owns the exact connection that acquired the lock. It is
610611
/// detached from the shared pool so a stable leader neither returns a locked
611612
/// session to other callers nor permanently consumes a pool slot. Dropping the
612613
/// guard closes the connection and releases the session-scoped lock.
613-
#[datastore_span(name = "try_lock_usage_metrics", system = "postgresql")]
614-
pub async fn try_lock_usage_metrics(
615-
&self,
616-
lock_key: i64,
617-
) -> Result<Option<UsageMetricsLeader>> {
614+
#[datastore_span(name = "try_lock_session_leader", system = "postgresql")]
615+
pub async fn try_lock_session_leader(&self, lock_key: i64) -> Result<Option<SessionLeader>> {
618616
let mut connection = observability::acquire_writer_with_legacy_metrics(
619617
&self.pool,
620618
observability::WriterOperation::Maintenance,
@@ -625,7 +623,7 @@ impl Db {
625623
.fetch_one(&mut *connection)
626624
.await?;
627625
if acquired {
628-
Ok(Some(UsageMetricsLeader {
626+
Ok(Some(SessionLeader {
629627
connection: connection.detach(),
630628
}))
631629
} else {
@@ -907,14 +905,14 @@ mod postgres_tests {
907905
let key = 0x4255_5A5A_4D45_5452;
908906

909907
let mut leader = first
910-
.try_lock_usage_metrics(key)
908+
.try_lock_session_leader(key)
911909
.await
912910
.expect("first lock attempt")
913911
.expect("first database handle becomes leader");
914912
assert!(leader.is_live().await, "lock owner remains reachable");
915913
assert!(
916914
second
917-
.try_lock_usage_metrics(key)
915+
.try_lock_session_leader(key)
918916
.await
919917
.expect("second lock attempt")
920918
.is_none(),
@@ -927,7 +925,7 @@ mod postgres_tests {
927925
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
928926
let successor = loop {
929927
let attempt = second
930-
.try_lock_usage_metrics(key)
928+
.try_lock_session_leader(key)
931929
.await
932930
.expect("lock attempt after leader drop");
933931
if attempt.is_some() || tokio::time::Instant::now() >= deadline {

‎crates/buzz-db/tests/observability_source.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,7 @@ fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() {
472472
.next()
473473
.expect("usage production source");
474474
let usage_leader_lock = usage_production
475-
.split_once("pub async fn try_lock_usage_metrics(")
475+
.split_once("pub async fn try_lock_session_leader(")
476476
.expect("usage store must expose the legacy leader-lock acquisition")
477477
.1
478478
.split_once("pub async fn usage_community_count(")

‎crates/buzz-relay/src/handlers/side_effects.rs‎

Lines changed: 121 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3407,30 +3407,55 @@ pub async fn reconcile_nip43_membership_snapshots(state: &Arc<AppState>) -> anyh
34073407
Ok(reconciled)
34083408
}
34093409

3410+
/// Deployment-wide leader lock for the NIP-43 snapshot reconciler.
3411+
pub const NIP43_RECONCILE_LOCK_KEY: i64 = 0x4255_5A5A_4E34_3352;
3412+
34103413
/// Spawn the background NIP-43 snapshot reconciler.
34113414
///
3412-
/// The first sweep starts as soon as the task runs; each later sweep starts
3413-
/// `every` after the previous one finishes. Startup never waits on it: the
3414-
/// snapshots live in the shared database, so a new replica has nothing of its
3415-
/// own to repair before it serves. Trade-off: on a single-replica deployment, a
3416-
/// missing or stale kind:13534 snapshot is served until this replica's first
3417-
/// sweep reaches that community.
3415+
/// Only the replica holding the `lock_key` session advisory lock sweeps; the
3416+
/// others retry the lock once per `every`. The holder keeps the lock across
3417+
/// sweeps, and Postgres releases it when that session ends, so a killed leader
3418+
/// is replaced on a standby's next attempt. Each sweep (and each attempt to
3419+
/// lead) starts `every` after the previous one finished; the first runs as soon
3420+
/// as the task does. Startup never waits on it: the snapshots live in the shared
3421+
/// database, so a new replica has nothing of its own to repair before it serves.
3422+
/// Trade-off: a missing or stale kind:13534 snapshot is served until the
3423+
/// leader's sweep reaches that community.
34183424
pub fn spawn_nip43_membership_reconciler(
34193425
state: Arc<AppState>,
3426+
lock_key: i64,
34203427
every: std::time::Duration,
34213428
) -> tokio::task::JoinHandle<()> {
34223429
tokio::spawn(async move {
3423-
// Sweep first, then sleep: the first sweep starts immediately and
3424-
// every later one waits a full `every` after the previous sweep ends,
3425-
// even when a sweep outlasts `every`.
3430+
let mut leader: Option<buzz_db::SessionLeader> = None;
34263431
loop {
3427-
match reconcile_nip43_membership_snapshots(&state).await {
3428-
Ok(repaired) if repaired > 0 => {
3429-
info!(count = repaired, "NIP-43 membership snapshots repaired")
3432+
if let Some(guard) = leader.as_mut() {
3433+
if !guard.is_live().await {
3434+
warn!("NIP-43 reconciler leader lock connection lost; demoting");
3435+
leader = None;
3436+
}
3437+
}
3438+
if leader.is_none() {
3439+
match state.db.try_lock_session_leader(lock_key).await {
3440+
Ok(Some(guard)) => {
3441+
info!("Acquired NIP-43 reconciler leader lock");
3442+
leader = Some(guard);
3443+
}
3444+
Ok(None) => {}
3445+
Err(error) => {
3446+
warn!(%error, "NIP-43 reconciler leader lock attempt failed")
3447+
}
34303448
}
3431-
Ok(_) => {}
3432-
Err(error) => {
3433-
warn!(%error, "periodic NIP-43 membership snapshot reconciliation failed")
3449+
}
3450+
if leader.is_some() {
3451+
match reconcile_nip43_membership_snapshots(&state).await {
3452+
Ok(repaired) if repaired > 0 => {
3453+
info!(count = repaired, "NIP-43 membership snapshots repaired")
3454+
}
3455+
Ok(_) => {}
3456+
Err(error) => {
3457+
warn!(%error, "periodic NIP-43 membership snapshot reconciliation failed")
3458+
}
34343459
}
34353460
}
34363461
tokio::time::sleep(every).await;
@@ -4354,6 +4379,7 @@ mod tests {
43544379
// sweep can publish the snapshot.
43554380
let reconciler = spawn_nip43_membership_reconciler(
43564381
Arc::clone(&state),
4382+
scratch_lock_key(),
43574383
std::time::Duration::from_secs(24 * 60 * 60),
43584384
);
43594385
let published = tokio::time::timeout(std::time::Duration::from_secs(60), async {
@@ -4370,6 +4396,86 @@ mod tests {
43704396
assert_eq!(membership_snapshots(&pool, community).await, 1);
43714397
}
43724398

4399+
/// A per-test leader key, so concurrent tests and any relay sharing
4400+
/// the test database never contend for the production lock.
4401+
fn scratch_lock_key() -> i64 {
4402+
Uuid::new_v4().as_u64_pair().0 as i64
4403+
}
4404+
4405+
/// Only the leader-lock holder sweeps. A replica that can't take the
4406+
/// lock never publishes, and it takes over once the holder lets go.
4407+
/// Runs in a private database so no other test's sweep can publish
4408+
/// the snapshot this test watches.
4409+
#[tokio::test]
4410+
#[ignore = "requires Postgres"]
4411+
async fn background_reconciler_sweeps_only_while_holding_leader_lock() {
4412+
let admin_url = crate::test_support::database_url();
4413+
let admin = sqlx::PgPool::connect(&admin_url)
4414+
.await
4415+
.expect("connect admin to create scratch db");
4416+
let name = format!("nip43_leader_{}", Uuid::new_v4().simple());
4417+
sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}")))
4418+
.execute(&admin)
4419+
.await
4420+
.expect("create scratch db");
4421+
let slash = admin_url
4422+
.rfind('/')
4423+
.expect("URL must have a path component");
4424+
let pool = sqlx::PgPool::connect(&format!("{}/{name}", &admin_url[..slash]))
4425+
.await
4426+
.expect("connect scratch db");
4427+
buzz_db::migration::run_migrations(&pool)
4428+
.await
4429+
.expect("migrate scratch db");
4430+
let state = crate::state::tests::test_state_with_database_pool(pool.clone()).await;
4431+
let community = insert_community(&pool).await;
4432+
4433+
// Another "replica" holds the leader lock.
4434+
let key = scratch_lock_key();
4435+
let mut holder = pool.acquire().await.expect("acquire lock holder");
4436+
sqlx::query("SELECT pg_advisory_lock($1)")
4437+
.bind(key)
4438+
.execute(&mut *holder)
4439+
.await
4440+
.expect("take leader lock");
4441+
4442+
let every = std::time::Duration::from_millis(100);
4443+
let reconciler = spawn_nip43_membership_reconciler(Arc::clone(&state), key, every);
4444+
// Twenty lock attempts' worth of standby: none may sweep.
4445+
tokio::time::sleep(every * 20).await;
4446+
let standby_snapshots = membership_snapshots(&pool, community).await;
4447+
4448+
sqlx::query("SELECT pg_advisory_unlock($1)")
4449+
.bind(key)
4450+
.execute(&mut *holder)
4451+
.await
4452+
.expect("release leader lock");
4453+
let published = tokio::time::timeout(std::time::Duration::from_secs(60), async {
4454+
while membership_snapshots(&pool, community).await == 0 {
4455+
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
4456+
}
4457+
})
4458+
.await;
4459+
reconciler.abort();
4460+
drop(holder);
4461+
drop(state);
4462+
pool.close().await;
4463+
let _ = sqlx::query(sqlx::AssertSqlSafe(format!(
4464+
"DROP DATABASE IF EXISTS {name} WITH (FORCE)"
4465+
)))
4466+
.execute(&admin)
4467+
.await;
4468+
4469+
assert_eq!(
4470+
standby_snapshots, 0,
4471+
"a replica without the leader lock must not sweep"
4472+
);
4473+
assert!(
4474+
published.is_ok(),
4475+
"the standby must take over and sweep once the leader lock is released"
4476+
);
4477+
}
4478+
43734479
/// Regression for #7558 on the worker path: repeated NIP-43
43744480
/// maintenance sweeps reconcile the active community once and never
43754481
/// hand archived, quiescing, fenced, or tombstoned communities to the

‎crates/buzz-relay/src/main.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1096,8 +1096,8 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
10961096
step.finish();
10971097

10981098
// NIP-43: repair event-backed rosters (`relay_members` is canonical) in
1099-
// the background. The first sweep starts immediately but never blocks the
1100-
// listener; see `spawn_nip43_membership_reconciler`.
1099+
// the background, on the one replica holding the reconciler leader lock.
1100+
// It never blocks the listener; see `spawn_nip43_membership_reconciler`.
11011101
if config.require_relay_membership {
11021102
let interval_secs = std::env::var("BUZZ_NIP43_RECONCILE_INTERVAL_SECS")
11031103
.ok()
@@ -1106,6 +1106,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
11061106
.max(1);
11071107
buzz_relay::handlers::side_effects::spawn_nip43_membership_reconciler(
11081108
Arc::clone(&state),
1109+
buzz_relay::handlers::side_effects::NIP43_RECONCILE_LOCK_KEY,
11091110
std::time::Duration::from_secs(interval_secs),
11101111
);
11111112
}
@@ -2414,7 +2415,7 @@ fn dropped_in_memory_keys(
24142415
async fn run_usage_metrics_tick(
24152416
state: &AppState,
24162417
emission_scope: &EmissionScope,
2417-
leader: &mut Option<buzz_db::UsageMetricsLeader>,
2418+
leader: &mut Option<buzz_db::SessionLeader>,
24182419
emitted_in_memory: &mut HashSet<InMemoryMetricKey>,
24192420
fleet_schedule: &mut FleetUsageSchedule,
24202421
) -> anyhow::Result<()> {
@@ -2456,7 +2457,7 @@ async fn run_usage_metrics_tick(
24562457
if leader.is_none() && !demoted {
24572458
*leader = state
24582459
.db
2459-
.try_lock_usage_metrics(USAGE_METRICS_LOCK_KEY)
2460+
.try_lock_session_leader(USAGE_METRICS_LOCK_KEY)
24602461
.await?;
24612462
if leader.is_some() {
24622463
info!("Acquired usage metrics leader lock");

0 commit comments

Comments
 (0)