Skip to content

Commit 28168e9

Browse files
author
Elrond
committed
fix(relay): lock the community row FOR NO KEY UPDATE in owner mutations
The NIP-43 list insert in every membership transaction takes FOR KEY SHARE on the community row through events.community_id. Owner transfer and bootstrap locked that row FOR UPDATE before reaching the list lock or a member row, so a concurrent repair or removal deadlocked with them. FOR NO KEY UPDATE still serializes with archive, unarchive and deletion admission, but not with foreign-key child inserts. Startup now fans the newest backfill/bootstrap list out once pub/sub is ready, so subscribers on other replicas see an owner rotation. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz>
1 parent 146baae commit 28168e9

4 files changed

Lines changed: 198 additions & 18 deletions

File tree

‎crates/buzz-db/src/store/community.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -710,8 +710,8 @@ impl Db {
710710
/// Idempotently archives a community when the asserted pubkey is its current owner.
711711
///
712712
/// Locks the community row before reading ownership so a concurrent owner
713-
/// transfer, whose `FOR UPDATE` conflicts with this `FOR NO KEY UPDATE`,
714-
/// has one serial order with the archive. The weaker lock lets the archive
713+
/// transfer, whose `FOR NO KEY UPDATE` conflicts with this one, has one
714+
/// serial order with the archive. The weaker lock lets the archive
715715
/// proceed past foreign-key child inserts' `FOR KEY SHARE`. A single
716716
/// `UPDATE ... FROM relay_members` would keep its pre-lock ownership
717717
/// snapshot after waiting and could archive the new owner's community on

‎crates/buzz-db/src/store/relay_members.rs‎

Lines changed: 162 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -494,9 +494,14 @@ async fn lock_owner_mutation_admission(
494494
mode: OwnerMutationMode,
495495
) -> Result<OwnerMutationAdmission> {
496496
crate::deletion::lock_community_deletion_shared(tx, community).await?;
497+
// `FOR NO KEY UPDATE` still serializes with archive (`FOR NO KEY UPDATE`),
498+
// unarchive and deletion admission (`FOR UPDATE`), but not with the
499+
// `FOR KEY SHARE` that every foreign-key child insert takes. A membership
500+
// writer holding a member row or the list lock inserts its kind:13534
501+
// list into `events` last, so `FOR UPDATE` here would deadlock with it.
497502
let target = sqlx::query(
498503
"SELECT archived_at, deletion_state, deleted_at FROM communities \
499-
WHERE id = $1 FOR UPDATE",
504+
WHERE id = $1 FOR NO KEY UPDATE",
500505
)
501506
.bind(community.as_uuid())
502507
.fetch_optional(&mut **tx)
@@ -1056,9 +1061,12 @@ impl<T> MembershipWrite<T> {
10561061
///
10571062
/// Call it as the last statement before commit of any transaction that
10581063
/// changes `relay_members`, so the list commits or rolls back with the change.
1059-
/// The per-community list lock is taken last: every other lock a membership
1060-
/// transaction takes comes first, so the final writer reads every committed
1061-
/// change and no new lock order appears. `created_at` is at least one second
1064+
/// The per-community list lock is the last explicit lock: every other lock a
1065+
/// membership transaction takes comes first, so the final writer reads every
1066+
/// committed change. The list insert then takes `FOR KEY SHARE` on the
1067+
/// community row through `events.community_id`'s foreign key; owner mutations
1068+
/// lock that row `FOR NO KEY UPDATE`, which doesn't conflict, so no lock cycle
1069+
/// forms. `created_at` is at least one second
10621070
/// past every earlier list's, so the id is new and a client replacing by
10631071
/// timestamp never keeps an older list from the same second. A burst of
10641072
/// changes runs the timestamp ahead of the clock by up to one second per
@@ -2537,6 +2545,156 @@ mod postgres_tests {
25372545
assert_membership_list_matches(&pool, community, &keys, newest.as_ref()).await;
25382546
}
25392547

2548+
/// Wait until some backend is blocked behind `holder_pid`.
2549+
async fn wait_until_blocked_by(pool: &PgPool, holder_pid: i32) {
2550+
for _ in 0..400 {
2551+
let blocked: bool = sqlx::query_scalar(
2552+
"SELECT EXISTS(SELECT 1 FROM pg_stat_activity \
2553+
WHERE $1 = ANY(pg_blocking_pids(pid)))",
2554+
)
2555+
.bind(holder_pid)
2556+
.fetch_one(pool)
2557+
.await
2558+
.expect("read blocked backends");
2559+
if blocked {
2560+
return;
2561+
}
2562+
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
2563+
}
2564+
panic!("nothing blocked behind backend {holder_pid}");
2565+
}
2566+
2567+
/// Run `transfer_ownership` in the background.
2568+
fn spawn_transfer(
2569+
pool: &PgPool,
2570+
community: CommunityId,
2571+
new_owner: &str,
2572+
owner: &str,
2573+
keys: &nostr::Keys,
2574+
) -> tokio::task::JoinHandle<Result<MembershipWrite<TransferResult>>> {
2575+
let (pool, keys) = (pool.clone(), keys.clone());
2576+
let (new_owner, owner) = (new_owner.to_owned(), owner.to_owned());
2577+
tokio::spawn(async move {
2578+
transfer_ownership(&pool, community, &new_owner, &owner, &keys).await
2579+
})
2580+
}
2581+
2582+
/// A list repair that holds the list lock while a transfer waits on it
2583+
/// still inserts its list: the transfer's community row lock doesn't
2584+
/// block the insert's foreign-key `FOR KEY SHARE`, so both commit.
2585+
#[tokio::test]
2586+
#[ignore = "requires Postgres"]
2587+
async fn owner_transfer_and_list_repair_both_commit() {
2588+
let pool = setup_pool().await;
2589+
let keys = nostr::Keys::generate();
2590+
let community = make_test_community(&pool).await;
2591+
let owner = test_pubkey();
2592+
let new_owner = test_pubkey();
2593+
let _ = bootstrap_owner(&pool, community, &owner, &keys)
2594+
.await
2595+
.expect("bootstrap owner");
2596+
2597+
let mut repair = crate::begin_community_write_transaction(
2598+
&pool,
2599+
community,
2600+
observability::WriterOperation::Authorization,
2601+
)
2602+
.await
2603+
.expect("begin repair");
2604+
let repair_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
2605+
.fetch_one(repair.conn())
2606+
.await
2607+
.expect("repair pid");
2608+
let lock_key = replaceable::event_replacement_lock_key(
2609+
community,
2610+
buzz_core::kind::KIND_NIP43_MEMBERSHIP_LIST as i32,
2611+
keys.public_key().to_bytes().as_slice(),
2612+
None,
2613+
);
2614+
sqlx::query("SELECT pg_advisory_xact_lock($1)")
2615+
.bind(lock_key)
2616+
.execute(repair.conn())
2617+
.await
2618+
.expect("hold list lock");
2619+
2620+
// The transfer locks the community row, then waits on the list lock.
2621+
let transfer = spawn_transfer(&pool, community, &new_owner, &owner, &keys);
2622+
wait_until_blocked_by(&pool, repair_pid).await;
2623+
2624+
let repaired = tokio::time::timeout(std::time::Duration::from_secs(30), async {
2625+
write_nip43_membership_snapshot(&mut repair, &keys).await?;
2626+
repair.commit().await
2627+
})
2628+
.await
2629+
.expect("repair finishes");
2630+
repaired.expect("repair commits alongside the transfer");
2631+
let write = tokio::time::timeout(std::time::Duration::from_secs(30), transfer)
2632+
.await
2633+
.expect("transfer finishes")
2634+
.expect("transfer task")
2635+
.expect("transfer commits alongside the repair");
2636+
assert!(matches!(write.outcome, TransferResult::Transferred { .. }));
2637+
assert_role(&pool, community, &new_owner, "owner").await;
2638+
assert_membership_list_matches(&pool, community, &keys, write.snapshot.as_ref()).await;
2639+
}
2640+
2641+
/// Removing the transferee while a transfer waits on its member row:
2642+
/// the removal's list insert doesn't wait on the transfer's community
2643+
/// row lock, so the removal commits and the transfer then re-adds the
2644+
/// transferee as owner.
2645+
#[tokio::test]
2646+
#[ignore = "requires Postgres"]
2647+
async fn owner_transfer_and_member_removal_both_commit() {
2648+
let pool = setup_pool().await;
2649+
let keys = nostr::Keys::generate();
2650+
let community = make_test_community(&pool).await;
2651+
let owner = test_pubkey();
2652+
let member = test_pubkey();
2653+
let _ = bootstrap_owner(&pool, community, &owner, &keys)
2654+
.await
2655+
.expect("bootstrap owner");
2656+
let _ = add_relay_member(&pool, community, &member, "member", None, &keys)
2657+
.await
2658+
.expect("add member");
2659+
2660+
let mut removal = crate::begin_community_write_transaction(
2661+
&pool,
2662+
community,
2663+
observability::WriterOperation::Authorization,
2664+
)
2665+
.await
2666+
.expect("begin removal");
2667+
let removal_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
2668+
.fetch_one(removal.conn())
2669+
.await
2670+
.expect("removal pid");
2671+
assert!(delete_member_with_acceptances(&mut removal, &member, None)
2672+
.await
2673+
.expect("delete member"));
2674+
2675+
// The transfer locks the community row, then waits on the member row.
2676+
let transfer = spawn_transfer(&pool, community, &member, &owner, &keys);
2677+
wait_until_blocked_by(&pool, removal_pid).await;
2678+
2679+
let removed = tokio::time::timeout(
2680+
std::time::Duration::from_secs(30),
2681+
commit_membership_write(removal, RemoveResult::Removed, true, &keys),
2682+
)
2683+
.await
2684+
.expect("removal finishes")
2685+
.expect("removal commits alongside the transfer");
2686+
assert!(removed.snapshot.is_some());
2687+
let write = tokio::time::timeout(std::time::Duration::from_secs(30), transfer)
2688+
.await
2689+
.expect("transfer finishes")
2690+
.expect("transfer task")
2691+
.expect("transfer commits alongside the removal");
2692+
assert!(matches!(write.outcome, TransferResult::Transferred { .. }));
2693+
assert_role(&pool, community, &member, "owner").await;
2694+
assert_role(&pool, community, &owner, "member").await;
2695+
assert_membership_list_matches(&pool, community, &keys, write.snapshot.as_ref()).await;
2696+
}
2697+
25402698
/// Returning to an earlier membership within the same second yields a new
25412699
/// list with a later `created_at`, never the earlier list's id.
25422700
#[tokio::test]

‎crates/buzz-relay/src/main.rs‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,10 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
629629
{
630630
Ok(Some(record)) => {
631631
info!(host = %record.host, community = %record.id, "Deployment community ensured");
632-
Some(record.id)
632+
Some(buzz_core::tenant::TenantContext::resolved(
633+
record.id,
634+
record.host,
635+
))
633636
}
634637
Ok(None) => {
635638
step.degrade();
@@ -659,14 +662,19 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
659662
// is promoted (otherwise enabling membership locks everyone out).
660663
//
661664
// Both steps write the NIP-43 membership list in their own transaction.
662-
// The listener isn't up yet, so there is nobody to deliver it to live;
663-
// clients fetch the stored list.
664-
if let Some(community) = deployment_community {
665+
// This listener isn't up yet, but in a rolling deploy other replicas have
666+
// live subscribers; the newest list is fanned out once pub/sub is ready.
667+
let mut startup_membership_list = None;
668+
if let Some(tenant) = &deployment_community {
665669
match db
666-
.backfill_from_allowlist(community, &relay_keypair)
670+
.backfill_from_allowlist(tenant.community(), &relay_keypair)
667671
.await
668-
.map(|write| write.outcome)
669-
{
672+
.map(|write| {
673+
if write.snapshot.is_some() {
674+
startup_membership_list = write.snapshot;
675+
}
676+
write.outcome
677+
}) {
670678
Ok(0) => {}
671679
Ok(n) => info!("Backfilled {n} pubkey_allowlist entries into relay_members"),
672680
Err(e) => {
@@ -687,14 +695,18 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
687695

688696
// NIP-43: ensure the configured relay owner always holds the owner role
689697
// within the deployment community.
690-
if let (Some(community), Some(owner_pubkey)) =
691-
(deployment_community, config.relay_owner_pubkey.as_ref())
698+
if let (Some(tenant), Some(owner_pubkey)) =
699+
(&deployment_community, config.relay_owner_pubkey.as_ref())
692700
{
693701
match db
694-
.bootstrap_owner(community, owner_pubkey, &relay_keypair)
702+
.bootstrap_owner(tenant.community(), owner_pubkey, &relay_keypair)
695703
.await
696-
.map(|write| write.outcome)
697-
{
704+
.map(|write| {
705+
if write.snapshot.is_some() {
706+
startup_membership_list = write.snapshot;
707+
}
708+
write.outcome
709+
}) {
698710
Ok(()) => info!(pubkey = %owner_pubkey, "Relay owner bootstrapped"),
699711
Err(e) => {
700712
if config.require_relay_membership {
@@ -899,6 +911,14 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
899911
app_state.nip_fi_resolver = Some(resolver);
900912
}
901913
let state = Arc::new(app_state);
914+
if let Some(tenant) = &deployment_community {
915+
buzz_relay::handlers::side_effects::fan_out_nip43_membership_list(
916+
tenant,
917+
&state,
918+
startup_membership_list,
919+
)
920+
.await;
921+
}
902922
let has_startup_partition_audit = startup_partition_audit.is_some();
903923
if let Some(audit) = startup_partition_audit {
904924
state.record_partition_audit(audit);

‎scripts/seed-admin-dashboard.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ SELECT s.community_id, m.pubkey, m.display_name
290290
FROM seeded s JOIN seed_member m ON m.n = ANY (s.members)
291291
ON CONFLICT (community_id, pubkey) DO UPDATE SET display_name = EXCLUDED.display_name;
292292
293+
-- Raw rows bypass the NIP-43 member list; run `buzz-admin republish-members`
294+
-- per community to rebuild it.
293295
INSERT INTO relay_members (community_id, pubkey, role, added_by)
294296
SELECT s.community_id, encode(m.pubkey, 'hex'), 'member', NULL
295297
FROM seeded s JOIN seed_member m ON m.n = ANY (s.members)

0 commit comments

Comments
 (0)