Skip to content

Commit 21f8dc6

Browse files
committed
Document serving lock order and verify artifact fence races
Signed-off-by: Ravneet Arora <[email protected]>
1 parent 4688052 commit 21f8dc6

2 files changed

Lines changed: 200 additions & 7 deletions

File tree

‎crates/buzz-db/src/store/artifact_postgres_tests.rs‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -698,3 +698,183 @@ async fn head_outside_authorized_channel_matches_missing_head() {
698698
ArtifactOutcome::Accepted(_)
699699
));
700700
}
701+
702+
/// Exercise the production Move transaction, including its second, relay-signed
703+
/// insert after domain row writes. Exclusive holders do not touch domain rows.
704+
#[tokio::test]
705+
#[ignore = "requires Postgres"]
706+
async fn artifact_move_and_identity_fences_finish_without_partial_writes_or_deadlocks() {
707+
use crate::user_deletion_fences::{self as fences, FenceIdentity};
708+
use std::{sync::Arc, time::Duration};
709+
for block_relay in [false, true] {
710+
let f = Arc::new(Fixture::new().await);
711+
let d = Uuid::new_v4();
712+
let create = f.revision(d, "create", f.a, None, &f.owner, vec![]);
713+
f.accept(&create, None).await;
714+
let moved = f.revision(d, "move", f.b, Some(&create), &f.owner, vec![]);
715+
let move_id = moved.id;
716+
let store = f.db.entity_deletion_store();
717+
let request =
718+
f.db.user_deletion_store()
719+
.submit(&format!("demo:{}", Uuid::new_v4()), "operator", None)
720+
.await
721+
.unwrap();
722+
store
723+
.approve(
724+
request.id,
725+
"reviewer",
726+
request.inventory_digest.as_deref().unwrap(),
727+
)
728+
.await
729+
.unwrap();
730+
let lease = store
731+
.claim(request.id, "worker", Duration::from_secs(300))
732+
.await
733+
.unwrap()
734+
.unwrap();
735+
let identity = FenceIdentity::user(f.owner.public_key());
736+
let mut gate = f.db.pool.begin().await.unwrap();
737+
if block_relay {
738+
fences::lock_exclusive(&mut gate, &[FenceIdentity::user(f.relay.public_key())])
739+
.await
740+
.unwrap();
741+
} else {
742+
fences::insert(&mut gate, identity, &lease, None)
743+
.await
744+
.unwrap();
745+
}
746+
let gate_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
747+
.fetch_one(&mut *gate)
748+
.await
749+
.unwrap();
750+
let task_f = f.clone();
751+
let task = tokio::spawn(async move {
752+
let env = artifact::validate(&moved).unwrap();
753+
task_f
754+
.db
755+
.accept_artifact(
756+
task_f.community,
757+
&moved,
758+
&env,
759+
Some(task_f.a),
760+
&task_f.relay,
761+
)
762+
.await
763+
});
764+
wait_for_artifact_blocker(&f.db.pool, gate_pid).await;
765+
let writer_pid: i32 = sqlx::query_scalar("SELECT pid FROM pg_stat_activity WHERE datname=current_database() AND $1=ANY(pg_blocking_pids(pid)) LIMIT 1")
766+
.bind(gate_pid).fetch_one(&f.db.pool).await.unwrap();
767+
let coordinate_held: bool = sqlx::query_scalar(
768+
"SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory' AND granted AND mode='ExclusiveLock'
769+
AND classid=((hashtextextended($2,0) >> 32) & 4294967295)::oid AND objid=(hashtextextended($2,0) & 4294967295)::oid)")
770+
.bind(writer_pid).bind(format!("artifact:{}:{}", f.community.as_uuid(), d)).fetch_one(&f.db.pool).await.unwrap();
771+
assert!(
772+
coordinate_held,
773+
"actual artifact caller must hold its domain lock before waiting"
774+
);
775+
let pending_user_fence = if block_relay {
776+
let mut user_gate = f.db.pool.begin().await.unwrap();
777+
let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
778+
.fetch_one(&mut *user_gate)
779+
.await
780+
.unwrap();
781+
let fence_task = tokio::spawn(async move {
782+
fences::insert(&mut user_gate, identity, &lease, None)
783+
.await
784+
.unwrap();
785+
user_gate.commit().await.unwrap();
786+
});
787+
tokio::time::timeout(Duration::from_secs(3), async {
788+
loop {
789+
let blockers: Vec<i32> = sqlx::query_scalar("SELECT pg_blocking_pids($1)")
790+
.bind(pid)
791+
.fetch_one(&f.db.pool)
792+
.await
793+
.unwrap();
794+
if blockers.contains(&writer_pid) {
795+
break;
796+
}
797+
tokio::time::sleep(Duration::from_millis(10)).await;
798+
}
799+
})
800+
.await
801+
.expect("fence must wait on Move's already acquired author lock");
802+
Some(fence_task)
803+
} else {
804+
None
805+
};
806+
gate.commit().await.unwrap();
807+
let outcome = tokio::time::timeout(Duration::from_secs(8), task)
808+
.await
809+
.unwrap()
810+
.unwrap();
811+
if let Some(task) = pending_user_fence {
812+
tokio::time::timeout(Duration::from_secs(8), task)
813+
.await
814+
.unwrap()
815+
.unwrap();
816+
}
817+
if block_relay {
818+
let ArtifactOutcome::Accepted(events) = outcome.unwrap() else {
819+
panic!("Move should finish before queued author fence");
820+
};
821+
assert_eq!(events.len(), 2);
822+
assert_eq!(events[1].event.kind.as_u16(), 45011);
823+
} else {
824+
assert!(matches!(
825+
outcome,
826+
Err(crate::DbError::UserDeletionRestricted)
827+
));
828+
}
829+
let head: Vec<u8> = sqlx::query_scalar(
830+
"SELECT event_id FROM artifact_heads WHERE community_id=$1 AND artifact_id=$2",
831+
)
832+
.bind(f.community.as_uuid())
833+
.bind(d)
834+
.fetch_one(&f.db.pool)
835+
.await
836+
.unwrap();
837+
assert_eq!(
838+
head,
839+
if block_relay {
840+
move_id.as_bytes()
841+
} else {
842+
create.id.as_bytes()
843+
}
844+
);
845+
let revision_exists: bool = sqlx::query_scalar(
846+
"SELECT EXISTS(SELECT 1 FROM artifact_revisions WHERE community_id=$1 AND event_id=$2)",
847+
)
848+
.bind(f.community.as_uuid())
849+
.bind(move_id.as_bytes().as_slice())
850+
.fetch_one(&f.db.pool)
851+
.await
852+
.unwrap();
853+
assert_eq!(revision_exists, block_relay);
854+
let removals: i64 =
855+
sqlx::query_scalar("SELECT count(*) FROM events WHERE community_id=$1 AND kind=45011")
856+
.bind(f.community.as_uuid())
857+
.fetch_one(&f.db.pool)
858+
.await
859+
.unwrap();
860+
assert_eq!(removals, i64::from(block_relay));
861+
let deadlocks: i64 = sqlx::query_scalar(
862+
"SELECT deadlocks FROM pg_stat_database WHERE datname=current_database()",
863+
)
864+
.fetch_one(&f.db.pool)
865+
.await
866+
.unwrap();
867+
assert_eq!(deadlocks, 0);
868+
}
869+
}
870+
871+
async fn wait_for_artifact_blocker(pool: &PgPool, pid: i32) {
872+
tokio::time::timeout(std::time::Duration::from_secs(3), async {
873+
loop {
874+
let waiting: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND $1=ANY(pg_blocking_pids(pid)))")
875+
.bind(pid).fetch_one(pool).await.unwrap();
876+
if waiting { return; }
877+
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
878+
}
879+
}).await.expect("artifact writer must wait on identity lock");
880+
}

‎crates/buzz-db/src/store/user_deletion_fences.rs‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,26 @@
11
//! Operator-global restriction storage and identity locks. Shared event insertion
22
//! paths consult these restrictions; the real user-deletion worker is not integrated.
33
//!
4-
//! Callers use a writer-pool READ COMMITTED transaction. Acquire existing
5-
//! community guards first, then `lock_request` for the worker, then all human
6-
//! locks, then scoped agent locks.
7-
//! Within each group acquire the whole set in one call (sorted here), never
8-
//! upgrade shared locks to exclusive. Gate workers may discover agents after
9-
//! locking humans, then lock the complete agent set. Keep the transaction open
10-
//! through the protected write/checkpoint; commit or rollback releases the locks.
4+
//! Callers use writer-pool READ COMMITTED transactions. Community admission
5+
//! precedes identity locking. Each lock call sorts global keys before scoped
6+
//! keys, but this is NOT a transaction-wide order for serving writes: artifact
7+
//! and channel writers can already hold domain row/advisory locks, and a later
8+
//! relay-signed event can acquire another global identity lock in the same tx.
9+
//!
10+
//! Exclusive fencing transactions must lock the request first, acquire their
11+
//! complete global identity set before their complete scoped set, then persist
12+
//! only restriction/request/checkpoint state. They must not lock or mutate
13+
//! users, channels, artifacts, events, or other serving-domain rows while
14+
//! holding exclusive identity locks. Perform cleanup in later transactions.
15+
//! Otherwise a writer holding a domain lock and waiting for a shared identity
16+
//! lock can cycle with the fencing transaction. Ownership discovery under the
17+
//! fence uses ordinary reads, not FOR UPDATE or owner mutation.
18+
//!
19+
//! Never upgrade shared identity locks to exclusive. Before enabling a worker,
20+
//! audit its complete exclusive sets against multi-author event transactions;
21+
//! per-call sorting alone does not prove arbitrary multi-key batches deadlock
22+
//! free. Keep locks through the protected write/checkpoint; commit or rollback
23+
//! releases them. The real deletion worker remains disabled.
1124
1225
use buzz_core::CommunityId;
1326
use chrono::{DateTime, Utc};

0 commit comments

Comments
 (0)