Skip to content

Commit 7b6b199

Browse files
Ian Oberstclaude
andcommitted
Re-read foreign-key counterparts under the parent lock
The counterpart set was read before the parent lock, and the locked re-audit checks only the partition layout. A foreign key on or to the parent that committed while maintenance waited for the parent lock was missing from the set, so attaching a partition waited on its table without NOWAIT while holding the parent ACCESS EXCLUSIVE, the stall the lock order is meant to rule out. Read the counterparts again once the parent is locked and use that set. Foreign-key DDL on or to the parent conflicts with the parent lock, so the set is stable from there. The pre-lock read remains as the cheap refusal. A new test commits a foreign key while maintenance waits on the parent, with a writer on the new counterpart: maintenance now fails fast with lock_timeout instead of waiting out the 2 s lock timeout under the parent. Also document why dropping the catch-all needs no stronger counterpart lock. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Ian Oberst <[email protected]>
1 parent 2b53073 commit 7b6b199

2 files changed

Lines changed: 102 additions & 4 deletions

File tree

‎crates/buzz-db/src/store/partition/maintenance.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
//!
88
//! 1. holds a schema-scoped advisory lock, so concurrent relays produce one
99
//! winner and `skipped_locked` losers;
10-
//! 2. locks the parent, then foreign-key counterparts, then the catch-all, in
11-
//! the order writers take them, waiting only on the parent and failing fast
12-
//! on any other contention;
10+
//! 2. locks the parent, then foreign-key counterparts read under the parent
11+
//! lock, then the catch-all, in the order writers take them, waiting only on
12+
//! the parent and failing fast on any other contention;
1313
//! 3. re-audits under those locks and refuses if the plan changed; and
1414
//! 4. re-audits the changed catalog and verifies index and constraint
1515
//! inheritance before commit.
@@ -570,6 +570,7 @@ async fn apply_change(
570570
return Ok(LockedAttempt::Refused(collision_message(&name)));
571571
}
572572

573+
// Refuse an unproven lock set before taking any lock.
573574
let parent = maintenance_parent(&mut transaction, table).await?;
574575
if let Some(reason) = parent.refusal {
575576
transaction.rollback().await?;
@@ -587,10 +588,23 @@ async fn apply_change(
587588
),
588589
)
589590
.await?;
591+
// Read the foreign keys again under the parent lock. The read above can
592+
// miss one committed while this transaction waited on the parent, and
593+
// attaching would then wait on its table while holding the parent. Adding
594+
// or dropping a foreign key on or to the parent needs a lock that
595+
// conflicts with this one, so the set cannot change from here on.
596+
let parent = maintenance_parent(&mut transaction, table).await?;
597+
if let Some(reason) = parent.refusal {
598+
transaction.rollback().await?;
599+
return Ok(LockedAttempt::Refused(reason));
600+
}
590601
// Attaching a partition adds its foreign key under SHARE ROW EXCLUSIVE on
591602
// each referenced table. Taking that mode up front still admits readers and
592603
// the `FOR KEY SHARE` checks of other writers, and fails at once rather
593604
// than queue the counterpart's own writers behind this transaction.
605+
// Dropping the catch-all removes only its inherited foreign key and check
606+
// triggers, which lock the catch-all itself, never the referenced table
607+
// (lock probes on PostgreSQL 16 and 17), so no stronger mode is needed.
594608
for counterpart in &parent.counterparts {
595609
lock(
596610
&mut transaction,

‎crates/buzz-db/src/store/partition/maintenance_tests.rs‎

Lines changed: 85 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,8 @@ mod postgres_tests {
328328

329329
use super::*;
330330
use crate::store::partition::tests::postgres_tests::{
331-
catalog_snapshot, create_child, drop_schema, fixed_now, scratch_pool, seed_parents,
331+
catalog_snapshot, create_child, drop_schema, fixed_now, scratch_pool,
332+
scratch_pool_with_max_connections, seed_parents,
332333
};
333334

334335
/// The repaired incident layout for both managed parents, optionally with
@@ -647,6 +648,89 @@ mod postgres_tests {
647648
drop_schema(&admin, &schema).await;
648649
}
649650

651+
async fn wait_until_blocked_by(admin: &PgPool, pid: i32, waiters: i64) {
652+
for _ in 0..100 {
653+
let blocked: i64 = sqlx::query_scalar(
654+
"SELECT count(*) FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid))",
655+
)
656+
.bind(pid)
657+
.fetch_one(admin)
658+
.await
659+
.expect("count blocked backends");
660+
if blocked >= waiters {
661+
return;
662+
}
663+
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
664+
}
665+
panic!("{waiters} backends never queued behind {pid}");
666+
}
667+
668+
#[tokio::test]
669+
#[ignore = "requires Postgres"]
670+
async fn foreign_key_committed_while_waiting_on_the_parent_is_locked_nowait() {
671+
let (pool, admin, schema) = scratch_pool_with_max_connections(6).await;
672+
seed_repaired_layout(&pool, true).await;
673+
sqlx::query("CREATE TABLE reviewers (id UUID PRIMARY KEY)")
674+
.execute(&pool)
675+
.await
676+
.expect("create new counterpart");
677+
// An in-flight migration adds a foreign key that maintenance's
678+
// pre-lock read cannot see yet.
679+
let mut migration = hold_lock(
680+
&pool,
681+
"ALTER TABLE events ADD FOREIGN KEY (community_id) REFERENCES reviewers(id)",
682+
)
683+
.await;
684+
let migration_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
685+
.fetch_one(&mut *migration)
686+
.await
687+
.expect("migration backend pid");
688+
// A writer on the new counterpart queues behind the migration and is
689+
// granted its lock when the migration commits.
690+
let (release_writer, writer_released) = tokio::sync::oneshot::channel::<()>();
691+
let writer = tokio::spawn({
692+
let pool = pool.clone();
693+
async move {
694+
let mut writer = pool.begin().await.expect("begin counterpart writer");
695+
sqlx::query("INSERT INTO reviewers(id) VALUES (gen_random_uuid())")
696+
.execute(&mut *writer)
697+
.await
698+
.expect("counterpart writer insert");
699+
let _ = writer_released.await;
700+
writer
701+
.rollback()
702+
.await
703+
.expect("roll back counterpart writer");
704+
}
705+
});
706+
wait_until_blocked_by(&admin, migration_pid, 1).await;
707+
let maintenance = tokio::spawn({
708+
let pool = pool.clone();
709+
async move { maintain(&pool).await }
710+
});
711+
wait_until_blocked_by(&admin, migration_pid, 2).await;
712+
713+
let started = Instant::now();
714+
migration.commit().await.expect("commit migration");
715+
let result = maintenance.await.expect("maintenance task");
716+
let elapsed = started.elapsed();
717+
718+
// The parent-locked read finds the new counterpart and fails on it at
719+
// once, instead of attaching against it under the parent lock until
720+
// the lock timeout.
721+
assert!(
722+
matches!(result, Err(DbError::InvalidData(ref message))
723+
if message.contains("events lock_timeout") && !message.contains("delivery_log")),
724+
"{result:?}"
725+
);
726+
assert!(elapsed < std::time::Duration::from_secs(1), "{elapsed:?}");
727+
728+
release_writer.send(()).expect("release counterpart writer");
729+
writer.await.expect("counterpart writer task");
730+
assert_advanced(&maintain(&pool).await.expect("advance after release"));
731+
drop_schema(&admin, &schema).await;
732+
}
733+
650734
#[tokio::test]
651735
#[ignore = "requires Postgres"]
652736
async fn deadlock_victim_is_classified_as_a_lock_timeout() {

0 commit comments

Comments
 (0)