Skip to content

Commit 0534277

Browse files
Haytwpfleger96
andcommitted
test(buzz-db): replace barrier race with controlled lock-wait-observation schedule
The previous concurrency regression used a tokio::sync::Barrier to synchronize two connections before racing to INSERT the same revision. That construction synchronizes client-side INSERT dispatch, not trigger execution; a lock-free schedule where one INSERT completes and commits before the other reads MAX still satisfies the XOR + 23514 assertions, so the test offered no deterministic proof that pg_advisory_xact_lock is required. Replace with a controlled two-connection schedule: 1. tx1 opens a transaction and inserts revision 102. The BEFORE INSERT trigger acquires pg_advisory_xact_lock and completes; tx1 holds the advisory lock until commit. 2. tx2 opens a transaction on a second backend, reports its pg_backend_pid over a oneshot channel, then issues INSERT for revision 103. The trigger fires and blocks on the advisory lock held by tx1. 3. The main task polls pg_stat_activity WHERE pid = tx2_pid AND wait_event_type = 'Lock' AND wait_event = 'advisory' with a 10 s bounded timeout. Without pg_advisory_xact_lock in the guard the trigger returns immediately, tx2 never enters the advisory wait, and the poll times out — making the regression deterministically red. 4. tx1 commits, releasing the lock. tx2 unblocks, its trigger reads the fresh MAX=102, and INSERT 103 succeeds. tx2 commits. 5. Final count asserts six revisions (1, 2, 100, 101, 102, 103). Zero production changes: migrations/0041, migrations/0042, and schema/schema.sql are byte-untouched (single-file diff confirmed). Co-authored-by: Will Pfleger <[email protected]> Signed-off-by: Will Pfleger <[email protected]> Signed-off-by: Hayt <9e1c23a3fd83f61da34420e4e88ff1b16e45cafcc0cd9019eb07d4ecfa8ca9b0@buzz.block.builderlab.xyz>
1 parent c52444b commit 0534277

1 file changed

Lines changed: 110 additions & 93 deletions

File tree

‎crates/buzz-db/src/runtime/migration.rs‎

Lines changed: 110 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -3345,113 +3345,130 @@ mod tests {
33453345
got: {replay_err}"
33463346
);
33473347

3348-
// Concurrency regression: prove the advisory lock actually serializes
3349-
// concurrent writers. Two connections race to insert the SAME next revision
3350-
// (102) for the same community. The advisory lock must cause one writer to
3351-
// block, see the other's committed MAX, and then be rejected with 23514 from
3352-
// the named guard. Without the lock, both could pass the MAX check before
3353-
// either commits; only a PK collision (23505) would catch the duplicate —
3354-
// not the guard. Removing pg_advisory_xact_lock from the guard function and
3355-
// re-running must produce 23505 (PK) rather than 23514 (guard), proving the
3356-
// test is mutation-sensitive to the lock.
3348+
// Concurrency regression: prove the advisory lock is load-bearing. The
3349+
// test uses a controlled two-connection schedule:
33573350
//
3358-
// A tokio::sync::Barrier synchronizes both connections so they both have a
3359-
// live transaction and are ready to INSERT before either proceeds. After the
3360-
// barrier both race to acquire the advisory lock; one wins, commits, and
3361-
// releases the lock; the other then sees the committed MAX and is rejected
3362-
// by the guard with 23514.
3351+
// 1. tx1 opens a transaction and inserts revision 102. The BEFORE INSERT
3352+
// trigger acquires `pg_advisory_xact_lock(lock_key)` and completes the
3353+
// INSERT — tx1 now holds the advisory lock until it commits.
3354+
// 2. tx2 opens a transaction on a second backend and issues INSERT for
3355+
// revision 103. The trigger fires and blocks inside
3356+
// `pg_advisory_xact_lock(lock_key)` waiting for tx1 to release.
3357+
// 3. We observe tx2's backend entering a Lock-wait state via
3358+
// pg_stat_activity (wait_event_type='Lock', wait_event='advisory'),
3359+
// with a bounded timeout — not a sleep. If the advisory-lock call is
3360+
// removed from the guard, the trigger returns immediately; tx2 never
3361+
// enters the advisory wait, and the poll times out, failing the test.
3362+
// This is the mutation-sensitivity guarantee.
3363+
// 4. tx1 commits, releasing the advisory lock. tx2 unblocks, its trigger
3364+
// reads the fresh MAX=102, and the INSERT succeeds (103 > 102).
3365+
// 5. tx2 commits. Both revisions 102 and 103 are present.
3366+
use std::time::Instant;
3367+
3368+
// tx1: open a transaction and insert revision 102. The INSERT returns after
3369+
// the trigger acquires the lock and succeeds; the advisory lock stays held
3370+
// until the transaction commits.
3371+
let mut conn1 = pool.acquire().await.expect("acquire conn1");
3372+
sqlx::query("BEGIN")
3373+
.execute(&mut *conn1)
3374+
.await
3375+
.expect("begin tx1");
3376+
sqlx::query(
3377+
"INSERT INTO identity_enrollment_policies \
3378+
(community_id, policy_revision, enrollment_mode, policy_digest, effective_at) \
3379+
VALUES ($1, 102, 1, $2, '2029-01-01T00:00:00Z')",
3380+
)
3381+
.bind(community_id)
3382+
.bind(vec![0xB1_u8; 32])
3383+
.execute(&mut *conn1)
3384+
.await
3385+
.expect("tx1 INSERT revision 102 must succeed");
3386+
// tx1 holds the advisory lock. Do NOT commit yet.
3387+
3388+
// tx2: acquire a separate backend, record its PID, then issue the INSERT.
3389+
// The trigger will block on the advisory lock held by tx1.
33633390
let pool2 = pool.clone();
33643391
let pool3 = pool.clone();
3365-
let community_id2 = community_id;
3366-
let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(2));
3367-
let barrier2 = barrier.clone();
3368-
3369-
let (tx1_result, tx2_result) = tokio::join!(
3370-
tokio::spawn(async move {
3371-
let mut conn = pool.acquire().await.expect("acquire conn1");
3372-
sqlx::query("BEGIN")
3373-
.execute(&mut *conn)
3374-
.await
3375-
.expect("begin1");
3376-
// Wait until both connections have open transactions before
3377-
// either races to INSERT — eliminates ordering accidents.
3378-
barrier.wait().await;
3379-
let insert_r = sqlx::query(
3380-
"INSERT INTO identity_enrollment_policies \
3381-
(community_id, policy_revision, enrollment_mode, policy_digest, effective_at) \
3382-
VALUES ($1, 102, 1, $2, '2029-01-01T00:00:00Z')",
3383-
)
3384-
.bind(community_id)
3385-
.bind(vec![0xB1_u8; 32])
3386-
.execute(&mut *conn)
3387-
.await;
3388-
let commit_r = sqlx::query("COMMIT").execute(&mut *conn).await;
3389-
(insert_r, commit_r)
3390-
}),
3391-
tokio::spawn(async move {
3392-
let mut conn = pool2.acquire().await.expect("acquire conn2");
3393-
sqlx::query("BEGIN")
3394-
.execute(&mut *conn)
3395-
.await
3396-
.expect("begin2");
3397-
// Mirror barrier wait so both are in-flight simultaneously.
3398-
barrier2.wait().await;
3399-
let insert_r = sqlx::query(
3400-
"INSERT INTO identity_enrollment_policies \
3401-
(community_id, policy_revision, enrollment_mode, policy_digest, effective_at) \
3402-
VALUES ($1, 102, 1, $2, '2029-01-01T00:00:00Z')",
3403-
)
3404-
.bind(community_id2)
3405-
.bind(vec![0xB2_u8; 32])
3406-
.execute(&mut *conn)
3407-
.await;
3408-
let commit_r = sqlx::query("COMMIT").execute(&mut *conn).await;
3409-
(insert_r, commit_r)
3410-
}),
3411-
);
3392+
let (pid_tx, pid_rx) = tokio::sync::oneshot::channel::<i32>();
3393+
let tx2_task = tokio::spawn(async move {
3394+
let mut conn2 = pool2.acquire().await.expect("acquire conn2");
3395+
// Report this backend's PID so the observer can poll pg_stat_activity.
3396+
let backend_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
3397+
.fetch_one(&mut *conn2)
3398+
.await
3399+
.expect("get conn2 backend pid");
3400+
let _ = pid_tx.send(backend_pid);
3401+
sqlx::query("BEGIN")
3402+
.execute(&mut *conn2)
3403+
.await
3404+
.expect("begin tx2");
3405+
// This INSERT will block inside the trigger waiting for tx1's advisory lock.
3406+
let insert_r = sqlx::query(
3407+
"INSERT INTO identity_enrollment_policies \
3408+
(community_id, policy_revision, enrollment_mode, policy_digest, effective_at) \
3409+
VALUES ($1, 103, 1, $2, '2029-06-01T00:00:00Z')",
3410+
)
3411+
.bind(community_id)
3412+
.bind(vec![0xB2_u8; 32])
3413+
.execute(&mut *conn2)
3414+
.await;
3415+
let commit_r = sqlx::query("COMMIT").execute(&mut *conn2).await;
3416+
(insert_r, commit_r)
3417+
});
34123418

3413-
let (insert1, commit1) = tx1_result.expect("tx1 task completed");
3414-
let (insert2, commit2) = tx2_result.expect("tx2 task completed");
3419+
// Receive tx2's backend PID and wait until it enters an advisory-lock wait.
3420+
// Mutation proof: without pg_advisory_xact_lock in the guard, the trigger
3421+
// returns immediately; tx2 never parks on an advisory lock; the poll below
3422+
// times out and panics, making this test deterministically red.
3423+
let tx2_pid = pid_rx.await.expect("tx2 reports its backend pid");
3424+
let deadline = Instant::now() + std::time::Duration::from_secs(10);
3425+
loop {
3426+
let waiting: bool = sqlx::query_scalar(
3427+
"SELECT EXISTS (\
3428+
SELECT 1 FROM pg_stat_activity \
3429+
WHERE pid = $1 \
3430+
AND wait_event_type = 'Lock' \
3431+
AND wait_event = 'advisory'\
3432+
)",
3433+
)
3434+
.bind(tx2_pid)
3435+
.fetch_one(&pool3)
3436+
.await
3437+
.expect("poll tx2 advisory-lock wait");
3438+
if waiting {
3439+
break;
3440+
}
3441+
assert!(
3442+
Instant::now() < deadline,
3443+
"tx2 never entered advisory-lock wait — pg_advisory_xact_lock \
3444+
must be present in the guard for the lock to serialize writers"
3445+
);
3446+
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
3447+
}
34153448

3416-
// The BEFORE INSERT trigger fires at statement time: the winner's INSERT
3417-
// succeeds (lock acquired, MAX check passes, INSERT completes), the loser's
3418-
// INSERT blocks waiting for the lock and then fails with 23514 when it sees
3419-
// the winner's committed MAX. Exactly one INSERT must succeed; exactly one
3420-
// must fail with 23514 from the named guard.
3421-
let (i1_ok, i2_ok) = (insert1.is_ok(), insert2.is_ok());
3422-
assert!(
3423-
i1_ok ^ i2_ok,
3424-
"exactly one of the two concurrent revision-102 inserts must succeed at INSERT; \
3425-
got insert1={i1_ok} insert2={i2_ok}"
3426-
);
3427-
let loser_insert = if i1_ok { insert2 } else { insert1 };
3428-
let loser_err = loser_insert.expect_err("loser INSERT must have failed");
3429-
assert!(
3430-
loser_err
3431-
.as_database_error()
3432-
.map(|e| e.code().as_deref() == Some("23514"))
3433-
.unwrap_or(false),
3434-
"loser must fail with check_violation (23514) from \
3435-
identity_enrollment_policy_revision_monotonic guard, not a PK \
3436-
collision — this proves the advisory lock serialized the writers; \
3437-
got: {loser_err}"
3438-
);
3449+
// tx2 is observably blocked. Commit tx1, releasing the advisory lock.
3450+
sqlx::query("COMMIT")
3451+
.execute(&mut *conn1)
3452+
.await
3453+
.expect("tx1 COMMIT must succeed");
34393454

3440-
// The winner's commit must succeed.
3441-
let winner_commit = if i1_ok { commit1 } else { commit2 };
3442-
winner_commit.expect("winner COMMIT must succeed");
3455+
// tx2 unblocks: the trigger re-runs its SELECT MAX, sees committed 102,
3456+
// and INSERT 103 succeeds. Both the INSERT and COMMIT must complete.
3457+
let (insert2, commit2) = tx2_task.await.expect("tx2 task completed");
3458+
insert2.expect("tx2 INSERT revision 103 must succeed after tx1 commits");
3459+
commit2.expect("tx2 COMMIT must succeed");
34433460

3444-
// Confirm exactly 5 policy revisions are now present (1, 2, 100, 101, 102).
3461+
// Both revisions 102 and 103 must be present (total: 1, 2, 100, 101, 102, 103).
34453462
let count: i64 = sqlx::query_scalar(
34463463
"SELECT count(*) FROM identity_enrollment_policies WHERE community_id = $1",
34473464
)
34483465
.bind(community_id)
3449-
.fetch_one(&pool3)
3466+
.fetch_one(&pool)
34503467
.await
34513468
.expect("count persisted policy revisions");
34523469
assert_eq!(
3453-
count, 5,
3454-
"exactly five revisions must persist after concurrency race"
3470+
count, 6,
3471+
"exactly six revisions must persist after the controlled concurrency sequence"
34553472
);
34563473
}
34573474

0 commit comments

Comments
 (0)