Skip to content

Commit 2b99682

Browse files
committed
fix(ci): stabilize PostgreSQL concurrency tests
Reserve all nextest workers for cluster-wide activity scans, including the floor-lock probes. Wait for server-side cleanup when the usage-metrics leader disconnects. Synchronize readiness cancellation with a blocked server query and await rollback after rejected deletion updates before claiming rows. Drive the reader acquisition budget with Tokio's controlled clock instead of a wall-clock upper bound. Preserve exclusive ownership, cancellation recovery, exact inventory approval, and single-budget fallback assertions. Signed-off-by: Tee8z <[email protected]>
1 parent 90d9be8 commit 2b99682

6 files changed

Lines changed: 124 additions & 61 deletions

File tree

‎.config/nextest.toml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,11 @@ fail-fast = false
2424
# wrapper retained one database per concurrently running test process.
2525
test-threads = 8
2626

27-
[test-groups.postgres-cluster-global]
28-
# These tests inspect cluster-wide activity or create least-privilege sessions,
29-
# so database-per-test isolation alone cannot make them independent.
30-
max-threads = 1
31-
3227
[[profile.postgres-ci.overrides]]
3328
filter = "test(/cluster_global_/)"
34-
test-group = "postgres-cluster-global"
29+
# Activity scans also observe ordinary tests connecting to the same cluster.
30+
# Reserve every worker so those tests and their wrappers cannot overlap a scan.
31+
threads-required = "num-test-threads"
3532

3633
[[profile.postgres-ci.scripts]]
3734
# Script filters are separate from default-filter: they attach the setup and

‎crates/buzz-db/TESTING.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ migration-applied coverage rather than making the bootstrap implicit.
4141

4242
Tests that inspect cluster-wide PostgreSQL state or open least-privilege
4343
sessions include `cluster_global_` in the function name. Migration-backed cases
44-
use `migration_schema_cluster_global_`. Nextest serializes this small group
45-
because separate databases still share `pg_stat_activity` and roles.
44+
use `migration_schema_cluster_global_`. Nextest reserves all test workers for
45+
each of these tests, preventing overlap with ordinary database tests and their
46+
wrappers. Separate databases still share `pg_stat_activity` and roles.
4647

4748
## Run the lane locally
4849

‎crates/buzz-db/src/runtime/replica_fence.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1050,7 +1050,7 @@ mod postgres_tests {
10501050
/// still held. It must block until release, then sample.
10511051
#[tokio::test]
10521052
#[ignore = "requires Postgres"]
1053-
async fn sample_writer_waits_for_shared_floor_lock_before_sampling_time() {
1053+
async fn cluster_global_sample_writer_waits_for_shared_floor_lock_before_sampling_time() {
10541054
let (admin, pool, name) = scratch_db().await;
10551055

10561056
let mut blocker = pool.begin().await.expect("begin shared-lock blocker");

‎crates/buzz-db/src/runtime/tests.rs‎

Lines changed: 68 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -316,24 +316,21 @@ async fn readiness_check_cancellation_balances_waiter_and_inflight_connection()
316316
.connect(&database_url)
317317
.await
318318
.expect("connect cancellation readiness test pool");
319-
let held = pool
319+
let mut held = pool
320320
.acquire()
321321
.await
322322
.expect("hold sole connection before waiter cancellation");
323+
let backend_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()")
324+
.fetch_one(&mut *held)
325+
.await
326+
.expect("identify readiness connection");
323327
let db = Db::from_pool(pool);
324328

325-
let waiting_db = db.clone();
326-
let waiting = tokio::spawn(async move {
327-
waiting_db
328-
.readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(5))
329-
.await
330-
});
331-
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
332-
waiting.abort();
333-
assert!(waiting
334-
.await
335-
.expect_err("waiting check must be cancelled")
336-
.is_cancelled());
329+
let mut waiting = Box::pin(
330+
db.readiness_check(tokio::time::Instant::now() + std::time::Duration::from_secs(5)),
331+
);
332+
assert!(futures_util::poll!(waiting.as_mut()).is_pending());
333+
drop(waiting);
337334
drop(held);
338335

339336
assert_eq!(
@@ -343,21 +340,52 @@ async fn readiness_check_cancellation_balances_waiter_and_inflight_connection()
343340
"cancelled pool waiter must not consume the released connection"
344341
);
345342

343+
// Hold the server query until cancellation is observed. A sleep with the
344+
// same duration as the recovery deadline races SQLx's asynchronous drain
345+
// of the cancelled query before it returns the connection to the pool.
346+
let mut blocker = sqlx::PgConnection::connect(&database_url)
347+
.await
348+
.expect("connect readiness query blocker");
349+
sqlx::query("SELECT pg_advisory_lock(86299171)")
350+
.execute(&mut blocker)
351+
.await
352+
.expect("block readiness query");
346353
let querying_db = db.clone();
347354
let querying = tokio::spawn(async move {
348355
querying_db
349356
.readiness_check_sql(
350-
tokio::time::Instant::now() + std::time::Duration::from_secs(5),
351-
"SELECT pg_sleep(5)",
357+
tokio::time::Instant::now() + std::time::Duration::from_secs(30),
358+
"SELECT pg_advisory_xact_lock(86299171)",
352359
)
353360
.await
354361
});
355-
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
362+
tokio::time::timeout(std::time::Duration::from_secs(5), async {
363+
loop {
364+
let blocked: bool = sqlx::query_scalar(
365+
"SELECT EXISTS (SELECT 1 FROM pg_stat_activity \
366+
WHERE pid = $1 AND wait_event_type = 'Lock' AND wait_event = 'advisory')",
367+
)
368+
.bind(backend_pid)
369+
.fetch_one(&mut blocker)
370+
.await
371+
.expect("observe blocked readiness query");
372+
if blocked {
373+
break;
374+
}
375+
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
376+
}
377+
})
378+
.await
379+
.expect("readiness query must reach the server before cancellation");
356380
querying.abort();
357381
assert!(querying
358382
.await
359383
.expect_err("querying check must be cancelled")
360384
.is_cancelled());
385+
sqlx::query("SELECT pg_advisory_unlock(86299171)")
386+
.execute(&mut blocker)
387+
.await
388+
.expect("release cancelled readiness query");
361389

362390
let recovered = tokio::time::timeout(std::time::Duration::from_secs(5), async {
363391
loop {
@@ -1906,11 +1934,10 @@ async fn is_relay_member_is_bounded_routed_and_fails_closed() {
19061934
/// directly, because `buzz_db_route_decision` is emitted by `route_read`
19071935
/// — a direct call would prove the timing but never emit the label.
19081936
///
1909-
/// Timing uses an upper bound of 2x the budget minus a margin: it must
1910-
/// fail for two stacked budgets (~300ms) while tolerating scheduler
1911-
/// jitter on one (~150ms). Asserting a lower bound too would pin the
1912-
/// budget's own value, which `reader_acquire_timeout_is_the_documented_budget`
1913-
/// already covers.
1937+
/// Pause Tokio time only while the reader is saturated. After advancing one
1938+
/// acquire budget, the production route must already have selected the writer.
1939+
/// Resume time before awaiting writer I/O so PostgreSQL scheduling cannot
1940+
/// affect the budget assertion.
19141941
#[tokio::test(flavor = "current_thread")]
19151942
#[ignore = "requires Postgres"]
19161943
async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
@@ -1975,24 +2002,17 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
19752002
// a multi-thread runtime the emit could land on a worker where no
19762003
// local recorder is installed and the label assertions would vacuously
19772004
// see an empty snapshot.
1978-
let start = std::time::Instant::now();
1979-
let count = {
1980-
let _guard = metrics::set_default_local_recorder(&recorder);
1981-
db.count_events_routed("one_budget_probe", &query).await
1982-
}
1983-
.expect("writer fallback still answers the read");
1984-
let elapsed = start.elapsed();
1985-
1986-
assert_eq!(count, 0, "writer answered on an empty scratch database");
1987-
assert!(
1988-
elapsed < Duration::from_millis(250),
1989-
"routed fallback must spend ONE {}ms acquire budget, not two; took {}ms",
1990-
Db::READER_ACQUIRE_TIMEOUT.as_millis(),
1991-
elapsed.as_millis()
1992-
);
1993-
1994-
let reasons: std::collections::HashMap<(String, String), u64> = snapshotter
1995-
.snapshot()
2005+
let _guard = metrics::set_default_local_recorder(&recorder);
2006+
tokio::time::pause();
2007+
let mut read = Box::pin(db.count_events_routed("one_budget_probe", &query));
2008+
assert!(futures_util::poll!(read.as_mut()).is_pending());
2009+
// Tokio timers have millisecond granularity; cross the deadline by one tick.
2010+
tokio::time::advance(Db::READER_ACQUIRE_TIMEOUT + Duration::from_millis(1)).await;
2011+
let completion = futures_util::poll!(read.as_mut());
2012+
let after_one_budget = snapshotter.snapshot();
2013+
tokio::time::resume();
2014+
2015+
let reasons: std::collections::HashMap<(String, String), u64> = after_one_budget
19962016
.into_vec()
19972017
.into_iter()
19982018
.filter(|(key, ..)| key.key().name() == "buzz_db_route_decision")
@@ -2015,7 +2035,7 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
20152035
assert_eq!(
20162036
reasons.get(&("writer".to_owned(), "reader_acquire_timeout".to_owned())),
20172037
Some(&1),
2018-
"saturated reader must fall back as writer/reader_acquire_timeout; got {reasons:?}"
2038+
"saturated reader must choose writer/reader_acquire_timeout after one budget; got {reasons:?}"
20192039
);
20202040
// `reader_validation_error` would mean we misclassified a timeout as a
20212041
// broken reader, and `pool_busy` is the retired name — neither may
@@ -2027,6 +2047,13 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
20272047
"no reader_validation_error or retired pool_busy label may be emitted; got {reasons:?}"
20282048
);
20292049

2050+
let count = match completion {
2051+
std::task::Poll::Ready(result) => result,
2052+
std::task::Poll::Pending => read.await,
2053+
}
2054+
.expect("writer fallback still answers the read");
2055+
assert_eq!(count, 0, "writer answered on an empty scratch database");
2056+
20302057
drop(held);
20312058
drop_scratch_db(&admin, db.pool.clone(), &wname).await;
20322059
}
@@ -3256,7 +3283,7 @@ async fn replica_floor_writer_transaction_holds_shared_lock() {
32563283

32573284
#[tokio::test]
32583285
#[ignore = "requires Postgres"]
3259-
async fn replica_floor_probe_waits_for_shared_writer_and_records_after_release() {
3286+
async fn cluster_global_replica_floor_probe_waits_for_shared_writer_and_records_after_release() {
32603287
let admin = PgPool::connect(&admin_url().await)
32613288
.await
32623289
.expect("connect admin");

‎crates/buzz-db/src/store/deletion.rs‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6243,6 +6243,11 @@ mod postgres_tests {
62436243
assert_eq!(current_schema.scoped_tables, inventory.schema.scoped_tables);
62446244
assert_eq!(current_schema.fenced_tables, inventory.schema.fenced_tables);
62456245

6246+
let mut rejected = db
6247+
.pool
6248+
.begin()
6249+
.await
6250+
.expect("begin rejected approval insert");
62466251
let mismatched_insert = sqlx::query(
62476252
"INSERT INTO community_deletion_approvals \
62486253
(request_id, community_id, inventory_digest, approved_by) \
@@ -6251,12 +6256,16 @@ mod postgres_tests {
62516256
.bind(request.id)
62526257
.bind(*request.community_id.as_uuid())
62536258
.bind(vec![0_u8; 32])
6254-
.execute(&db.pool)
6259+
.execute(&mut *rejected)
62556260
.await;
62566261
assert!(
62576262
mismatched_insert.is_err(),
62586263
"a mismatched approval must be unrepresentable"
62596264
);
6265+
rejected
6266+
.rollback()
6267+
.await
6268+
.expect("roll back rejected approval insert");
62606269
let approved = store
62616270
.approve(request.id, "approver-a", Some("reviewed"))
62626271
.await
@@ -6276,28 +6285,48 @@ mod postgres_tests {
62766285
approved.inventory_digest,
62776286
Some(hex::encode(inventory.digest().unwrap()))
62786287
);
6288+
let mut rejected = db
6289+
.pool
6290+
.begin()
6291+
.await
6292+
.expect("begin rejected approval update");
62796293
let mismatched_approval = sqlx::query(
62806294
"UPDATE community_deletion_approvals SET inventory_digest = $2 WHERE request_id = $1",
62816295
)
62826296
.bind(request.id)
62836297
.bind(vec![0_u8; 32])
6284-
.execute(&db.pool)
6298+
.execute(&mut *rejected)
62856299
.await;
62866300
assert!(
62876301
mismatched_approval.is_err(),
62886302
"approval digest must remain database-bound to the frozen request digest"
62896303
);
6304+
rejected
6305+
.rollback()
6306+
.await
6307+
.expect("roll back rejected approval update");
6308+
let mut rejected = db
6309+
.pool
6310+
.begin()
6311+
.await
6312+
.expect("begin rejected request update");
62906313
let mismatched_request = sqlx::query(
62916314
"UPDATE community_deletion_requests SET inventory_digest = $2 WHERE id = $1",
62926315
)
62936316
.bind(request.id)
62946317
.bind(vec![1_u8; 32])
6295-
.execute(&db.pool)
6318+
.execute(&mut *rejected)
62966319
.await;
62976320
assert!(
62986321
mismatched_request.is_err(),
62996322
"the frozen request digest must remain bound to its approval"
63006323
);
6324+
// ErrorResponse can arrive before PostgreSQL releases the row lock.
6325+
// Await rollback before the claimant uses FOR UPDATE SKIP LOCKED.
6326+
rejected
6327+
.rollback()
6328+
.await
6329+
.expect("roll back rejected request update");
63016330
assert!(store
63026331
.claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION)
63036332
.await

‎crates/buzz-db/src/store/usage.rs‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -924,16 +924,25 @@ mod postgres_tests {
924924
);
925925

926926
drop(leader);
927-
assert!(
928-
second
929-
.try_lock_usage_metrics(key)
930-
.await
931-
.expect("lock attempt after leader drop")
932-
.is_some(),
933-
"dropping the detached session releases its advisory lock"
934-
);
927+
// Closing the socket does not wait for PostgreSQL to process the
928+
// disconnect. Observe lock release instead of racing server cleanup.
929+
let successor = tokio::time::timeout(std::time::Duration::from_secs(5), async {
930+
loop {
931+
if let Some(leader) = second
932+
.try_lock_usage_metrics(key)
933+
.await
934+
.expect("lock attempt after leader drop")
935+
{
936+
break leader;
937+
}
938+
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
939+
}
940+
})
941+
.await
942+
.expect("dropping the detached session releases its advisory lock");
935943

936944
// Release any remaining session state before DROP DATABASE.
945+
drop(successor);
937946
drop(first);
938947
drop(second);
939948
drop_scratch_db(&admin, pool, &scratch_name).await;

0 commit comments

Comments
 (0)