@@ -316,24 +316,21 @@ async fn readiness_check_cancellation_balances_waiter_and_inflight_connection()
316316 . connect ( & database_url)
317317 . await
318318 . expect ( "connect cancellation readiness test pool" ) ;
319- let held = pool
319+ let mut held = pool
320320 . acquire ( )
321321 . await
322322 . expect ( "hold sole connection before waiter cancellation" ) ;
323+ let backend_pid: i32 = sqlx:: query_scalar ( "SELECT pg_backend_pid()" )
324+ . fetch_one ( & mut * held)
325+ . await
326+ . expect ( "identify readiness connection" ) ;
323327 let db = Db :: from_pool ( pool) ;
324328
325- let waiting_db = db. clone ( ) ;
326- let waiting = tokio:: spawn ( async move {
327- waiting_db
328- . readiness_check ( tokio:: time:: Instant :: now ( ) + std:: time:: Duration :: from_secs ( 5 ) )
329- . await
330- } ) ;
331- tokio:: time:: sleep ( std:: time:: Duration :: from_millis ( 25 ) ) . await ;
332- waiting. abort ( ) ;
333- assert ! ( waiting
334- . await
335- . expect_err( "waiting check must be cancelled" )
336- . is_cancelled( ) ) ;
329+ let mut waiting = Box :: pin (
330+ db. readiness_check ( tokio:: time:: Instant :: now ( ) + std:: time:: Duration :: from_secs ( 5 ) ) ,
331+ ) ;
332+ assert ! ( futures_util:: poll!( waiting. as_mut( ) ) . is_pending( ) ) ;
333+ drop ( waiting) ;
337334 drop ( held) ;
338335
339336 assert_eq ! (
@@ -343,21 +340,52 @@ async fn readiness_check_cancellation_balances_waiter_and_inflight_connection()
343340 "cancelled pool waiter must not consume the released connection"
344341 ) ;
345342
343+ // Hold the server query until cancellation is observed. A sleep with the
344+ // same duration as the recovery deadline races SQLx's asynchronous drain
345+ // of the cancelled query before it returns the connection to the pool.
346+ let mut blocker = sqlx:: PgConnection :: connect ( & database_url)
347+ . await
348+ . expect ( "connect readiness query blocker" ) ;
349+ sqlx:: query ( "SELECT pg_advisory_lock(86299171)" )
350+ . execute ( & mut blocker)
351+ . await
352+ . expect ( "block readiness query" ) ;
346353 let querying_db = db. clone ( ) ;
347354 let querying = tokio:: spawn ( async move {
348355 querying_db
349356 . readiness_check_sql (
350- tokio:: time:: Instant :: now ( ) + std:: time:: Duration :: from_secs ( 5 ) ,
351- "SELECT pg_sleep(5 )" ,
357+ tokio:: time:: Instant :: now ( ) + std:: time:: Duration :: from_secs ( 30 ) ,
358+ "SELECT pg_advisory_xact_lock(86299171 )" ,
352359 )
353360 . await
354361 } ) ;
355- tokio:: time:: sleep ( std:: time:: Duration :: from_millis ( 25 ) ) . await ;
362+ tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , async {
363+ loop {
364+ let blocked: bool = sqlx:: query_scalar (
365+ "SELECT EXISTS (SELECT 1 FROM pg_stat_activity \
366+ WHERE pid = $1 AND wait_event_type = 'Lock' AND wait_event = 'advisory')",
367+ )
368+ . bind ( backend_pid)
369+ . fetch_one ( & mut blocker)
370+ . await
371+ . expect ( "observe blocked readiness query" ) ;
372+ if blocked {
373+ break ;
374+ }
375+ tokio:: time:: sleep ( std:: time:: Duration :: from_millis ( 10 ) ) . await ;
376+ }
377+ } )
378+ . await
379+ . expect ( "readiness query must reach the server before cancellation" ) ;
356380 querying. abort ( ) ;
357381 assert ! ( querying
358382 . await
359383 . expect_err( "querying check must be cancelled" )
360384 . is_cancelled( ) ) ;
385+ sqlx:: query ( "SELECT pg_advisory_unlock(86299171)" )
386+ . execute ( & mut blocker)
387+ . await
388+ . expect ( "release cancelled readiness query" ) ;
361389
362390 let recovered = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , async {
363391 loop {
@@ -1906,11 +1934,10 @@ async fn is_relay_member_is_bounded_routed_and_fails_closed() {
19061934/// directly, because `buzz_db_route_decision` is emitted by `route_read`
19071935/// — a direct call would prove the timing but never emit the label.
19081936///
1909- /// Timing uses an upper bound of 2x the budget minus a margin: it must
1910- /// fail for two stacked budgets (~300ms) while tolerating scheduler
1911- /// jitter on one (~150ms). Asserting a lower bound too would pin the
1912- /// budget's own value, which `reader_acquire_timeout_is_the_documented_budget`
1913- /// already covers.
1937+ /// Pause Tokio time only while the reader is saturated. After advancing one
1938+ /// acquire budget, the production route must already have selected the writer.
1939+ /// Resume time before awaiting writer I/O so PostgreSQL scheduling cannot
1940+ /// affect the budget assertion.
19141941#[ tokio:: test( flavor = "current_thread" ) ]
19151942#[ ignore = "requires Postgres" ]
19161943async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold ( ) {
@@ -1975,24 +2002,17 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
19752002 // a multi-thread runtime the emit could land on a worker where no
19762003 // local recorder is installed and the label assertions would vacuously
19772004 // see an empty snapshot.
1978- let start = std:: time:: Instant :: now ( ) ;
1979- let count = {
1980- let _guard = metrics:: set_default_local_recorder ( & recorder) ;
1981- db. count_events_routed ( "one_budget_probe" , & query) . await
1982- }
1983- . expect ( "writer fallback still answers the read" ) ;
1984- let elapsed = start. elapsed ( ) ;
1985-
1986- assert_eq ! ( count, 0 , "writer answered on an empty scratch database" ) ;
1987- assert ! (
1988- elapsed < Duration :: from_millis( 250 ) ,
1989- "routed fallback must spend ONE {}ms acquire budget, not two; took {}ms" ,
1990- Db :: READER_ACQUIRE_TIMEOUT . as_millis( ) ,
1991- elapsed. as_millis( )
1992- ) ;
1993-
1994- let reasons: std:: collections:: HashMap < ( String , String ) , u64 > = snapshotter
1995- . snapshot ( )
2005+ let _guard = metrics:: set_default_local_recorder ( & recorder) ;
2006+ tokio:: time:: pause ( ) ;
2007+ let mut read = Box :: pin ( db. count_events_routed ( "one_budget_probe" , & query) ) ;
2008+ assert ! ( futures_util:: poll!( read. as_mut( ) ) . is_pending( ) ) ;
2009+ // Tokio timers have millisecond granularity; cross the deadline by one tick.
2010+ tokio:: time:: advance ( Db :: READER_ACQUIRE_TIMEOUT + Duration :: from_millis ( 1 ) ) . await ;
2011+ let completion = futures_util:: poll!( read. as_mut( ) ) ;
2012+ let after_one_budget = snapshotter. snapshot ( ) ;
2013+ tokio:: time:: resume ( ) ;
2014+
2015+ let reasons: std:: collections:: HashMap < ( String , String ) , u64 > = after_one_budget
19962016 . into_vec ( )
19972017 . into_iter ( )
19982018 . filter ( |( key, ..) | key. key ( ) . name ( ) == "buzz_db_route_decision" )
@@ -2015,7 +2035,7 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
20152035 assert_eq ! (
20162036 reasons. get( & ( "writer" . to_owned( ) , "reader_acquire_timeout" . to_owned( ) ) ) ,
20172037 Some ( & 1 ) ,
2018- "saturated reader must fall back as writer/reader_acquire_timeout; got {reasons:?}"
2038+ "saturated reader must choose writer/reader_acquire_timeout after one budget ; got {reasons:?}"
20192039 ) ;
20202040 // `reader_validation_error` would mean we misclassified a timeout as a
20212041 // broken reader, and `pool_busy` is the retired name — neither may
@@ -2027,6 +2047,13 @@ async fn routed_fallback_spends_one_acquire_budget_when_aurora_cache_is_cold() {
20272047 "no reader_validation_error or retired pool_busy label may be emitted; got {reasons:?}"
20282048 ) ;
20292049
2050+ let count = match completion {
2051+ std:: task:: Poll :: Ready ( result) => result,
2052+ std:: task:: Poll :: Pending => read. await ,
2053+ }
2054+ . expect ( "writer fallback still answers the read" ) ;
2055+ assert_eq ! ( count, 0 , "writer answered on an empty scratch database" ) ;
2056+
20302057 drop ( held) ;
20312058 drop_scratch_db ( & admin, db. pool . clone ( ) , & wname) . await ;
20322059}
@@ -3256,7 +3283,7 @@ async fn replica_floor_writer_transaction_holds_shared_lock() {
32563283
32573284#[ tokio:: test]
32583285#[ ignore = "requires Postgres" ]
3259- async fn replica_floor_probe_waits_for_shared_writer_and_records_after_release ( ) {
3286+ async fn cluster_global_replica_floor_probe_waits_for_shared_writer_and_records_after_release ( ) {
32603287 let admin = PgPool :: connect ( & admin_url ( ) . await )
32613288 . await
32623289 . expect ( "connect admin" ) ;
0 commit comments