@@ -569,6 +569,17 @@ fn read_budget_from_ms(ms: u64) -> Option<Duration> {
569569 }
570570}
571571
572+ /// Outcome of one reader-pool warm pass (see [`Db::warm_read_pool`]).
573+ #[ derive( Debug , Clone , Copy , PartialEq , Eq ) ]
574+ struct ReaderWarmReport {
575+ /// Connections the pass aimed to hold: the pool's ceiling.
576+ target : u32 ,
577+ /// Connections opened (or reused idle) and then released back to idle.
578+ held : u32 ,
579+ /// Acquires that errored or timed out.
580+ failed : u32 ,
581+ }
582+
572583/// Snapshot of Postgres connection pool utilisation.
573584#[ derive( Debug , Clone , Copy ) ]
574585pub struct DbPoolStats {
@@ -1135,6 +1146,9 @@ impl Db {
11351146 /// pool up to [`Db::reader_min_connections`] in the background, so a
11361147 /// freshly started pod does not meet its first traffic with an empty
11371148 /// reader pool and pay every dial inside the 150ms request budget.
1149+ /// It then warms the pool toward its ceiling ([`Db::warm_read_pool`]),
1150+ /// still in the spawned task: a failed or partial warm only logs and
1151+ /// never affects startup or readiness.
11381152 /// Probe acquires that overrun that budget are retried with
11391153 /// [`Db::READER_BOOT_PROBE_BACKOFF`].
11401154 ///
@@ -1150,7 +1164,147 @@ impl Db {
11501164 return ;
11511165 } ;
11521166 let aurora_identity = self . reader_aurora_identity . clone ( ) ;
1153- tokio:: spawn ( Self :: read_pool_boot_ping_once ( read_pool, aurora_identity) ) ;
1167+ tokio:: spawn ( Self :: read_pool_boot_ping_then_warm (
1168+ read_pool,
1169+ aurora_identity,
1170+ ) ) ;
1171+ }
1172+
1173+ /// Connections each warm wave dials at once. Small enough that a pod
1174+ /// starting alongside its peers does not stampede the reader with
1175+ /// simultaneous TCP+TLS+SCRAM handshakes (a stampede is what pushes a
1176+ /// dial past [`Db::READER_ACQUIRE_TIMEOUT`]), large enough that a full
1177+ /// pool is warm within a handful of dial times.
1178+ const READER_WARM_CONCURRENCY : u32 = 5 ;
1179+
1180+ /// Bound on one warm acquire. The pool's own acquire timeout
1181+ /// ([`Db::READER_ACQUIRE_TIMEOUT`]) already caps a dial; this is the
1182+ /// explicit backstop so a warm acquire can never outlive a wave.
1183+ const READER_WARM_ACQUIRE_TIMEOUT : Duration = Duration :: from_secs ( 1 ) ;
1184+
1185+ /// Connections the next warm wave should open: the remaining shortfall,
1186+ /// at most [`Db::READER_WARM_CONCURRENCY`].
1187+ fn reader_warm_wave_size ( target : u32 , held : u32 ) -> u32 {
1188+ target
1189+ . saturating_sub ( held)
1190+ . min ( Self :: READER_WARM_CONCURRENCY )
1191+ }
1192+
1193+ /// The boot ping, then (only if the reader answered) a warm pass.
1194+ ///
1195+ /// Returns `None` when the ping found the reader unreachable: warming a
1196+ /// pool whose reader is down would only add abandoned dials.
1197+ async fn read_pool_boot_ping_then_warm (
1198+ read_pool : PgPool ,
1199+ aurora_identity : std:: sync:: Arc < std:: sync:: OnceLock < bool > > ,
1200+ ) -> Option < ReaderWarmReport > {
1201+ if !Self :: read_pool_boot_ping_once ( read_pool. clone ( ) , aurora_identity) . await {
1202+ return None ;
1203+ }
1204+ let report = Self :: warm_read_pool ( & read_pool) . await ;
1205+ if report. held < report. target {
1206+ tracing:: warn!(
1207+ target = report. target,
1208+ opened = report. held,
1209+ failed = report. failed,
1210+ "read pool warm-up opened fewer connections than its ceiling; \
1211+ the rest dial on demand (warm-up is best effort)"
1212+ ) ;
1213+ } else {
1214+ tracing:: info!(
1215+ target = report. target,
1216+ opened = report. held,
1217+ failed = report. failed,
1218+ "read pool warmed to its ceiling"
1219+ ) ;
1220+ }
1221+ Some ( report)
1222+ }
1223+
1224+ /// Open the reader pool toward its ceiling, once, then release every
1225+ /// connection back to the pool idle.
1226+ ///
1227+ /// Why: the boot ping leaves only the warm floor
1228+ /// ([`Db::reader_min_connections`]). A pod that meets a reconnect burst
1229+ /// with that floor dials the rest inside [`Db::READER_ACQUIRE_TIMEOUT`],
1230+ /// overruns it, and serves the read from the writer. Warming moves those
1231+ /// dials off the request path. sqlx's idle reaper closes the surplus
1232+ /// after `idle_timeout`, back down to the floor, so no reaping logic
1233+ /// lives here.
1234+ ///
1235+ /// Connections are held until the pass ends (an acquire from a pool
1236+ /// with an idle connection reuses it instead of dialing) and dropped
1237+ /// rather than `close_on_drop`ped, so they return idle. Bounded: a
1238+ /// single pass of bounded waves of [`Db::READER_WARM_CONCURRENCY`]
1239+ /// dials (see [`Db::warm_toward`]), each acquire bounded by
1240+ /// [`Db::READER_WARM_ACQUIRE_TIMEOUT`]; a wave that opens nothing (the
1241+ /// reader is down or saturated) ends the pass. There is no backoff and
1242+ /// no second pass.
1243+ ///
1244+ /// Uses plain `acquire`, not the instrumented reader acquire: warm
1245+ /// misses must not inflate the `buzz_db_reader_*` acquire-timeout
1246+ /// signal that the fallback alerts watch. The pool gauges
1247+ /// (`buzz_db_read_pool_*`) show the result.
1248+ ///
1249+ /// Holding connections briefly lowers the capacity left for routed
1250+ /// reads. The window is a few dial times at boot, before the pod takes
1251+ /// traffic; a read that misses in it falls back to the writer as it
1252+ /// would on any exhausted pool.
1253+ async fn warm_read_pool ( read_pool : & PgPool ) -> ReaderWarmReport {
1254+ let target = read_pool. options ( ) . get_max_connections ( ) ;
1255+ let ( held, failed) = Self :: warm_toward ( target, || async {
1256+ tokio:: time:: timeout ( Self :: READER_WARM_ACQUIRE_TIMEOUT , read_pool. acquire ( ) )
1257+ . await
1258+ . ok ( )
1259+ . and_then ( std:: result:: Result :: ok)
1260+ } )
1261+ . await ;
1262+ let report = ReaderWarmReport {
1263+ target,
1264+ held : held. len ( ) as u32 ,
1265+ failed,
1266+ } ;
1267+ // Dropping returns each connection to the pool idle (after the
1268+ // reader release check); it does not close it.
1269+ drop ( held) ;
1270+ report
1271+ }
1272+
1273+ /// The warm pass's loop, apart from the pool: runs `acquire` in waves of
1274+ /// at most [`Db::READER_WARM_CONCURRENCY`] until `target` connections are
1275+ /// held, and returns them with the number of failed acquires (`None`).
1276+ ///
1277+ /// Each wave tops up the remaining shortfall, so an acquire that missed
1278+ /// its deadline in one wave is covered by the next. Bounded three ways:
1279+ /// at most twice the waves a failure-free pass needs, so a reader that
1280+ /// admits only some dials cannot keep the pass running; a wave that
1281+ /// holds nothing new ends it; and no wave dials more than the shortfall.
1282+ /// There is no sleep or backoff between waves.
1283+ async fn warm_toward < C , F , Fut > ( target : u32 , mut acquire : F ) -> ( Vec < C > , u32 )
1284+ where
1285+ F : FnMut ( ) -> Fut ,
1286+ Fut : std:: future:: Future < Output = Option < C > > ,
1287+ {
1288+ let mut held = Vec :: new ( ) ;
1289+ let mut failed = 0_u32 ;
1290+ let max_waves = target. div_ceil ( Self :: READER_WARM_CONCURRENCY ) * 2 ;
1291+ for _ in 0 ..max_waves {
1292+ let wave = Self :: reader_warm_wave_size ( target, held. len ( ) as u32 ) ;
1293+ if wave == 0 {
1294+ break ;
1295+ }
1296+ let held_before = held. len ( ) ;
1297+ for result in futures_util:: future:: join_all ( ( 0 ..wave) . map ( |_| acquire ( ) ) ) . await {
1298+ match result {
1299+ Some ( conn) => held. push ( conn) ,
1300+ None => failed += 1 ,
1301+ }
1302+ }
1303+ if held. len ( ) == held_before {
1304+ break ;
1305+ }
1306+ }
1307+ ( held, failed)
11541308 }
11551309
11561310 /// Backoff before each boot-probe retry after an acquire that overran
@@ -1165,10 +1319,11 @@ impl Db {
11651319 Duration :: from_secs ( 2 ) ,
11661320 ] ;
11671321
1322+ /// Returns whether the reader answered.
11681323 async fn read_pool_boot_ping_once (
11691324 read_pool : PgPool ,
11701325 aurora_identity : std:: sync:: Arc < std:: sync:: OnceLock < bool > > ,
1171- ) {
1326+ ) -> bool {
11721327 let mut backoff = Self :: READER_BOOT_PROBE_BACKOFF . into_iter ( ) ;
11731328 let mut conn = loop {
11741329 let error = match observability:: acquire_reader_with_legacy_metrics (
@@ -1186,7 +1341,7 @@ impl Db {
11861341 tracing:: warn!(
11871342 "read replica unreachable at boot; serving all-writer until it recovers: {e}"
11881343 ) ;
1189- return ;
1344+ return false ;
11901345 }
11911346 }
11921347 } ;
@@ -1205,6 +1360,7 @@ impl Db {
12051360 // in the background under its own 300s deadline, outside the 150ms
12061361 // request budget, releasing each connection idle as it lands.
12071362 conn. close_on_drop ( ) ;
1363+ true
12081364 }
12091365
12101366 #[ cfg( test) ]
@@ -1825,6 +1981,119 @@ impl Db {
18251981 }
18261982}
18271983
1984+ #[ cfg( test) ]
1985+ mod reader_warm_tests {
1986+ use super :: { Db , ReaderWarmReport } ;
1987+ use std:: sync:: { Arc , OnceLock } ;
1988+
1989+ /// A lazy reader pool whose server accepts TCP and immediately closes
1990+ /// it, so every dial fails fast without Postgres.
1991+ async fn dead_reader_pool ( max_connections : u32 ) -> sqlx:: PgPool {
1992+ let listener = tokio:: net:: TcpListener :: bind ( "127.0.0.1:0" )
1993+ . await
1994+ . expect ( "bind loopback listener" ) ;
1995+ let port = listener. local_addr ( ) . expect ( "listener address" ) . port ( ) ;
1996+ tokio:: spawn ( async move {
1997+ while let Ok ( ( socket, _) ) = listener. accept ( ) . await {
1998+ drop ( socket) ;
1999+ }
2000+ } ) ;
2001+ let url =
format ! ( "postgres://[email protected] :{port}/db?sslmode=disable" ) ; 2002+ let options: sqlx:: postgres:: PgConnectOptions = url. parse ( ) . expect ( "parse url" ) ;
2003+ Db :: read_pool_options ( max_connections) . connect_lazy_with ( options)
2004+ }
2005+
2006+ /// Table test for the wave sizing: the remaining shortfall, capped at the
2007+ /// concurrency bound, never underflowing once the target is met.
2008+ #[ test]
2009+ fn warm_wave_size_is_the_shortfall_capped_at_the_concurrency_bound ( ) {
2010+ let c = Db :: READER_WARM_CONCURRENCY ;
2011+ for ( target, held, expected) in [
2012+ ( 25 , 0 , c) ,
2013+ ( 25 , 20 , c) ,
2014+ ( 25 , 22 , 3 ) ,
2015+ ( 25 , 24 , 1 ) ,
2016+ ( 25 , 25 , 0 ) ,
2017+ ( 25 , 30 , 0 ) ,
2018+ ( 1 , 0 , 1 ) ,
2019+ ( 0 , 0 , 0 ) ,
2020+ ] {
2021+ assert_eq ! (
2022+ Db :: reader_warm_wave_size( target, held) ,
2023+ expected,
2024+ "target={target} held={held}"
2025+ ) ;
2026+ }
2027+ }
2028+
2029+ /// A fake acquire that succeeds when `ok(call_index)` says so, counting
2030+ /// every call.
2031+ fn counting_acquire (
2032+ calls : & std:: sync:: atomic:: AtomicU32 ,
2033+ ok : impl Fn ( u32 ) -> bool ,
2034+ ) -> impl std:: future:: Future < Output = Option < u32 > > {
2035+ let n = calls. fetch_add ( 1 , std:: sync:: atomic:: Ordering :: SeqCst ) ;
2036+ std:: future:: ready ( ok ( n) . then_some ( n) )
2037+ }
2038+
2039+ /// With every dial succeeding the pass opens exactly the target, no more.
2040+ #[ tokio:: test]
2041+ async fn warm_toward_opens_exactly_the_target ( ) {
2042+ let calls = std:: sync:: atomic:: AtomicU32 :: new ( 0 ) ;
2043+ let ( held, failed) = Db :: warm_toward ( 25 , || counting_acquire ( & calls, |_| true ) ) . await ;
2044+ assert_eq ! ( ( held. len( ) , failed) , ( 25 , 0 ) ) ;
2045+ assert_eq ! ( calls. load( std:: sync:: atomic:: Ordering :: SeqCst ) , 25 ) ;
2046+ }
2047+
2048+ /// A reader that admits one dial per wave cannot keep the pass running:
2049+ /// attempts stop at twice the waves a clean pass needs (target 25, wave
2050+ /// 5: 10 waves of 5), well short of the shortfall-driven 25 waves.
2051+ #[ tokio:: test]
2052+ async fn warm_toward_is_bounded_when_dials_keep_missing ( ) {
2053+ let calls = std:: sync:: atomic:: AtomicU32 :: new ( 0 ) ;
2054+ let w = Db :: READER_WARM_CONCURRENCY ;
2055+ let ( held, failed) = Db :: warm_toward ( 25 , || counting_acquire ( & calls, |n| n % w == 0 ) ) . await ;
2056+ assert_eq ! ( calls. load( std:: sync:: atomic:: Ordering :: SeqCst ) , 50 ) ;
2057+ assert_eq ! ( ( held. len( ) , failed) , ( 10 , 40 ) ) ;
2058+ }
2059+
2060+ /// A wave that misses some dials is topped up by the next, so a slow
2061+ /// first wave does not leave the pool short.
2062+ #[ tokio:: test]
2063+ async fn warm_toward_tops_up_a_partially_failed_wave ( ) {
2064+ let calls = std:: sync:: atomic:: AtomicU32 :: new ( 0 ) ;
2065+ let ( held, failed) = Db :: warm_toward ( 10 , || counting_acquire ( & calls, |n| n != 1 ) ) . await ;
2066+ assert_eq ! ( ( held. len( ) , failed) , ( 10 , 1 ) ) ;
2067+ }
2068+
2069+ /// A reader that refuses every dial ends the pass after one wave: warming
2070+ /// is a single bounded attempt, not a loop that hammers a dead reader
2071+ /// `target` times.
2072+ #[ tokio:: test]
2073+ async fn warm_pass_stops_after_one_wave_that_opens_nothing ( ) {
2074+ let pool = dead_reader_pool ( 25 ) . await ;
2075+ let report = Db :: warm_read_pool ( & pool) . await ;
2076+ assert_eq ! (
2077+ report,
2078+ ReaderWarmReport {
2079+ target: 25 ,
2080+ held: 0 ,
2081+ failed: Db :: READER_WARM_CONCURRENCY ,
2082+ }
2083+ ) ;
2084+ }
2085+
2086+ /// An unreachable reader at boot is not warmed: the ping's failure ends
2087+ /// the task before any warm dial, and nothing panics or errors.
2088+ #[ tokio:: test]
2089+ async fn unreachable_reader_skips_the_warm_pass ( ) {
2090+ let pool = dead_reader_pool ( 25 ) . await ;
2091+ let report =
2092+ Db :: read_pool_boot_ping_then_warm ( pool. clone ( ) , Arc :: new ( OnceLock :: new ( ) ) ) . await ;
2093+ assert_eq ! ( report, None ) ;
2094+ }
2095+ }
2096+
18282097#[ cfg( test) ]
18292098mod pool_role_tests {
18302099 use super :: { DbPoolRole , DbPoolStats } ;
0 commit comments