Skip to content

Commit 3dce4ae

Browse files
TheSentinel454Elrondclaude
committed
db: warm the relay read pool toward its ceiling at boot
The reader pool boots with only its floor (max/5). A pod that meets a reconnect burst with that floor dials the rest inside the 150ms reader acquire budget, overruns it, and falls back to the writer. During a rolling deploy the newest pods saw exactly that. After a successful boot ping, a background pass (never on the startup path) opens the pool toward max_connections in bounded waves of 5 dials, then releases the connections idle; sqlx's idle_timeout reaps them back to the floor. Single pass, no retry; a wave that opens nothing ends it. Failures log at warn with counts and never affect startup. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent b310fea commit 3dce4ae

2 files changed

Lines changed: 323 additions & 3 deletions

File tree

‎crates/buzz-db/src/runtime/mod.rs‎

Lines changed: 272 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,17 @@ fn read_budget_from_ms(ms: u64) -> Option<Duration> {
569569
}
570570
}
571571

572+
/// Outcome of one reader-pool warm pass (see [`Db::warm_read_pool`]).
573+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
574+
struct ReaderWarmReport {
575+
/// Connections the pass aimed to hold: the pool's ceiling.
576+
target: u32,
577+
/// Connections opened (or reused idle) and then released back to idle.
578+
held: u32,
579+
/// Acquires that errored or timed out.
580+
failed: u32,
581+
}
582+
572583
/// Snapshot of Postgres connection pool utilisation.
573584
#[derive(Debug, Clone, Copy)]
574585
pub struct DbPoolStats {
@@ -1135,6 +1146,9 @@ impl Db {
11351146
/// pool up to [`Db::reader_min_connections`] in the background, so a
11361147
/// freshly started pod does not meet its first traffic with an empty
11371148
/// reader pool and pay every dial inside the 150ms request budget.
1149+
/// It then warms the pool toward its ceiling ([`Db::warm_read_pool`]),
1150+
/// still in the spawned task: a failed or partial warm only logs and
1151+
/// never affects startup or readiness.
11381152
/// Probe acquires that overrun that budget are retried with
11391153
/// [`Db::READER_BOOT_PROBE_BACKOFF`].
11401154
///
@@ -1150,7 +1164,147 @@ impl Db {
11501164
return;
11511165
};
11521166
let aurora_identity = self.reader_aurora_identity.clone();
1153-
tokio::spawn(Self::read_pool_boot_ping_once(read_pool, aurora_identity));
1167+
tokio::spawn(Self::read_pool_boot_ping_then_warm(
1168+
read_pool,
1169+
aurora_identity,
1170+
));
1171+
}
1172+
1173+
/// Connections each warm wave dials at once. Small enough that a pod
1174+
/// starting alongside its peers does not stampede the reader with
1175+
/// simultaneous TCP+TLS+SCRAM handshakes (a stampede is what pushes a
1176+
/// dial past [`Db::READER_ACQUIRE_TIMEOUT`]), large enough that a full
1177+
/// pool is warm within a handful of dial times.
1178+
const READER_WARM_CONCURRENCY: u32 = 5;
1179+
1180+
/// Bound on one warm acquire. The pool's own acquire timeout
1181+
/// ([`Db::READER_ACQUIRE_TIMEOUT`]) already caps a dial; this is the
1182+
/// explicit backstop so a warm acquire can never outlive a wave.
1183+
const READER_WARM_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(1);
1184+
1185+
/// Connections the next warm wave should open: the remaining shortfall,
1186+
/// at most [`Db::READER_WARM_CONCURRENCY`].
1187+
fn reader_warm_wave_size(target: u32, held: u32) -> u32 {
1188+
target
1189+
.saturating_sub(held)
1190+
.min(Self::READER_WARM_CONCURRENCY)
1191+
}
1192+
1193+
/// The boot ping, then (only if the reader answered) a warm pass.
1194+
///
1195+
/// Returns `None` when the ping found the reader unreachable: warming a
1196+
/// pool whose reader is down would only add abandoned dials.
1197+
async fn read_pool_boot_ping_then_warm(
1198+
read_pool: PgPool,
1199+
aurora_identity: std::sync::Arc<std::sync::OnceLock<bool>>,
1200+
) -> Option<ReaderWarmReport> {
1201+
if !Self::read_pool_boot_ping_once(read_pool.clone(), aurora_identity).await {
1202+
return None;
1203+
}
1204+
let report = Self::warm_read_pool(&read_pool).await;
1205+
if report.held < report.target {
1206+
tracing::warn!(
1207+
target = report.target,
1208+
opened = report.held,
1209+
failed = report.failed,
1210+
"read pool warm-up opened fewer connections than its ceiling; \
1211+
the rest dial on demand (warm-up is best effort)"
1212+
);
1213+
} else {
1214+
tracing::info!(
1215+
target = report.target,
1216+
opened = report.held,
1217+
failed = report.failed,
1218+
"read pool warmed to its ceiling"
1219+
);
1220+
}
1221+
Some(report)
1222+
}
1223+
1224+
/// Open the reader pool toward its ceiling, once, then release every
1225+
/// connection back to the pool idle.
1226+
///
1227+
/// Why: the boot ping leaves only the warm floor
1228+
/// ([`Db::reader_min_connections`]). A pod that meets a reconnect burst
1229+
/// with that floor dials the rest inside [`Db::READER_ACQUIRE_TIMEOUT`],
1230+
/// overruns it, and serves the read from the writer. Warming moves those
1231+
/// dials off the request path. sqlx's idle reaper closes the surplus
1232+
/// after `idle_timeout`, back down to the floor, so no reaping logic
1233+
/// lives here.
1234+
///
1235+
/// Connections are held until the pass ends (an acquire from a pool
1236+
/// with an idle connection reuses it instead of dialing) and dropped
1237+
/// rather than `close_on_drop`ped, so they return idle. Bounded: a
1238+
/// single pass of bounded waves of [`Db::READER_WARM_CONCURRENCY`]
1239+
/// dials (see [`Db::warm_toward`]), each acquire bounded by
1240+
/// [`Db::READER_WARM_ACQUIRE_TIMEOUT`]; a wave that opens nothing (the
1241+
/// reader is down or saturated) ends the pass. There is no backoff and
1242+
/// no second pass.
1243+
///
1244+
/// Uses plain `acquire`, not the instrumented reader acquire: warm
1245+
/// misses must not inflate the `buzz_db_reader_*` acquire-timeout
1246+
/// signal that the fallback alerts watch. The pool gauges
1247+
/// (`buzz_db_read_pool_*`) show the result.
1248+
///
1249+
/// Holding connections briefly lowers the capacity left for routed
1250+
/// reads. The window is a few dial times at boot, before the pod takes
1251+
/// traffic; a read that misses in it falls back to the writer as it
1252+
/// would on any exhausted pool.
1253+
async fn warm_read_pool(read_pool: &PgPool) -> ReaderWarmReport {
1254+
let target = read_pool.options().get_max_connections();
1255+
let (held, failed) = Self::warm_toward(target, || async {
1256+
tokio::time::timeout(Self::READER_WARM_ACQUIRE_TIMEOUT, read_pool.acquire())
1257+
.await
1258+
.ok()
1259+
.and_then(std::result::Result::ok)
1260+
})
1261+
.await;
1262+
let report = ReaderWarmReport {
1263+
target,
1264+
held: held.len() as u32,
1265+
failed,
1266+
};
1267+
// Dropping returns each connection to the pool idle (after the
1268+
// reader release check); it does not close it.
1269+
drop(held);
1270+
report
1271+
}
1272+
1273+
/// The warm pass's loop, apart from the pool: runs `acquire` in waves of
1274+
/// at most [`Db::READER_WARM_CONCURRENCY`] until `target` connections are
1275+
/// held, and returns them with the number of failed acquires (`None`).
1276+
///
1277+
/// Each wave tops up the remaining shortfall, so an acquire that missed
1278+
/// its deadline in one wave is covered by the next. Bounded three ways:
1279+
/// at most twice the waves a failure-free pass needs, so a reader that
1280+
/// admits only some dials cannot keep the pass running; a wave that
1281+
/// holds nothing new ends it; and no wave dials more than the shortfall.
1282+
/// There is no sleep or backoff between waves.
1283+
async fn warm_toward<C, F, Fut>(target: u32, mut acquire: F) -> (Vec<C>, u32)
1284+
where
1285+
F: FnMut() -> Fut,
1286+
Fut: std::future::Future<Output = Option<C>>,
1287+
{
1288+
let mut held = Vec::new();
1289+
let mut failed = 0_u32;
1290+
let max_waves = target.div_ceil(Self::READER_WARM_CONCURRENCY) * 2;
1291+
for _ in 0..max_waves {
1292+
let wave = Self::reader_warm_wave_size(target, held.len() as u32);
1293+
if wave == 0 {
1294+
break;
1295+
}
1296+
let held_before = held.len();
1297+
for result in futures_util::future::join_all((0..wave).map(|_| acquire())).await {
1298+
match result {
1299+
Some(conn) => held.push(conn),
1300+
None => failed += 1,
1301+
}
1302+
}
1303+
if held.len() == held_before {
1304+
break;
1305+
}
1306+
}
1307+
(held, failed)
11541308
}
11551309

11561310
/// Backoff before each boot-probe retry after an acquire that overran
@@ -1165,10 +1319,11 @@ impl Db {
11651319
Duration::from_secs(2),
11661320
];
11671321

1322+
/// Returns whether the reader answered.
11681323
async fn read_pool_boot_ping_once(
11691324
read_pool: PgPool,
11701325
aurora_identity: std::sync::Arc<std::sync::OnceLock<bool>>,
1171-
) {
1326+
) -> bool {
11721327
let mut backoff = Self::READER_BOOT_PROBE_BACKOFF.into_iter();
11731328
let mut conn = loop {
11741329
let error = match observability::acquire_reader_with_legacy_metrics(
@@ -1186,7 +1341,7 @@ impl Db {
11861341
tracing::warn!(
11871342
"read replica unreachable at boot; serving all-writer until it recovers: {e}"
11881343
);
1189-
return;
1344+
return false;
11901345
}
11911346
}
11921347
};
@@ -1205,6 +1360,7 @@ impl Db {
12051360
// in the background under its own 300s deadline, outside the 150ms
12061361
// request budget, releasing each connection idle as it lands.
12071362
conn.close_on_drop();
1363+
true
12081364
}
12091365

12101366
#[cfg(test)]
@@ -1825,6 +1981,119 @@ impl Db {
18251981
}
18261982
}
18271983

1984+
#[cfg(test)]
1985+
mod reader_warm_tests {
1986+
use super::{Db, ReaderWarmReport};
1987+
use std::sync::{Arc, OnceLock};
1988+
1989+
/// A lazy reader pool whose server accepts TCP and immediately closes
1990+
/// it, so every dial fails fast without Postgres.
1991+
async fn dead_reader_pool(max_connections: u32) -> sqlx::PgPool {
1992+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
1993+
.await
1994+
.expect("bind loopback listener");
1995+
let port = listener.local_addr().expect("listener address").port();
1996+
tokio::spawn(async move {
1997+
while let Ok((socket, _)) = listener.accept().await {
1998+
drop(socket);
1999+
}
2000+
});
2001+
let url = format!("postgres://[email protected]:{port}/db?sslmode=disable");
2002+
let options: sqlx::postgres::PgConnectOptions = url.parse().expect("parse url");
2003+
Db::read_pool_options(max_connections).connect_lazy_with(options)
2004+
}
2005+
2006+
/// Table test for the wave sizing: the remaining shortfall, capped at the
2007+
/// concurrency bound, never underflowing once the target is met.
2008+
#[test]
2009+
fn warm_wave_size_is_the_shortfall_capped_at_the_concurrency_bound() {
2010+
let c = Db::READER_WARM_CONCURRENCY;
2011+
for (target, held, expected) in [
2012+
(25, 0, c),
2013+
(25, 20, c),
2014+
(25, 22, 3),
2015+
(25, 24, 1),
2016+
(25, 25, 0),
2017+
(25, 30, 0),
2018+
(1, 0, 1),
2019+
(0, 0, 0),
2020+
] {
2021+
assert_eq!(
2022+
Db::reader_warm_wave_size(target, held),
2023+
expected,
2024+
"target={target} held={held}"
2025+
);
2026+
}
2027+
}
2028+
2029+
/// A fake acquire that succeeds when `ok(call_index)` says so, counting
2030+
/// every call.
2031+
fn counting_acquire(
2032+
calls: &std::sync::atomic::AtomicU32,
2033+
ok: impl Fn(u32) -> bool,
2034+
) -> impl std::future::Future<Output = Option<u32>> {
2035+
let n = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
2036+
std::future::ready(ok(n).then_some(n))
2037+
}
2038+
2039+
/// With every dial succeeding the pass opens exactly the target, no more.
2040+
#[tokio::test]
2041+
async fn warm_toward_opens_exactly_the_target() {
2042+
let calls = std::sync::atomic::AtomicU32::new(0);
2043+
let (held, failed) = Db::warm_toward(25, || counting_acquire(&calls, |_| true)).await;
2044+
assert_eq!((held.len(), failed), (25, 0));
2045+
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 25);
2046+
}
2047+
2048+
/// A reader that admits one dial per wave cannot keep the pass running:
2049+
/// attempts stop at twice the waves a clean pass needs (target 25, wave
2050+
/// 5: 10 waves of 5), well short of the shortfall-driven 25 waves.
2051+
#[tokio::test]
2052+
async fn warm_toward_is_bounded_when_dials_keep_missing() {
2053+
let calls = std::sync::atomic::AtomicU32::new(0);
2054+
let w = Db::READER_WARM_CONCURRENCY;
2055+
let (held, failed) = Db::warm_toward(25, || counting_acquire(&calls, |n| n % w == 0)).await;
2056+
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 50);
2057+
assert_eq!((held.len(), failed), (10, 40));
2058+
}
2059+
2060+
/// A wave that misses some dials is topped up by the next, so a slow
2061+
/// first wave does not leave the pool short.
2062+
#[tokio::test]
2063+
async fn warm_toward_tops_up_a_partially_failed_wave() {
2064+
let calls = std::sync::atomic::AtomicU32::new(0);
2065+
let (held, failed) = Db::warm_toward(10, || counting_acquire(&calls, |n| n != 1)).await;
2066+
assert_eq!((held.len(), failed), (10, 1));
2067+
}
2068+
2069+
/// A reader that refuses every dial ends the pass after one wave: warming
2070+
/// is a single bounded attempt, not a loop that hammers a dead reader
2071+
/// `target` times.
2072+
#[tokio::test]
2073+
async fn warm_pass_stops_after_one_wave_that_opens_nothing() {
2074+
let pool = dead_reader_pool(25).await;
2075+
let report = Db::warm_read_pool(&pool).await;
2076+
assert_eq!(
2077+
report,
2078+
ReaderWarmReport {
2079+
target: 25,
2080+
held: 0,
2081+
failed: Db::READER_WARM_CONCURRENCY,
2082+
}
2083+
);
2084+
}
2085+
2086+
/// An unreachable reader at boot is not warmed: the ping's failure ends
2087+
/// the task before any warm dial, and nothing panics or errors.
2088+
#[tokio::test]
2089+
async fn unreachable_reader_skips_the_warm_pass() {
2090+
let pool = dead_reader_pool(25).await;
2091+
let report =
2092+
Db::read_pool_boot_ping_then_warm(pool.clone(), Arc::new(OnceLock::new())).await;
2093+
assert_eq!(report, None);
2094+
}
2095+
}
2096+
18282097
#[cfg(test)]
18292098
mod pool_role_tests {
18302099
use super::{DbPoolRole, DbPoolStats};

‎crates/buzz-db/src/runtime/tests/reader_pool_postgres_tests.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,57 @@ async fn reader_boot_ping_warms_pool_to_min_connections() {
126126
drop_scratch_db(&admin, db.pool.clone(), &wname).await;
127127
}
128128

129+
/// After the boot ping the background warm pass opens the reader pool to its
130+
/// ceiling and releases every connection idle. The caller must not wait for
131+
/// it: `spawn_read_pool_boot_ping` returns with the pool still empty. Without
132+
/// the warm pass the pool stops at the floor; if the pass closed its
133+
/// connections instead of releasing them, `size` and `idle` never reach the
134+
/// ceiling together.
135+
#[tokio::test]
136+
#[ignore = "requires Postgres"]
137+
async fn reader_boot_ping_warms_pool_to_max_in_background() {
138+
let max = 15;
139+
let (admin, mut db, wname, scratch_url) = same_db_reader("warm_max", max).await;
140+
// The production reader acquire budget (150ms) lets a dial on a loaded
141+
// test host miss; this test is about the warm pass reaching the ceiling,
142+
// so give each dial room.
143+
db.read_pool = Some(
144+
Db::read_pool_options(max)
145+
.acquire_timeout(Duration::from_secs(5))
146+
.connect_lazy_with(Db::reader_connect_options(
147+
scratch_url.parse().expect("parse scratch url"),
148+
)),
149+
);
150+
let read_pool = db.read_pool.clone().expect("reader pool configured");
151+
assert!(
152+
Db::reader_min_connections(max) < max,
153+
"fixture needs a floor below the ceiling"
154+
);
155+
156+
db.spawn_read_pool_boot_ping();
157+
assert_eq!(
158+
read_pool.size(),
159+
0,
160+
"spawning the ping and warm pass must not wait on them"
161+
);
162+
163+
await_idle_floor(&read_pool, max).await;
164+
// Settled at the ceiling and no further.
165+
tokio::time::sleep(Duration::from_millis(200)).await;
166+
assert_eq!(
167+
read_pool.size(),
168+
max,
169+
"warm pass must stay within the ceiling"
170+
);
171+
assert_eq!(
172+
read_pool.num_idle(),
173+
max as usize,
174+
"warmed connections stay idle"
175+
);
176+
177+
drop_scratch_db(&admin, db.pool.clone(), &wname).await;
178+
}
179+
129180
/// A warm reader connection whose server stops answering (here: the
130181
/// heartbeat observation blocks behind an exclusive lock) must fail the
131182
/// routed read closed to the writer within the proof budget, not hang the

0 commit comments

Comments
 (0)