Skip to content

Commit a929532

Browse files
authored
fix(admin): allow cold storage worker DB startup (#7770)
## Summary The standalone S3 accounting worker can exit before its scan begins because database startup inherits the relay’s three-second connection budget. A cold DNS probe took about five seconds. Give `buzz-admin storage-snapshot` its own pool: one connection, no minimum idle connections, and a 30-second acquisition budget. Retry startup timeouts and transport failures at most twice, after two and five seconds. Total startup wait is bounded to 97 seconds. Configuration, authentication, TLS, and protocol errors fail immediately. Log each startup attempt’s elapsed time and error category without connection credentials. Relay defaults stay unchanged. Retries stop before lock acquisition; the same lock-owning database session still publishes the completed snapshot. ### Related issue Follow-up to #7543. No duplicate startup fix found. ### Testing Archived-command check: built a nextest archive, temporarily moved `target/debug/buzz-admin` out of the checkout, and ran `scripts/postgres-test-run.sh --archive-file <archive>`. Both tests failed before the path fix and passed afterward. The tests use nextest’s relocated executable path, with the original Cargo path retained for local runs. Manually restored the old startup call and ran `scripts/postgres-test-run.sh -p buzz-admin --test postgres_storage_snapshot -E 'test(storage_snapshot_retries_then)'`. It failed before reaching S3; restoring the fix passed. This confirms the test exercises the actual worker command. Local `just ci` passed all stages before mobile tests, which stopped at SDK discovery in the `objective_c` native hook. The default Xcode tools report an unaccepted license. Two existing PostgreSQL tests hit connection timeouts in the parallel run and passed individually. After merge, build a worker image and repeat the cold-pod staging smoke test through snapshot publication before changing relay metrics mode. Generated with Codex --------- Signed-off-by: Ravneet Arora <[email protected]>
1 parent 77729ab commit a929532

7 files changed

Lines changed: 581 additions & 13 deletions

File tree

‎Justfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,7 @@ test-unit:
390390
cargo nextest run -p buzz-media --lib \
391391
-E 'test(=bucket_index::tests::bucket_snapshot_json_round_trip_preserves_community_keys)'
392392
cargo nextest run -p buzz-admin \
393-
-E 'test(=storage_snapshot_tests::failed_fold_never_invokes_snapshot_persistence)'
393+
-E 'test(storage_snapshot)'
394394
# Multi-tenant conformance gate (buzz-conformance): the independent
395395
# replay checker + golden fixtures. No infra — pure in-process trace
396396
# replay — so it belongs in the unit job. Run all targets (lib + the

‎crates/buzz-admin/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,6 @@ sqlx = { workspace = true }
3737
url = { workspace = true }
3838
uuid = { workspace = true }
3939
clap = { version = "4", features = ["derive"] }
40+
41+
[dev-dependencies]
42+
tokio = { workspace = true, features = ["test-util"] }

‎crates/buzz-admin/src/main.rs‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
//! the guard against parallel adds (e.g. `xargs -P`).
2222
2323
mod deletions;
24+
mod storage_snapshot_startup;
2425

2526
use std::future::Future;
2627
use std::sync::atomic::{AtomicU64, Ordering};
@@ -186,7 +187,7 @@ async fn cmd_storage_snapshot(max_objects: u64) -> Result<i32> {
186187
return Err(anyhow::anyhow!("--max-objects must be greater than zero"));
187188
}
188189

189-
let db = connect_db().await?;
190+
let db = storage_snapshot_startup::connect_db().await?;
190191
let mut leader = db.try_lock_storage_accounting().await?.ok_or_else(|| {
191192
anyhow::anyhow!("another storage-snapshot worker already holds the lease")
192193
})?;
@@ -576,17 +577,17 @@ async fn connect_member_services() -> Result<(Db, Arc<PubSubManager>, Keys)> {
576577
}
577578

578579
async fn connect_db() -> Result<Db> {
580+
Ok(Db::new(&db_config_from_env()).await?)
581+
}
582+
583+
fn db_config_from_env() -> DbConfig {
579584
let db_url = std::env::var("DATABASE_URL")
580585
.unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string());
581-
let db = Db::new(
582-
&DbConfig {
583-
database_url: db_url,
584-
..DbConfig::default()
585-
}
586-
.with_session_timeouts_from_env(),
587-
)
588-
.await?;
589-
Ok(db)
586+
DbConfig {
587+
database_url: db_url,
588+
..DbConfig::default()
589+
}
590+
.with_session_timeouts_from_env()
590591
}
591592

592593
/// Resolve the deployment's tenant from the configured `RELAY_URL` host.
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
//! Bounded cold database startup for the run-once storage accounting worker.
2+
3+
use std::future::Future;
4+
use std::io::ErrorKind;
5+
use std::time::Duration;
6+
7+
use anyhow::{Context, Result};
8+
use buzz_db::{Db, DbConfig, DbError};
9+
use tokio::time::{sleep, timeout, Instant};
10+
11+
const ACQUIRE_TIMEOUT: Duration = Duration::from_secs(30);
12+
const RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(2), Duration::from_secs(5)];
13+
14+
/// Establish the worker's database pool before it acquires its accounting lease.
15+
pub(super) async fn connect_db() -> Result<Db> {
16+
let config = worker_config(crate::db_config_from_env());
17+
connect_with_retry(|| Db::new(&config)).await
18+
}
19+
20+
fn worker_config(config: DbConfig) -> DbConfig {
21+
DbConfig {
22+
max_connections: 1,
23+
// The worker detaches its lock-owning session. Do not open idle
24+
// replacements while that session scans S3 and publishes the result.
25+
min_connections: 0,
26+
acquire_timeout_secs: ACQUIRE_TIMEOUT.as_secs(),
27+
..config
28+
}
29+
}
30+
31+
// Only initial connection establishment is retried. Once the command owns
32+
// the advisory lock, reconnecting would lose its publication fence.
33+
async fn connect_with_retry<T, Connect, Attempt>(mut connect: Connect) -> Result<T>
34+
where
35+
Connect: FnMut() -> Attempt,
36+
Attempt: Future<Output = buzz_db::Result<T>>,
37+
{
38+
let started = Instant::now();
39+
// Three attempts of at most 30s, plus 2s and 5s backoffs: at most 97s.
40+
for attempt in 0..=RETRY_DELAYS.len() {
41+
let attempt_started = Instant::now();
42+
eprintln!(
43+
"{}",
44+
serde_json::json!({
45+
"event": "storage_snapshot_db_connect_started",
46+
"stage": "db_connect",
47+
"attempt": attempt + 1,
48+
"timeout_ms": ACQUIRE_TIMEOUT.as_millis(),
49+
})
50+
);
51+
// Bound the entire initialization future, including session setup.
52+
let result = timeout(ACQUIRE_TIMEOUT, connect())
53+
.await
54+
.unwrap_or_else(|_| Err(sqlx::Error::PoolTimedOut.into()));
55+
match result {
56+
Ok(db) => {
57+
eprintln!(
58+
"{}",
59+
serde_json::json!({
60+
"event": "storage_snapshot_db_connect_completed",
61+
"stage": "db_connect",
62+
"attempt": attempt + 1,
63+
"attempt_elapsed_ms": attempt_started.elapsed().as_millis(),
64+
"elapsed_ms": started.elapsed().as_millis(),
65+
})
66+
);
67+
return Ok(db);
68+
}
69+
Err(error) => {
70+
let delay = RETRY_DELAYS
71+
.get(attempt)
72+
.copied()
73+
.filter(|_| retryable(&error));
74+
let class = error_class(&error);
75+
// Avoid raw connection errors/URLs: configuration and driver
76+
// messages can contain credentials. Keep the source on the
77+
// returned error, but print only this bounded classification.
78+
eprintln!(
79+
"{}",
80+
serde_json::json!({
81+
"event": "storage_snapshot_db_connect_failed",
82+
"stage": "db_connect",
83+
"attempt": attempt + 1,
84+
"attempt_elapsed_ms": attempt_started.elapsed().as_millis(),
85+
"elapsed_ms": started.elapsed().as_millis(),
86+
"error_class": class,
87+
"io_kind": match &error {
88+
DbError::Sqlx(sqlx::Error::Io(error)) => Some(format!("{:?}", error.kind())),
89+
_ => None,
90+
},
91+
"sqlstate": match &error {
92+
DbError::Sqlx(sqlx::Error::Database(error)) => error.code(),
93+
_ => None,
94+
},
95+
"retry_in_ms": delay.map(|value| value.as_millis()),
96+
})
97+
);
98+
match delay {
99+
Some(delay) => sleep(delay).await,
100+
None => {
101+
return Err(error).with_context(|| {
102+
format!(
103+
"storage snapshot database startup failed after {} attempt(s) and {}ms ({class})",
104+
attempt + 1,
105+
started.elapsed().as_millis(),
106+
)
107+
});
108+
}
109+
}
110+
}
111+
}
112+
}
113+
unreachable!("the last connection attempt always returns")
114+
}
115+
116+
fn retryable(error: &DbError) -> bool {
117+
match error {
118+
DbError::Sqlx(sqlx::Error::PoolTimedOut) => true,
119+
// SQLx already backs off on connection refusal and transient server
120+
// errors. Other transport/resolver failures may also be transient;
121+
// allow only the bounded startup retries, excluding local input and
122+
// permission errors. Unknown resolver errors are not labeled as DNS.
123+
DbError::Sqlx(sqlx::Error::Io(error)) => !matches!(
124+
error.kind(),
125+
ErrorKind::InvalidInput
126+
| ErrorKind::InvalidData
127+
| ErrorKind::PermissionDenied
128+
| ErrorKind::NotFound
129+
| ErrorKind::Unsupported
130+
),
131+
// Includes authentication, TLS, URL configuration and protocol errors.
132+
_ => false,
133+
}
134+
}
135+
136+
fn error_class(error: &DbError) -> &'static str {
137+
match error {
138+
DbError::Sqlx(sqlx::Error::PoolTimedOut) => "timeout",
139+
DbError::Sqlx(sqlx::Error::Io(_)) => "io",
140+
DbError::Sqlx(sqlx::Error::Tls(_)) => "tls",
141+
DbError::Sqlx(sqlx::Error::Configuration(_)) => "configuration",
142+
DbError::Sqlx(sqlx::Error::Database(error)) => {
143+
if error.code().is_some_and(|code| code.starts_with("28")) {
144+
"authentication"
145+
} else {
146+
"database"
147+
}
148+
}
149+
DbError::Sqlx(sqlx::Error::Protocol(_)) => "protocol",
150+
_ => "other",
151+
}
152+
}
153+
154+
#[cfg(test)]
155+
mod tests;
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
use std::cell::Cell;
2+
3+
use super::*;
4+
5+
#[test]
6+
fn worker_pool_overrides_do_not_change_relay_defaults_or_session_policy() {
7+
let defaults = DbConfig::default();
8+
let worker = worker_config(DbConfig {
9+
lock_timeout_ms: 123,
10+
idle_txn_timeout_ms: 456,
11+
statement_timeout_ms: 789,
12+
..defaults.clone()
13+
});
14+
assert_eq!(worker.max_connections, 1);
15+
assert_eq!(worker.min_connections, 0);
16+
assert_eq!(worker.acquire_timeout_secs, 30);
17+
assert_eq!(worker.lock_timeout_ms, 123);
18+
assert_eq!(worker.idle_txn_timeout_ms, 456);
19+
assert_eq!(worker.statement_timeout_ms, 789);
20+
assert_eq!(defaults.max_connections, 20);
21+
assert_eq!(defaults.min_connections, 2);
22+
assert_eq!(defaults.acquire_timeout_secs, 3);
23+
}
24+
25+
#[tokio::test(start_paused = true)]
26+
async fn cold_connection_can_exceed_the_old_three_second_budget() {
27+
let started = Instant::now();
28+
connect_with_retry(|| async {
29+
sleep(Duration::from_secs(6)).await;
30+
Ok(())
31+
})
32+
.await
33+
.expect("cold startup succeeds");
34+
assert_eq!(started.elapsed(), Duration::from_secs(6));
35+
}
36+
37+
#[tokio::test(start_paused = true)]
38+
async fn transient_failure_backs_off_before_a_successful_attempt() {
39+
let attempts = Cell::new(0);
40+
let started = Instant::now();
41+
let result = connect_with_retry(|| {
42+
attempts.set(attempts.get() + 1);
43+
let attempt = attempts.get();
44+
async move {
45+
if attempt == 1 {
46+
Err(sqlx::Error::Io(ErrorKind::ConnectionReset.into()).into())
47+
} else {
48+
Ok(42)
49+
}
50+
}
51+
})
52+
.await;
53+
assert_eq!(result.expect("second attempt succeeds"), 42);
54+
assert_eq!(attempts.get(), 2);
55+
assert_eq!(started.elapsed(), Duration::from_secs(2));
56+
}
57+
58+
#[tokio::test(start_paused = true)]
59+
async fn retry_exhaustion_propagates_the_last_error_after_three_attempts() {
60+
let attempts = Cell::new(0);
61+
let started = Instant::now();
62+
let error = connect_with_retry(|| {
63+
attempts.set(attempts.get() + 1);
64+
async { Err::<(), _>(sqlx::Error::Io(ErrorKind::ConnectionReset.into()).into()) }
65+
})
66+
.await
67+
.expect_err("exhausted startup must fail");
68+
assert_eq!(attempts.get(), 3);
69+
assert_eq!(started.elapsed(), Duration::from_secs(7));
70+
assert!(error.to_string().contains("3 attempt(s)"));
71+
assert!(matches!(
72+
error.downcast_ref::<DbError>(),
73+
Some(DbError::Sqlx(sqlx::Error::Io(_)))
74+
));
75+
}
76+
77+
#[tokio::test(start_paused = true)]
78+
async fn hung_initialization_is_bounded_to_97_seconds_including_backoff() {
79+
let attempts = Cell::new(0);
80+
let started = Instant::now();
81+
let error = connect_with_retry(|| {
82+
attempts.set(attempts.get() + 1);
83+
std::future::pending::<buzz_db::Result<()>>()
84+
})
85+
.await
86+
.expect_err("hung connection setup must time out");
87+
assert_eq!(attempts.get(), 3);
88+
assert_eq!(started.elapsed(), Duration::from_secs(97));
89+
assert!(matches!(
90+
error.downcast_ref::<DbError>(),
91+
Some(DbError::Sqlx(sqlx::Error::PoolTimedOut))
92+
));
93+
}
94+
95+
#[tokio::test(start_paused = true)]
96+
async fn permanent_errors_fail_immediately_and_do_not_expose_connection_details() {
97+
let attempts = Cell::new(0);
98+
let started = Instant::now();
99+
for driver_error in [
100+
sqlx::Error::Configuration("sensitive-url".into()),
101+
sqlx::Error::Tls("sensitive-certificate".into()),
102+
sqlx::Error::Protocol("sensitive-server-reply".into()),
103+
sqlx::Error::Io(ErrorKind::PermissionDenied.into()),
104+
] {
105+
let mut error = Some(driver_error);
106+
let result = connect_with_retry(|| {
107+
attempts.set(attempts.get() + 1);
108+
let error = error.take().expect("permanent errors must not retry");
109+
async move { Err::<(), _>(error.into()) }
110+
})
111+
.await
112+
.expect_err("permanent error");
113+
assert!(!result.to_string().contains("sensitive"));
114+
}
115+
assert_eq!(attempts.get(), 4);
116+
assert_eq!(started.elapsed(), Duration::ZERO);
117+
}

0 commit comments

Comments
 (0)