Skip to content

Commit 57ede12

Browse files
ravarora2codex
andcommitted
Fence storage snapshots to lock session
Publish the completed snapshot through the same PostgreSQL session that owns the advisory lock, so a disconnected worker cannot overwrite its successor. Add a PostgreSQL regression that terminates the first lock backend and proves only the successor snapshot remains. Co-authored-by: Codex <[email protected]> Signed-off-by: Ravneet Arora <[email protected]>
1 parent 6cb30a1 commit 57ede12

2 files changed

Lines changed: 107 additions & 40 deletions

File tree

‎crates/buzz-admin/src/main.rs‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ async fn cmd_storage_snapshot(max_objects: u64) -> Result<i32> {
187187
}
188188

189189
let db = connect_db().await?;
190-
let _leader = db.try_lock_storage_accounting().await?.ok_or_else(|| {
190+
let mut leader = db.try_lock_storage_accounting().await?.ok_or_else(|| {
191191
anyhow::anyhow!("another storage-snapshot worker already holds the lease")
192192
})?;
193193
let storage = Arc::new(MediaStorage::new(&storage_config_from_env()?)?);
@@ -230,16 +230,10 @@ async fn cmd_storage_snapshot(max_objects: u64) -> Result<i32> {
230230
Ok(page)
231231
}
232232
});
233-
let snapshot_db = db.clone();
234233
let snapshot_code_sha = code_sha.clone();
235234
let persisted = persist_completed_fold(fold, move |encoded, duration_ms| async move {
236-
snapshot_db
237-
.save_storage_accounting_snapshot(
238-
&encoded,
239-
duration_ms,
240-
max_objects_db,
241-
&snapshot_code_sha,
242-
)
235+
leader
236+
.save_snapshot(&encoded, duration_ms, max_objects_db, &snapshot_code_sha)
243237
.await?;
244238
Ok(())
245239
})

‎crates/buzz-db/src/store/storage_accounting.rs‎

Lines changed: 104 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ pub const STORAGE_ACCOUNTING_LOCK_KEY: i64 = 0x4255_5a5a_5354_4f52;
1111

1212
/// Owns the detached PostgreSQL session that excludes overlapping workers.
1313
pub struct StorageAccountingLeader {
14-
_connection: PgConnection,
14+
connection: PgConnection,
1515
}
1616

1717
/// The newest complete snapshot stored by the worker.
@@ -29,36 +29,16 @@ pub struct StoredStorageSnapshot {
2929
pub code_sha: String,
3030
}
3131

32-
impl Db {
33-
/// Try to acquire the deployment-global storage-worker lease.
34-
#[datastore_span(name = "try_lock_storage_accounting", system = "postgresql")]
35-
pub async fn try_lock_storage_accounting(&self) -> Result<Option<StorageAccountingLeader>> {
36-
let mut connection = observability::acquire_writer_with_legacy_metrics(
37-
&self.pool,
38-
observability::WriterOperation::Maintenance,
39-
)
40-
.await?;
41-
let acquired = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)")
42-
.bind(STORAGE_ACCOUNTING_LOCK_KEY)
43-
.fetch_one(&mut *connection)
44-
.await?;
45-
Ok(acquired.then(|| StorageAccountingLeader {
46-
_connection: connection.detach(),
47-
}))
48-
}
49-
50-
/// Atomically replace the singleton with one fully computed snapshot.
32+
impl StorageAccountingLeader {
33+
/// Atomically replace the singleton through the lock-owning session.
5134
#[datastore_span(name = "save_storage_accounting_snapshot", system = "postgresql")]
52-
pub async fn save_storage_accounting_snapshot(
53-
&self,
35+
pub async fn save_snapshot(
36+
&mut self,
5437
snapshot: &serde_json::Value,
5538
duration_ms: i64,
5639
max_objects: i64,
5740
code_sha: &str,
5841
) -> Result<()> {
59-
let mut connection =
60-
observability::acquire_writer(&self.pool, observability::WriterOperation::Maintenance)
61-
.await?;
6242
sqlx::query(
6343
"INSERT INTO storage_accounting_snapshots \
6444
(singleton, snapshot, completed_at, duration_ms, max_objects, code_sha) \
@@ -74,10 +54,29 @@ impl Db {
7454
.bind(duration_ms)
7555
.bind(max_objects)
7656
.bind(code_sha)
77-
.execute(&mut *connection)
57+
.execute(&mut self.connection)
7858
.await?;
7959
Ok(())
8060
}
61+
}
62+
63+
impl Db {
64+
/// Try to acquire the deployment-global storage-worker lease.
65+
#[datastore_span(name = "try_lock_storage_accounting", system = "postgresql")]
66+
pub async fn try_lock_storage_accounting(&self) -> Result<Option<StorageAccountingLeader>> {
67+
let mut connection = observability::acquire_writer_with_legacy_metrics(
68+
&self.pool,
69+
observability::WriterOperation::Maintenance,
70+
)
71+
.await?;
72+
let acquired = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)")
73+
.bind(STORAGE_ACCOUNTING_LOCK_KEY)
74+
.fetch_one(&mut *connection)
75+
.await?;
76+
Ok(acquired.then(|| StorageAccountingLeader {
77+
connection: connection.detach(),
78+
}))
79+
}
8180

8281
/// Load the newest complete worker snapshot from the writer.
8382
#[datastore_span(name = "load_storage_accounting_snapshot", system = "postgresql")]
@@ -139,7 +138,7 @@ mod postgres_tests {
139138
let first = Db::from_pool(pool.clone());
140139
let second = Db::from_pool(pool.clone());
141140

142-
let leader = first
141+
let mut leader = first
143142
.try_lock_storage_accounting()
144143
.await
145144
.expect("first lock")
@@ -153,12 +152,12 @@ mod postgres_tests {
153152
"overlapping worker must not start"
154153
);
155154

156-
first
157-
.save_storage_accounting_snapshot(&serde_json::json!({"version": 1}), 10, 100, "a")
155+
leader
156+
.save_snapshot(&serde_json::json!({"version": 1}), 10, 100, "a")
158157
.await
159158
.expect("save first snapshot");
160-
first
161-
.save_storage_accounting_snapshot(&serde_json::json!({"version": 2}), 20, 200, "b")
159+
leader
160+
.save_snapshot(&serde_json::json!({"version": 2}), 20, 200, "b")
162161
.await
163162
.expect("replace snapshot");
164163
let stored = second
@@ -181,4 +180,78 @@ mod postgres_tests {
181180
.execute(&admin)
182181
.await;
183182
}
183+
184+
#[tokio::test]
185+
#[ignore = "requires Postgres"]
186+
async fn lost_lock_session_cannot_overwrite_successor_snapshot() {
187+
let admin = PgPoolOptions::new()
188+
.max_connections(1)
189+
.connect(&crate::test_support::database_url())
190+
.await
191+
.expect("connect admin");
192+
let (pool, name) = create_scratch_db(&admin).await;
193+
let first = Db::from_pool(pool.clone());
194+
let second = Db::from_pool(pool.clone());
195+
196+
let mut stale_leader = first
197+
.try_lock_storage_accounting()
198+
.await
199+
.expect("first lock")
200+
.expect("first worker owns lock");
201+
let stale_backend_pid = sqlx::query_scalar::<_, i32>("SELECT pg_backend_pid()")
202+
.fetch_one(&mut stale_leader.connection)
203+
.await
204+
.expect("load first worker backend pid");
205+
let terminated = sqlx::query_scalar::<_, bool>("SELECT pg_terminate_backend($1)")
206+
.bind(stale_backend_pid)
207+
.fetch_one(&admin)
208+
.await
209+
.expect("terminate first worker backend");
210+
assert!(terminated, "first worker backend must terminate");
211+
212+
let mut fresh_leader = tokio::time::timeout(std::time::Duration::from_secs(5), async {
213+
loop {
214+
if let Some(leader) = second
215+
.try_lock_storage_accounting()
216+
.await
217+
.expect("successor lock attempt")
218+
{
219+
break leader;
220+
}
221+
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
222+
}
223+
})
224+
.await
225+
.expect("successor acquires released lock");
226+
227+
fresh_leader
228+
.save_snapshot(&serde_json::json!({"worker": "fresh"}), 20, 200, "b")
229+
.await
230+
.expect("successor publishes fresh snapshot");
231+
stale_leader
232+
.save_snapshot(&serde_json::json!({"worker": "stale"}), 30, 100, "a")
233+
.await
234+
.expect_err("worker that lost its lock session cannot publish");
235+
236+
let stored = second
237+
.load_storage_accounting_snapshot()
238+
.await
239+
.expect("load snapshot")
240+
.expect("snapshot exists");
241+
assert_eq!(stored.snapshot, serde_json::json!({"worker": "fresh"}));
242+
assert_eq!(stored.duration_ms, 20);
243+
assert_eq!(stored.max_objects, 200);
244+
assert_eq!(stored.code_sha, "b");
245+
246+
drop(stale_leader);
247+
drop(fresh_leader);
248+
drop(first);
249+
drop(second);
250+
pool.close().await;
251+
let _ = sqlx::query(sqlx::AssertSqlSafe(format!(
252+
"DROP DATABASE IF EXISTS {name} WITH (FORCE)"
253+
)))
254+
.execute(&admin)
255+
.await;
256+
}
184257
}

0 commit comments

Comments
 (0)