@@ -11,7 +11,7 @@ pub const STORAGE_ACCOUNTING_LOCK_KEY: i64 = 0x4255_5a5a_5354_4f52;
1111
1212/// Owns the detached PostgreSQL session that excludes overlapping workers.
1313pub struct StorageAccountingLeader {
14- _connection : PgConnection ,
14+ connection : PgConnection ,
1515}
1616
1717/// The newest complete snapshot stored by the worker.
@@ -29,36 +29,16 @@ pub struct StoredStorageSnapshot {
2929 pub code_sha : String ,
3030}
3131
32- impl Db {
33- /// Try to acquire the deployment-global storage-worker lease.
34- #[ datastore_span( name = "try_lock_storage_accounting" , system = "postgresql" ) ]
35- pub async fn try_lock_storage_accounting ( & self ) -> Result < Option < StorageAccountingLeader > > {
36- let mut connection = observability:: acquire_writer_with_legacy_metrics (
37- & self . pool ,
38- observability:: WriterOperation :: Maintenance ,
39- )
40- . await ?;
41- let acquired = sqlx:: query_scalar :: < _ , bool > ( "SELECT pg_try_advisory_lock($1)" )
42- . bind ( STORAGE_ACCOUNTING_LOCK_KEY )
43- . fetch_one ( & mut * connection)
44- . await ?;
45- Ok ( acquired. then ( || StorageAccountingLeader {
46- _connection : connection. detach ( ) ,
47- } ) )
48- }
49-
50- /// Atomically replace the singleton with one fully computed snapshot.
32+ impl StorageAccountingLeader {
33+ /// Atomically replace the singleton through the lock-owning session.
5134 #[ datastore_span( name = "save_storage_accounting_snapshot" , system = "postgresql" ) ]
52- pub async fn save_storage_accounting_snapshot (
53- & self ,
35+ pub async fn save_snapshot (
36+ & mut self ,
5437 snapshot : & serde_json:: Value ,
5538 duration_ms : i64 ,
5639 max_objects : i64 ,
5740 code_sha : & str ,
5841 ) -> Result < ( ) > {
59- let mut connection =
60- observability:: acquire_writer ( & self . pool , observability:: WriterOperation :: Maintenance )
61- . await ?;
6242 sqlx:: query (
6343 "INSERT INTO storage_accounting_snapshots \
6444 (singleton, snapshot, completed_at, duration_ms, max_objects, code_sha) \
@@ -74,10 +54,29 @@ impl Db {
7454 . bind ( duration_ms)
7555 . bind ( max_objects)
7656 . bind ( code_sha)
77- . execute ( & mut * connection)
57+ . execute ( & mut self . connection )
7858 . await ?;
7959 Ok ( ( ) )
8060 }
61+ }
62+
63+ impl Db {
64+ /// Try to acquire the deployment-global storage-worker lease.
65+ #[ datastore_span( name = "try_lock_storage_accounting" , system = "postgresql" ) ]
66+ pub async fn try_lock_storage_accounting ( & self ) -> Result < Option < StorageAccountingLeader > > {
67+ let mut connection = observability:: acquire_writer_with_legacy_metrics (
68+ & self . pool ,
69+ observability:: WriterOperation :: Maintenance ,
70+ )
71+ . await ?;
72+ let acquired = sqlx:: query_scalar :: < _ , bool > ( "SELECT pg_try_advisory_lock($1)" )
73+ . bind ( STORAGE_ACCOUNTING_LOCK_KEY )
74+ . fetch_one ( & mut * connection)
75+ . await ?;
76+ Ok ( acquired. then ( || StorageAccountingLeader {
77+ connection : connection. detach ( ) ,
78+ } ) )
79+ }
8180
8281 /// Load the newest complete worker snapshot from the writer.
8382 #[ datastore_span( name = "load_storage_accounting_snapshot" , system = "postgresql" ) ]
@@ -139,7 +138,7 @@ mod postgres_tests {
139138 let first = Db :: from_pool ( pool. clone ( ) ) ;
140139 let second = Db :: from_pool ( pool. clone ( ) ) ;
141140
142- let leader = first
141+ let mut leader = first
143142 . try_lock_storage_accounting ( )
144143 . await
145144 . expect ( "first lock" )
@@ -153,12 +152,12 @@ mod postgres_tests {
153152 "overlapping worker must not start"
154153 ) ;
155154
156- first
157- . save_storage_accounting_snapshot ( & serde_json:: json!( { "version" : 1 } ) , 10 , 100 , "a" )
155+ leader
156+ . save_snapshot ( & serde_json:: json!( { "version" : 1 } ) , 10 , 100 , "a" )
158157 . await
159158 . expect ( "save first snapshot" ) ;
160- first
161- . save_storage_accounting_snapshot ( & serde_json:: json!( { "version" : 2 } ) , 20 , 200 , "b" )
159+ leader
160+ . save_snapshot ( & serde_json:: json!( { "version" : 2 } ) , 20 , 200 , "b" )
162161 . await
163162 . expect ( "replace snapshot" ) ;
164163 let stored = second
@@ -181,4 +180,78 @@ mod postgres_tests {
181180 . execute ( & admin)
182181 . await ;
183182 }
183+
184+ #[ tokio:: test]
185+ #[ ignore = "requires Postgres" ]
186+ async fn lost_lock_session_cannot_overwrite_successor_snapshot ( ) {
187+ let admin = PgPoolOptions :: new ( )
188+ . max_connections ( 1 )
189+ . connect ( & crate :: test_support:: database_url ( ) )
190+ . await
191+ . expect ( "connect admin" ) ;
192+ let ( pool, name) = create_scratch_db ( & admin) . await ;
193+ let first = Db :: from_pool ( pool. clone ( ) ) ;
194+ let second = Db :: from_pool ( pool. clone ( ) ) ;
195+
196+ let mut stale_leader = first
197+ . try_lock_storage_accounting ( )
198+ . await
199+ . expect ( "first lock" )
200+ . expect ( "first worker owns lock" ) ;
201+ let stale_backend_pid = sqlx:: query_scalar :: < _ , i32 > ( "SELECT pg_backend_pid()" )
202+ . fetch_one ( & mut stale_leader. connection )
203+ . await
204+ . expect ( "load first worker backend pid" ) ;
205+ let terminated = sqlx:: query_scalar :: < _ , bool > ( "SELECT pg_terminate_backend($1)" )
206+ . bind ( stale_backend_pid)
207+ . fetch_one ( & admin)
208+ . await
209+ . expect ( "terminate first worker backend" ) ;
210+ assert ! ( terminated, "first worker backend must terminate" ) ;
211+
212+ let mut fresh_leader = tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , async {
213+ loop {
214+ if let Some ( leader) = second
215+ . try_lock_storage_accounting ( )
216+ . await
217+ . expect ( "successor lock attempt" )
218+ {
219+ break leader;
220+ }
221+ tokio:: time:: sleep ( std:: time:: Duration :: from_millis ( 25 ) ) . await ;
222+ }
223+ } )
224+ . await
225+ . expect ( "successor acquires released lock" ) ;
226+
227+ fresh_leader
228+ . save_snapshot ( & serde_json:: json!( { "worker" : "fresh" } ) , 20 , 200 , "b" )
229+ . await
230+ . expect ( "successor publishes fresh snapshot" ) ;
231+ stale_leader
232+ . save_snapshot ( & serde_json:: json!( { "worker" : "stale" } ) , 30 , 100 , "a" )
233+ . await
234+ . expect_err ( "worker that lost its lock session cannot publish" ) ;
235+
236+ let stored = second
237+ . load_storage_accounting_snapshot ( )
238+ . await
239+ . expect ( "load snapshot" )
240+ . expect ( "snapshot exists" ) ;
241+ assert_eq ! ( stored. snapshot, serde_json:: json!( { "worker" : "fresh" } ) ) ;
242+ assert_eq ! ( stored. duration_ms, 20 ) ;
243+ assert_eq ! ( stored. max_objects, 200 ) ;
244+ assert_eq ! ( stored. code_sha, "b" ) ;
245+
246+ drop ( stale_leader) ;
247+ drop ( fresh_leader) ;
248+ drop ( first) ;
249+ drop ( second) ;
250+ pool. close ( ) . await ;
251+ let _ = sqlx:: query ( sqlx:: AssertSqlSafe ( format ! (
252+ "DROP DATABASE IF EXISTS {name} WITH (FORCE)"
253+ ) ) )
254+ . execute ( & admin)
255+ . await ;
256+ }
184257}
0 commit comments