@@ -71,6 +71,13 @@ struct DeleteGateRegistry {
7171}
7272
7373impl DeleteGateRegistry {
74+ async fn lock_for ( & self , sandbox_id : & str ) -> SandboxDeleteGuard {
75+ let gate = self . gate_for ( sandbox_id) ;
76+ SandboxDeleteGuard {
77+ _guard : gate. lock_owned ( ) . await ,
78+ }
79+ }
80+
7481 fn gate_for ( & self , sandbox_id : & str ) -> Arc < Mutex < ( ) > > {
7582 let mut gates = self
7683 . gates
@@ -96,6 +103,16 @@ impl DeleteGateRegistry {
96103 }
97104}
98105
106+ /// Proof that the current delete operation holds its sandbox-ID gate.
107+ ///
108+ /// Delete code must acquire this guard before taking `ComputeRuntime::sync_lock`.
109+ /// Passing it to `lock_global_for_delete` makes that ordering visible at every
110+ /// global-lock acquisition in the delete path.
111+ #[ derive( Debug ) ]
112+ struct SandboxDeleteGuard {
113+ _guard : tokio:: sync:: OwnedMutexGuard < ( ) > ,
114+ }
115+
99116#[ derive( Debug ) ]
100117struct DeleteTransition {
101118 /// Snapshot restored if the driver result is ambiguous and no newer write
@@ -407,6 +424,16 @@ impl ComputeRuntime {
407424 self . sync_lock . clone ( ) . lock_owned ( ) . await
408425 }
409426
427+ /// Acquires the process-wide lock for code that already holds the
428+ /// sandbox-ID delete gate. The guard parameter documents and enforces that
429+ /// delete-path callers acquire locks in delete-gate -> global-lock order.
430+ async fn lock_global_for_delete (
431+ & self ,
432+ _delete_guard : & SandboxDeleteGuard ,
433+ ) -> tokio:: sync:: OwnedMutexGuard < ( ) > {
434+ self . sync_lock . clone ( ) . lock_owned ( ) . await
435+ }
436+
410437 pub async fn new_docker (
411438 config : openshell_core:: Config ,
412439 docker_config : DockerComputeConfig ,
@@ -673,10 +700,9 @@ impl ComputeRuntime {
673700 . map_err ( |e| Status :: internal ( format ! ( "fetch sandbox failed: {e}" ) ) ) ?
674701 . ok_or_else ( || Status :: not_found ( "sandbox not found" ) ) ?;
675702 let candidate_id = candidate. object_id ( ) . to_string ( ) ;
676- let delete_gate = self . delete_gates . gate_for ( & candidate_id) ;
677- let _delete_guard = delete_gate. lock ( ) . await ;
703+ let delete_guard = self . delete_gates . lock_for ( & candidate_id) . await ;
678704
679- let guard = self . sync_lock . lock ( ) . await ;
705+ let guard = self . lock_global_for_delete ( & delete_guard ) . await ;
680706 let current = self
681707 . store
682708 . get_message :: < Sandbox > ( & candidate_id)
@@ -722,17 +748,20 @@ impl ComputeRuntime {
722748 Ok ( response) => {
723749 let deleted = response. into_inner ( ) . deleted ;
724750 if deleted {
725- self . cleanup_local_state_if_sandbox_absent ( & candidate_id)
751+ self . cleanup_local_state_if_sandbox_absent ( & delete_guard , & candidate_id)
726752 . await ?;
727- } else if !self . remove_deleting_sandbox_record ( & candidate_id) . await {
753+ } else if !self
754+ . remove_deleting_sandbox_record ( & delete_guard, & candidate_id)
755+ . await
756+ {
728757 return Err ( Status :: internal (
729758 "compute resource was absent, but gateway cleanup did not complete" ,
730759 ) ) ;
731760 }
732761 Ok ( deleted)
733762 }
734763 Err ( err) => {
735- self . recover_failed_delete ( & transition) . await ;
764+ self . recover_failed_delete ( & delete_guard , & transition) . await ;
736765 Err ( Status :: internal ( format ! (
737766 "delete sandbox failed: {}" ,
738767 err. message( )
@@ -812,8 +841,12 @@ impl ComputeRuntime {
812841 ///
813842 /// Benign resource-version changes are retried, but cleanup stops if the
814843 /// row leaves `Deleting`; that state belongs to a concurrent writer.
815- async fn remove_deleting_sandbox_record ( & self , sandbox_id : & str ) -> bool {
816- let _guard = self . sync_lock . lock ( ) . await ;
844+ async fn remove_deleting_sandbox_record (
845+ & self ,
846+ delete_guard : & SandboxDeleteGuard ,
847+ sandbox_id : & str ,
848+ ) -> bool {
849+ let _guard = self . lock_global_for_delete ( delete_guard) . await ;
817850 for attempt in 1 ..=DELETE_PHASE_CAS_RETRY_LIMIT {
818851 let record = match self . store . get ( Sandbox :: object_type ( ) , sandbox_id) . await {
819852 Ok ( Some ( record) ) => record,
@@ -941,14 +974,18 @@ impl ComputeRuntime {
941974 /// exact `Deleting` resource version to apply one of three outcomes:
942975 /// reconcile an observed backend snapshot, remove a confirmed-absent
943976 /// backend, or restore the pre-delete snapshot when lookup is inconclusive.
944- async fn recover_failed_delete ( & self , transition : & DeleteTransition ) {
977+ async fn recover_failed_delete (
978+ & self ,
979+ delete_guard : & SandboxDeleteGuard ,
980+ transition : & DeleteTransition ,
981+ ) {
945982 let sandbox_id = transition. deleting . object_id ( ) ;
946983 let sandbox_name = transition. deleting . object_name ( ) ;
947984 let deleting_resource_version = sandbox_resource_version ( & transition. deleting ) ;
948985
949986 // The driver lookup is deliberately outside the process-wide guard.
950987 let observed = self . get_driver_sandbox ( sandbox_id, sandbox_name) . await ;
951- let _guard = self . sync_lock . lock ( ) . await ;
988+ let _guard = self . lock_global_for_delete ( delete_guard ) . await ;
952989
953990 match observed {
954991 Ok ( Some ( snapshot) ) if snapshot. id == sandbox_id && snapshot. status . is_some ( ) => {
@@ -1708,8 +1745,12 @@ impl ComputeRuntime {
17081745 }
17091746 }
17101747
1711- async fn cleanup_local_state_if_sandbox_absent ( & self , sandbox_id : & str ) -> Result < ( ) , Status > {
1712- let _guard = self . sync_lock . lock ( ) . await ;
1748+ async fn cleanup_local_state_if_sandbox_absent (
1749+ & self ,
1750+ delete_guard : & SandboxDeleteGuard ,
1751+ sandbox_id : & str ,
1752+ ) -> Result < ( ) , Status > {
1753+ let _guard = self . lock_global_for_delete ( delete_guard) . await ;
17131754 let record = self
17141755 . store
17151756 . get ( Sandbox :: object_type ( ) , sandbox_id)
0 commit comments