77//! See [`crate::persistence::mutation_lock`] for the key hierarchy and the
88//! ordering rules. In short: lifecycle gate first, then local keys in
99//! ascending order, then `PostgreSQL` keys in ascending order on one
10- //! connection. A task never acquires a guard while it holds one.
10+ //! connection. A task never acquires a guard while it holds one, which debug
11+ //! builds check.
1112
1213use super :: { ComputeRuntime , SandboxLifecycleGuard } ;
1314use crate :: gateway_metrics:: { self , LockScope } ;
1415use crate :: grpc:: workspace:: DEFAULT_WORKSPACE_NAME ;
16+ use crate :: persistence:: lock_order;
1517use crate :: persistence:: mutation_lock:: MUTATION_LOCK_TIMEOUT ;
1618use crate :: persistence:: {
1719 DistributedMutationGuard , LockMode , MutationLockKey , MutationLockSet , PersistenceError ,
@@ -150,6 +152,7 @@ impl LocalMutationLocks {
150152 /// Acquire `set` in ascending key order. Dropping the future releases the
151153 /// keys already taken and leaves no queue entry behind.
152154 async fn acquire ( & self , set : & MutationLockSet ) -> LocalMutationGuard {
155+ lock_order:: check ( lock_order:: Lock :: Mutation ) ;
153156 let mut guards = Vec :: new ( ) ;
154157 for ( key, mode) in set. iter ( ) {
155158 let lock = self . lock_for ( key) ;
@@ -162,7 +165,10 @@ impl LocalMutationLocks {
162165 } ,
163166 } ) ;
164167 }
165- LocalMutationGuard { _guards : guards }
168+ LocalMutationGuard {
169+ _guards : guards,
170+ _order : lock_order:: hold ( lock_order:: Lock :: Mutation ) ,
171+ }
166172 }
167173
168174 fn timeout ( & self ) -> Duration {
@@ -222,6 +228,7 @@ enum LocalKeyGuard {
222228#[ must_use = "dropping the guard releases the local mutation locks" ]
223229pub struct LocalMutationGuard {
224230 _guards : Vec < LocalKeyGuard > ,
231+ _order : lock_order:: Held ,
225232}
226233
227234/// Local and, on `PostgreSQL`, distributed keys of one guarded mutation.
@@ -647,8 +654,7 @@ mod tests {
647654 #[ tokio:: test]
648655 async fn local_registry_drops_released_entries ( ) {
649656 let runtime = test_runtime ( ) . await ;
650- let sandbox = runtime
651- . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
657+ let sandbox = lock_order:: branch ( runtime. mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) ) )
652658 . await
653659 . unwrap ( ) ;
654660 let lifecycle = runtime. lock_sandbox_local ( "b" ) . await ;
@@ -690,8 +696,7 @@ mod tests {
690696 async fn local_timeout_returns_lock_timeout_and_unavailable ( ) {
691697 let runtime = test_runtime ( ) . await ;
692698 runtime. set_mutation_lock_timeout_for_tests ( Duration :: from_millis ( 50 ) ) ;
693- let held = runtime
694- . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
699+ let held = lock_order:: branch ( runtime. mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) ) )
695700 . await
696701 . unwrap ( ) ;
697702
@@ -724,8 +729,7 @@ mod tests {
724729 let runtime = test_runtime ( ) . await ;
725730 runtime. set_mutation_lock_timeout_for_tests ( Duration :: from_millis ( 50 ) ) ;
726731
727- let held = runtime
728- . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
732+ let held = lock_order:: branch ( runtime. mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) ) )
729733 . await
730734 . unwrap ( ) ;
731735 assert_eq ! ( metrics. value( WAITS ) , Some ( 1 ) ) ;
@@ -1027,6 +1031,144 @@ mod tests {
10271031 run_random_scope_mix ( & [ runtime] , "" , 64 , 50 , Duration :: from_secs ( 20 ) ) . await ;
10281032 }
10291033
1034+ #[ cfg( debug_assertions) ]
1035+ #[ tokio:: test]
1036+ #[ should_panic( expected = "mutation lock ordering violated" ) ]
1037+ async fn nested_mutation_guard_panics_in_debug_builds ( ) {
1038+ let runtime = test_runtime ( ) . await ;
1039+ let _held = runtime
1040+ . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
1041+ . await
1042+ . unwrap ( ) ;
1043+ // A different sandbox, so the scheme alone would not deadlock here.
1044+ let _nested = runtime
1045+ . mutation_guard ( MutationScope :: sandbox ( "w" , "b" ) )
1046+ . await ;
1047+ }
1048+
1049+ #[ cfg( debug_assertions) ]
1050+ #[ tokio:: test]
1051+ async fn lock_order_branches_are_separate_owners ( ) {
1052+ let runtime = test_runtime ( ) . await ;
1053+ let ( gate_tx, gate_rx) = tokio:: sync:: oneshot:: channel :: < ( ) > ( ) ;
1054+ let first = async {
1055+ let _guard = runtime
1056+ . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
1057+ . await
1058+ . unwrap ( ) ;
1059+ gate_rx. await . unwrap ( ) ;
1060+ } ;
1061+ let second = async {
1062+ let _guard = runtime
1063+ . mutation_guard ( MutationScope :: sandbox ( "w" , "b" ) )
1064+ . await
1065+ . unwrap ( ) ;
1066+ gate_tx. send ( ( ) ) . unwrap ( ) ;
1067+ } ;
1068+ tokio:: join!( lock_order:: branch( first) , lock_order:: branch( second) ) ;
1069+
1070+ let nested = tokio:: spawn ( {
1071+ let runtime = runtime. clone ( ) ;
1072+ async move {
1073+ lock_order:: branch ( async {
1074+ let _outer = runtime. lock_sandbox_local ( "c" ) . await ;
1075+ drop ( runtime. lock_sandbox_local ( "d" ) . await ) ;
1076+ } )
1077+ . await ;
1078+ }
1079+ } ) ;
1080+ assert ! (
1081+ nested. await . unwrap_err( ) . is_panic( ) ,
1082+ "nesting inside a branch"
1083+ ) ;
1084+ }
1085+
1086+ #[ cfg( debug_assertions) ]
1087+ #[ tokio:: test]
1088+ async fn lock_order_check_is_per_task_and_covers_rules_1_and_5 ( ) {
1089+ let runtime = test_runtime ( ) . await ;
1090+ let held = runtime
1091+ . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
1092+ . await
1093+ . unwrap ( ) ;
1094+ // Another task may take a guard while this one holds its own.
1095+ drop (
1096+ assert_proceeds (
1097+ spawn_guard ( & runtime, MutationScope :: sandbox ( "w" , "b" ) ) ,
1098+ "another task" ,
1099+ )
1100+ . await ,
1101+ ) ;
1102+
1103+ let nested_in_task = tokio:: spawn ( {
1104+ let runtime = runtime. clone ( ) ;
1105+ async move {
1106+ let _outer = runtime. lock_sandbox_local ( "c" ) . await ;
1107+ drop ( runtime. lock_sandbox_local ( "d" ) . await ) ;
1108+ }
1109+ } ) ;
1110+ assert ! (
1111+ nested_in_task. await . unwrap_err( ) . is_panic( ) ,
1112+ "rule 4 in a spawned task"
1113+ ) ;
1114+
1115+ let gate_under_local = tokio:: spawn ( {
1116+ let runtime = runtime. clone ( ) ;
1117+ async move {
1118+ let _local = runtime. lock_sandbox_local ( "e" ) . await ;
1119+ drop ( runtime. lifecycle_gates . lock_for ( "e" ) . await ) ;
1120+ }
1121+ } ) ;
1122+ assert ! ( gate_under_local. await . unwrap_err( ) . is_panic( ) , "rule 1" ) ;
1123+
1124+ let guard_under_ssh = tokio:: spawn ( {
1125+ let runtime = runtime. clone ( ) ;
1126+ async move {
1127+ let _ssh = runtime
1128+ . store
1129+ . acquire_ssh_identity_mutation_guard ( )
1130+ . await
1131+ . unwrap ( ) ;
1132+ drop (
1133+ runtime
1134+ . mutation_guard ( MutationScope :: sandbox ( "w" , "f" ) )
1135+ . await ,
1136+ ) ;
1137+ }
1138+ } ) ;
1139+ assert ! ( guard_under_ssh. await . unwrap_err( ) . is_panic( ) , "rule 5" ) ;
1140+
1141+ // Gate first, then local keys, then SSH identity under a guard: allowed.
1142+ let allowed = tokio:: spawn ( {
1143+ let runtime = runtime. clone ( ) ;
1144+ async move {
1145+ let gate = runtime. lifecycle_gates . lock_for ( "g" ) . await ;
1146+ let guard = runtime
1147+ . mutation_guard ( MutationScope :: sandbox ( "w" , "g" ) )
1148+ . await
1149+ . unwrap ( ) ;
1150+ drop (
1151+ runtime
1152+ . store
1153+ . acquire_ssh_identity_mutation_guard ( )
1154+ . await
1155+ . unwrap ( ) ,
1156+ ) ;
1157+ drop ( guard) ;
1158+ drop ( runtime. lock_sandbox_for_lifecycle ( & gate) . await ) ;
1159+ }
1160+ } ) ;
1161+ allowed. await . expect ( "ordered acquisitions" ) ;
1162+
1163+ drop ( held) ;
1164+ drop (
1165+ runtime
1166+ . mutation_guard ( MutationScope :: sandbox ( "w" , "a" ) )
1167+ . await
1168+ . unwrap ( ) ,
1169+ ) ;
1170+ }
1171+
10301172 #[ tokio:: test]
10311173 async fn unrelated_supervisor_state_update_does_not_wait_for_sandbox_guard ( ) {
10321174 let runtime = test_runtime ( ) . await ;
@@ -1041,13 +1183,12 @@ mod tests {
10411183 } ;
10421184 sandbox. set_phase ( SandboxPhase :: Provisioning as i32 ) ;
10431185 runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
1044- let held = runtime
1045- . mutation_guard ( MutationScope :: sandbox (
1046- "default" ,
1047- "mutation-guard-unrelated-b" ,
1048- ) )
1049- . await
1050- . unwrap ( ) ;
1186+ let held = lock_order:: branch ( runtime. mutation_guard ( MutationScope :: sandbox (
1187+ "default" ,
1188+ "mutation-guard-unrelated-b" ,
1189+ ) ) )
1190+ . await
1191+ . unwrap ( ) ;
10511192
10521193 tokio:: time:: timeout (
10531194 PROCEEDS_WITHIN ,
0 commit comments