Skip to content

Commit ae62a03

Browse files
committed
feat(server): check mutation lock ordering in debug builds
The mutation lock ordering rules were prose only, so a nested guard on a different key passed every test and could only deadlock under contention. Debug builds now register each held guard with the Tokio task (or the thread, outside a task) that acquired it, and panic before waiting when a local key, a blocking lifecycle gate or the SSH identity lock would break rules 1, 4 or 5. Startup endpoint-status reconciliation and the tests that hold a guard while driving guarded code run under lock_order::branch(), which marks deliberate concurrency within one task. Release builds compile the check out. Part of NVIDIA#3528 Signed-off-by: Emilien Macchi <[email protected]>
1 parent eb3eb15 commit ae62a03

12 files changed

Lines changed: 555 additions & 79 deletions

File tree

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ struct LifecycleGateRegistry {
222222

223223
impl LifecycleGateRegistry {
224224
async fn lock_for(&self, sandbox_id: &str) -> SandboxLifecycleGuard {
225+
crate::persistence::lock_order::check(crate::persistence::lock_order::Lock::LifecycleGate);
225226
let gate = self.gate_for(sandbox_id);
226227
SandboxLifecycleGuard {
227228
sandbox_id: sandbox_id.to_string(),
@@ -10295,10 +10296,11 @@ mod tests {
1029510296
.await
1029610297
.unwrap();
1029710298

10298-
let unrelated = runtime
10299-
.mutation_guard(MutationScope::sandbox("default", "sb-2"))
10300-
.await
10301-
.unwrap();
10299+
let unrelated = crate::persistence::lock_order::branch(
10300+
runtime.mutation_guard(MutationScope::sandbox("default", "sb-2")),
10301+
)
10302+
.await
10303+
.unwrap();
1030210304
let held = runtime.lock_sandbox_local("sb-1").await;
1030310305
let mut cleanup = tokio::spawn({
1030410306
let runtime = runtime.clone();

‎crates/openshell-server/src/compute/mutation_guard.rs‎

Lines changed: 156 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@
77
//! See [`crate::persistence::mutation_lock`] for the key hierarchy and the
88
//! ordering rules. In short: lifecycle gate first, then local keys in
99
//! ascending order, then `PostgreSQL` keys in ascending order on one
10-
//! connection. A task never acquires a guard while it holds one.
10+
//! connection. A task never acquires a guard while it holds one, which debug
11+
//! builds check.
1112
1213
use super::{ComputeRuntime, SandboxLifecycleGuard};
1314
use crate::gateway_metrics::{self, LockScope};
1415
use crate::grpc::workspace::DEFAULT_WORKSPACE_NAME;
16+
use crate::persistence::lock_order;
1517
use crate::persistence::mutation_lock::MUTATION_LOCK_TIMEOUT;
1618
use crate::persistence::{
1719
DistributedMutationGuard, LockMode, MutationLockKey, MutationLockSet, PersistenceError,
@@ -150,6 +152,7 @@ impl LocalMutationLocks {
150152
/// Acquire `set` in ascending key order. Dropping the future releases the
151153
/// keys already taken and leaves no queue entry behind.
152154
async fn acquire(&self, set: &MutationLockSet) -> LocalMutationGuard {
155+
lock_order::check(lock_order::Lock::Mutation);
153156
let mut guards = Vec::new();
154157
for (key, mode) in set.iter() {
155158
let lock = self.lock_for(key);
@@ -162,7 +165,10 @@ impl LocalMutationLocks {
162165
},
163166
});
164167
}
165-
LocalMutationGuard { _guards: guards }
168+
LocalMutationGuard {
169+
_guards: guards,
170+
_order: lock_order::hold(lock_order::Lock::Mutation),
171+
}
166172
}
167173

168174
fn timeout(&self) -> Duration {
@@ -222,6 +228,7 @@ enum LocalKeyGuard {
222228
#[must_use = "dropping the guard releases the local mutation locks"]
223229
pub struct LocalMutationGuard {
224230
_guards: Vec<LocalKeyGuard>,
231+
_order: lock_order::Held,
225232
}
226233

227234
/// Local and, on `PostgreSQL`, distributed keys of one guarded mutation.
@@ -647,8 +654,7 @@ mod tests {
647654
#[tokio::test]
648655
async fn local_registry_drops_released_entries() {
649656
let runtime = test_runtime().await;
650-
let sandbox = runtime
651-
.mutation_guard(MutationScope::sandbox("w", "a"))
657+
let sandbox = lock_order::branch(runtime.mutation_guard(MutationScope::sandbox("w", "a")))
652658
.await
653659
.unwrap();
654660
let lifecycle = runtime.lock_sandbox_local("b").await;
@@ -690,8 +696,7 @@ mod tests {
690696
async fn local_timeout_returns_lock_timeout_and_unavailable() {
691697
let runtime = test_runtime().await;
692698
runtime.set_mutation_lock_timeout_for_tests(Duration::from_millis(50));
693-
let held = runtime
694-
.mutation_guard(MutationScope::sandbox("w", "a"))
699+
let held = lock_order::branch(runtime.mutation_guard(MutationScope::sandbox("w", "a")))
695700
.await
696701
.unwrap();
697702

@@ -724,8 +729,7 @@ mod tests {
724729
let runtime = test_runtime().await;
725730
runtime.set_mutation_lock_timeout_for_tests(Duration::from_millis(50));
726731

727-
let held = runtime
728-
.mutation_guard(MutationScope::sandbox("w", "a"))
732+
let held = lock_order::branch(runtime.mutation_guard(MutationScope::sandbox("w", "a")))
729733
.await
730734
.unwrap();
731735
assert_eq!(metrics.value(WAITS), Some(1));
@@ -1027,6 +1031,144 @@ mod tests {
10271031
run_random_scope_mix(&[runtime], "", 64, 50, Duration::from_secs(20)).await;
10281032
}
10291033

1034+
#[cfg(debug_assertions)]
1035+
#[tokio::test]
1036+
#[should_panic(expected = "mutation lock ordering violated")]
1037+
async fn nested_mutation_guard_panics_in_debug_builds() {
1038+
let runtime = test_runtime().await;
1039+
let _held = runtime
1040+
.mutation_guard(MutationScope::sandbox("w", "a"))
1041+
.await
1042+
.unwrap();
1043+
// A different sandbox, so the scheme alone would not deadlock here.
1044+
let _nested = runtime
1045+
.mutation_guard(MutationScope::sandbox("w", "b"))
1046+
.await;
1047+
}
1048+
1049+
#[cfg(debug_assertions)]
1050+
#[tokio::test]
1051+
async fn lock_order_branches_are_separate_owners() {
1052+
let runtime = test_runtime().await;
1053+
let (gate_tx, gate_rx) = tokio::sync::oneshot::channel::<()>();
1054+
let first = async {
1055+
let _guard = runtime
1056+
.mutation_guard(MutationScope::sandbox("w", "a"))
1057+
.await
1058+
.unwrap();
1059+
gate_rx.await.unwrap();
1060+
};
1061+
let second = async {
1062+
let _guard = runtime
1063+
.mutation_guard(MutationScope::sandbox("w", "b"))
1064+
.await
1065+
.unwrap();
1066+
gate_tx.send(()).unwrap();
1067+
};
1068+
tokio::join!(lock_order::branch(first), lock_order::branch(second));
1069+
1070+
let nested = tokio::spawn({
1071+
let runtime = runtime.clone();
1072+
async move {
1073+
lock_order::branch(async {
1074+
let _outer = runtime.lock_sandbox_local("c").await;
1075+
drop(runtime.lock_sandbox_local("d").await);
1076+
})
1077+
.await;
1078+
}
1079+
});
1080+
assert!(
1081+
nested.await.unwrap_err().is_panic(),
1082+
"nesting inside a branch"
1083+
);
1084+
}
1085+
1086+
#[cfg(debug_assertions)]
1087+
#[tokio::test]
1088+
async fn lock_order_check_is_per_task_and_covers_rules_1_and_5() {
1089+
let runtime = test_runtime().await;
1090+
let held = runtime
1091+
.mutation_guard(MutationScope::sandbox("w", "a"))
1092+
.await
1093+
.unwrap();
1094+
// Another task may take a guard while this one holds its own.
1095+
drop(
1096+
assert_proceeds(
1097+
spawn_guard(&runtime, MutationScope::sandbox("w", "b")),
1098+
"another task",
1099+
)
1100+
.await,
1101+
);
1102+
1103+
let nested_in_task = tokio::spawn({
1104+
let runtime = runtime.clone();
1105+
async move {
1106+
let _outer = runtime.lock_sandbox_local("c").await;
1107+
drop(runtime.lock_sandbox_local("d").await);
1108+
}
1109+
});
1110+
assert!(
1111+
nested_in_task.await.unwrap_err().is_panic(),
1112+
"rule 4 in a spawned task"
1113+
);
1114+
1115+
let gate_under_local = tokio::spawn({
1116+
let runtime = runtime.clone();
1117+
async move {
1118+
let _local = runtime.lock_sandbox_local("e").await;
1119+
drop(runtime.lifecycle_gates.lock_for("e").await);
1120+
}
1121+
});
1122+
assert!(gate_under_local.await.unwrap_err().is_panic(), "rule 1");
1123+
1124+
let guard_under_ssh = tokio::spawn({
1125+
let runtime = runtime.clone();
1126+
async move {
1127+
let _ssh = runtime
1128+
.store
1129+
.acquire_ssh_identity_mutation_guard()
1130+
.await
1131+
.unwrap();
1132+
drop(
1133+
runtime
1134+
.mutation_guard(MutationScope::sandbox("w", "f"))
1135+
.await,
1136+
);
1137+
}
1138+
});
1139+
assert!(guard_under_ssh.await.unwrap_err().is_panic(), "rule 5");
1140+
1141+
// Gate first, then local keys, then SSH identity under a guard: allowed.
1142+
let allowed = tokio::spawn({
1143+
let runtime = runtime.clone();
1144+
async move {
1145+
let gate = runtime.lifecycle_gates.lock_for("g").await;
1146+
let guard = runtime
1147+
.mutation_guard(MutationScope::sandbox("w", "g"))
1148+
.await
1149+
.unwrap();
1150+
drop(
1151+
runtime
1152+
.store
1153+
.acquire_ssh_identity_mutation_guard()
1154+
.await
1155+
.unwrap(),
1156+
);
1157+
drop(guard);
1158+
drop(runtime.lock_sandbox_for_lifecycle(&gate).await);
1159+
}
1160+
});
1161+
allowed.await.expect("ordered acquisitions");
1162+
1163+
drop(held);
1164+
drop(
1165+
runtime
1166+
.mutation_guard(MutationScope::sandbox("w", "a"))
1167+
.await
1168+
.unwrap(),
1169+
);
1170+
}
1171+
10301172
#[tokio::test]
10311173
async fn unrelated_supervisor_state_update_does_not_wait_for_sandbox_guard() {
10321174
let runtime = test_runtime().await;
@@ -1041,13 +1183,12 @@ mod tests {
10411183
};
10421184
sandbox.set_phase(SandboxPhase::Provisioning as i32);
10431185
runtime.store.put_message(&sandbox).await.unwrap();
1044-
let held = runtime
1045-
.mutation_guard(MutationScope::sandbox(
1046-
"default",
1047-
"mutation-guard-unrelated-b",
1048-
))
1049-
.await
1050-
.unwrap();
1186+
let held = lock_order::branch(runtime.mutation_guard(MutationScope::sandbox(
1187+
"default",
1188+
"mutation-guard-unrelated-b",
1189+
)))
1190+
.await
1191+
.unwrap();
10511192

10521193
tokio::time::timeout(
10531194
PROCEEDS_WITHIN,

‎crates/openshell-server/src/grpc/policy.rs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8155,8 +8155,14 @@ mod tests {
81558155
// A duplicate report racing the initial registration must share one
81568156
// persisted transition; either may acquire the lifecycle fence first.
81578157
let (first, duplicate) = tokio::join!(
8158-
handle_report_sandbox_configuration(&state, request()),
8159-
handle_report_sandbox_configuration(&state, request()),
8158+
crate::persistence::lock_order::branch(handle_report_sandbox_configuration(
8159+
&state,
8160+
request()
8161+
)),
8162+
crate::persistence::lock_order::branch(handle_report_sandbox_configuration(
8163+
&state,
8164+
request()
8165+
)),
81608166
);
81618167
first.unwrap();
81628168
duplicate.unwrap();
@@ -21652,11 +21658,13 @@ mod tests {
2165221658
))
2165321659
.await
2165421660
.unwrap();
21655-
let unrelated_guard = state
21656-
.compute
21657-
.mutation_guard(MutationScope::sandbox("default", "sb-setting-unrelated"))
21658-
.await
21659-
.unwrap();
21661+
let unrelated_guard = crate::persistence::lock_order::branch(
21662+
state
21663+
.compute
21664+
.mutation_guard(MutationScope::sandbox("default", "sb-setting-unrelated")),
21665+
)
21666+
.await
21667+
.unwrap();
2166021668

2166121669
tokio::time::timeout(
2166221670
std::time::Duration::from_secs(5),

‎crates/openshell-server/src/grpc/policy/endpoint_status.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -553,15 +553,20 @@ pub async fn invalidate_endpoint_status_on_startup(state: &Arc<ServerState>) ->
553553
))
554554
})?;
555555
// Each future holds at most its own sandbox guard and never waits on
556-
// another, so running them in one task cannot deadlock.
556+
// another, so running them in one task cannot deadlock. Each runs as
557+
// its own lock-order branch so the debug-build nesting check agrees.
557558
futures::stream::iter(
558559
page.messages
559560
.iter()
560561
.filter(|sandbox| has_endpoint_status(sandbox))
561562
.map(Ok::<_, Status>),
562563
)
563564
.try_for_each_concurrent(ENDPOINT_STARTUP_RECONCILIATION_CONCURRENCY, |candidate| {
564-
invalidate_sandbox_endpoint_status_on_startup(state, candidate, retry_deadline)
565+
crate::persistence::lock_order::branch(invalidate_sandbox_endpoint_status_on_startup(
566+
state,
567+
candidate,
568+
retry_deadline,
569+
))
565570
})
566571
.await?;
567572
let Some(next_cursor) = page.next_cursor else {

0 commit comments

Comments
 (0)