Skip to content

Commit 6a15fd9

Browse files
committed
refactor(server): make delete lock ordering explicit
Signed-off-by: Piotr Mlocek <[email protected]>
1 parent 8283eec commit 6a15fd9

1 file changed

Lines changed: 53 additions & 12 deletions

File tree

  • crates/openshell-server/src/compute

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 53 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,13 @@ struct DeleteGateRegistry {
7171
}
7272

7373
impl DeleteGateRegistry {
74+
async fn lock_for(&self, sandbox_id: &str) -> SandboxDeleteGuard {
75+
let gate = self.gate_for(sandbox_id);
76+
SandboxDeleteGuard {
77+
_guard: gate.lock_owned().await,
78+
}
79+
}
80+
7481
fn gate_for(&self, sandbox_id: &str) -> Arc<Mutex<()>> {
7582
let mut gates = self
7683
.gates
@@ -96,6 +103,16 @@ impl DeleteGateRegistry {
96103
}
97104
}
98105

106+
/// Proof that the current delete operation holds its sandbox-ID gate.
107+
///
108+
/// Delete code must acquire this guard before taking `ComputeRuntime::sync_lock`.
109+
/// Passing it to `lock_global_for_delete` makes that ordering visible at every
110+
/// global-lock acquisition in the delete path.
111+
#[derive(Debug)]
112+
struct SandboxDeleteGuard {
113+
_guard: tokio::sync::OwnedMutexGuard<()>,
114+
}
115+
99116
#[derive(Debug)]
100117
struct DeleteTransition {
101118
/// Snapshot restored if the driver result is ambiguous and no newer write
@@ -407,6 +424,16 @@ impl ComputeRuntime {
407424
self.sync_lock.clone().lock_owned().await
408425
}
409426

427+
/// Acquires the process-wide lock for code that already holds the
428+
/// sandbox-ID delete gate. The guard parameter documents and enforces that
429+
/// delete-path callers acquire locks in delete-gate -> global-lock order.
430+
async fn lock_global_for_delete(
431+
&self,
432+
_delete_guard: &SandboxDeleteGuard,
433+
) -> tokio::sync::OwnedMutexGuard<()> {
434+
self.sync_lock.clone().lock_owned().await
435+
}
436+
410437
pub async fn new_docker(
411438
config: openshell_core::Config,
412439
docker_config: DockerComputeConfig,
@@ -673,10 +700,9 @@ impl ComputeRuntime {
673700
.map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))?
674701
.ok_or_else(|| Status::not_found("sandbox not found"))?;
675702
let candidate_id = candidate.object_id().to_string();
676-
let delete_gate = self.delete_gates.gate_for(&candidate_id);
677-
let _delete_guard = delete_gate.lock().await;
703+
let delete_guard = self.delete_gates.lock_for(&candidate_id).await;
678704

679-
let guard = self.sync_lock.lock().await;
705+
let guard = self.lock_global_for_delete(&delete_guard).await;
680706
let current = self
681707
.store
682708
.get_message::<Sandbox>(&candidate_id)
@@ -722,17 +748,20 @@ impl ComputeRuntime {
722748
Ok(response) => {
723749
let deleted = response.into_inner().deleted;
724750
if deleted {
725-
self.cleanup_local_state_if_sandbox_absent(&candidate_id)
751+
self.cleanup_local_state_if_sandbox_absent(&delete_guard, &candidate_id)
726752
.await?;
727-
} else if !self.remove_deleting_sandbox_record(&candidate_id).await {
753+
} else if !self
754+
.remove_deleting_sandbox_record(&delete_guard, &candidate_id)
755+
.await
756+
{
728757
return Err(Status::internal(
729758
"compute resource was absent, but gateway cleanup did not complete",
730759
));
731760
}
732761
Ok(deleted)
733762
}
734763
Err(err) => {
735-
self.recover_failed_delete(&transition).await;
764+
self.recover_failed_delete(&delete_guard, &transition).await;
736765
Err(Status::internal(format!(
737766
"delete sandbox failed: {}",
738767
err.message()
@@ -812,8 +841,12 @@ impl ComputeRuntime {
812841
///
813842
/// Benign resource-version changes are retried, but cleanup stops if the
814843
/// row leaves `Deleting`; that state belongs to a concurrent writer.
815-
async fn remove_deleting_sandbox_record(&self, sandbox_id: &str) -> bool {
816-
let _guard = self.sync_lock.lock().await;
844+
async fn remove_deleting_sandbox_record(
845+
&self,
846+
delete_guard: &SandboxDeleteGuard,
847+
sandbox_id: &str,
848+
) -> bool {
849+
let _guard = self.lock_global_for_delete(delete_guard).await;
817850
for attempt in 1..=DELETE_PHASE_CAS_RETRY_LIMIT {
818851
let record = match self.store.get(Sandbox::object_type(), sandbox_id).await {
819852
Ok(Some(record)) => record,
@@ -941,14 +974,18 @@ impl ComputeRuntime {
941974
/// exact `Deleting` resource version to apply one of three outcomes:
942975
/// reconcile an observed backend snapshot, remove a confirmed-absent
943976
/// backend, or restore the pre-delete snapshot when lookup is inconclusive.
944-
async fn recover_failed_delete(&self, transition: &DeleteTransition) {
977+
async fn recover_failed_delete(
978+
&self,
979+
delete_guard: &SandboxDeleteGuard,
980+
transition: &DeleteTransition,
981+
) {
945982
let sandbox_id = transition.deleting.object_id();
946983
let sandbox_name = transition.deleting.object_name();
947984
let deleting_resource_version = sandbox_resource_version(&transition.deleting);
948985

949986
// The driver lookup is deliberately outside the process-wide guard.
950987
let observed = self.get_driver_sandbox(sandbox_id, sandbox_name).await;
951-
let _guard = self.sync_lock.lock().await;
988+
let _guard = self.lock_global_for_delete(delete_guard).await;
952989

953990
match observed {
954991
Ok(Some(snapshot)) if snapshot.id == sandbox_id && snapshot.status.is_some() => {
@@ -1708,8 +1745,12 @@ impl ComputeRuntime {
17081745
}
17091746
}
17101747

1711-
async fn cleanup_local_state_if_sandbox_absent(&self, sandbox_id: &str) -> Result<(), Status> {
1712-
let _guard = self.sync_lock.lock().await;
1748+
async fn cleanup_local_state_if_sandbox_absent(
1749+
&self,
1750+
delete_guard: &SandboxDeleteGuard,
1751+
sandbox_id: &str,
1752+
) -> Result<(), Status> {
1753+
let _guard = self.lock_global_for_delete(delete_guard).await;
17131754
let record = self
17141755
.store
17151756
.get(Sandbox::object_type(), sandbox_id)

0 commit comments

Comments
 (0)