Skip to content

Commit 859406f

Browse files
committed
fix(server): bound the provisioning-deadline scan's lock wait
The provisioning-deadline scan takes each provisioning sandbox's local S(global) S(workspace) X(sandbox) keys in ascending key order and waited for them with no deadline. When a sandbox's key sorts before its workspace's key, the scan held X(sandbox) while it waited behind a provider or profile writer, so start, stop and delete for that sandbox waited for the writer too. In the other order, the scan held S(workspace) while it waited for a busy sandbox, and provider writers on that workspace queued behind it. Either way the rest of the pass waited as well. The scan now waits at most 500 ms for a sandbox's keys. If they aren't free, it skips the sandbox and picks it up on the next pass a second later. The local acquisition is cancel-safe, so giving up releases the keys it already took. A skip logs at debug, or as a warning when the sandbox is already past its deadline, since its expiry is then delayed. The test that pinned the old wait is replaced by tests for both key orders: a lifecycle lock or a provider writer proceeds and the scan finishes while the other side holds its key, the skipped sandbox is untouched and expires once the writer releases, and a skip doesn't end the pass for other sandboxes. Part of #4307 Closes #4547 Signed-off-by: Emilien Macchi <[email protected]>
1 parent fc0e9db commit 859406f

4 files changed

Lines changed: 231 additions & 16 deletions

File tree

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 189 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18811,40 +18811,217 @@ mod tests {
1881118811
}
1881218812

1881318813
#[tokio::test]
18814-
async fn provisioning_reconcile_waits_for_a_local_workspace_writer() {
18814+
async fn provisioning_reconcile_skips_a_sandbox_behind_a_workspace_writer() {
18815+
use crate::persistence::MutationLockKey;
1881518816
let runtime = test_runtime(ControlledDriver::new()).await;
18817+
// Keys are taken in ascending order. With the sandbox key first, the
18818+
// scan holds X(sandbox) while it waits for S(workspace).
18819+
let workspace_key = MutationLockKey::Workspace("default").advisory_key();
18820+
let sandbox_id = (0..1000)
18821+
.map(|i| format!("sb-ws-writer-{i}"))
18822+
.find(|id| MutationLockKey::Sandbox(id).advisory_key() < workspace_key)
18823+
.expect("a sandbox key that sorts before the workspace key");
18824+
let sandbox_key = MutationLockKey::Sandbox(&sandbox_id).advisory_key();
1881618825
runtime
1881718826
.store
1881818827
.put_message(&sandbox_record(
18819-
"sb-ws-writer",
18828+
&sandbox_id,
1882018829
"ws-writer",
1882118830
SandboxPhase::Provisioning,
1882218831
))
1882318832
.await
1882418833
.unwrap();
18834+
let now = openshell_core::time::now_ms();
18835+
runtime.reconcile_provisioning_deadlines(now).await.unwrap();
18836+
1882518837
// Reconcile re-derives configuration from provider and profile
18826-
// records, so it waits for their writers, which hold X(workspace).
18838+
// records, so it must not run under their writers, which hold
18839+
// X(workspace).
1882718840
let writer = runtime
1882818841
.mutation_guard(MutationScope::Workspace("default"))
1882918842
.await
1883018843
.unwrap();
18831-
let mut reconcile = {
18844+
let reconcile = {
1883218845
let runtime = runtime.clone();
18833-
let now = openshell_core::time::now_ms();
18834-
tokio::spawn(async move { runtime.reconcile_provisioning_deadlines(now).await })
18846+
tokio::spawn(async move {
18847+
runtime
18848+
.reconcile_provisioning_deadlines(now + 300_000)
18849+
.await
18850+
})
1883518851
};
18836-
assert!(
18837-
tokio::time::timeout(Duration::from_millis(100), &mut reconcile)
18838-
.await
18839-
.is_err(),
18840-
"provisioning reconcile must wait for the workspace writer"
18852+
tokio::time::timeout(Duration::from_secs(5), async {
18853+
while runtime.mutation_locks.key_references(sandbox_key) == 0
18854+
|| runtime.mutation_locks.key_references(workspace_key) < 2
18855+
{
18856+
tokio::task::yield_now().await;
18857+
}
18858+
})
18859+
.await
18860+
.expect("the scan holds the sandbox key and waits for the workspace key");
18861+
18862+
// Start, stop, and delete take the sandbox key; they must not wait
18863+
// for the writer through the scan.
18864+
let lifecycle = {
18865+
let runtime = runtime.clone();
18866+
let sandbox_id = sandbox_id.clone();
18867+
tokio::spawn(async move { drop(runtime.lock_sandbox_local(&sandbox_id).await) })
18868+
};
18869+
tokio::time::timeout(Duration::from_secs(5), lifecycle)
18870+
.await
18871+
.expect("a lifecycle lock proceeds while the writer holds the workspace")
18872+
.unwrap();
18873+
tokio::time::timeout(Duration::from_secs(5), reconcile)
18874+
.await
18875+
.expect("the scan skips the busy sandbox instead of waiting for the writer")
18876+
.unwrap()
18877+
.unwrap();
18878+
assert_eq!(runtime.mutation_locks.key_references(sandbox_key), 0);
18879+
let skipped = runtime
18880+
.store
18881+
.get_message::<Sandbox>(&sandbox_id)
18882+
.await
18883+
.unwrap()
18884+
.unwrap();
18885+
assert_eq!(skipped.phase(), i32::from(SandboxPhase::Provisioning));
18886+
18887+
drop(writer);
18888+
runtime
18889+
.reconcile_provisioning_deadlines(now + 300_000)
18890+
.await
18891+
.unwrap();
18892+
let expired = runtime
18893+
.store
18894+
.get_message::<Sandbox>(&sandbox_id)
18895+
.await
18896+
.unwrap()
18897+
.unwrap();
18898+
assert_eq!(expired.phase(), i32::from(SandboxPhase::Error));
18899+
}
18900+
18901+
#[tokio::test]
18902+
async fn provisioning_reconcile_continues_past_a_skipped_sandbox() {
18903+
let runtime = test_runtime(ControlledDriver::new()).await;
18904+
// The scan orders by insert time, then name: the busy sandbox is stored
18905+
// first and named first, so a skip that ended the pass would leave the
18906+
// other sandbox alone.
18907+
let busy = sandbox_record("sb-skip-busy", "a-busy", SandboxPhase::Provisioning);
18908+
let mut other = sandbox_record("sb-skip-other", "b-other", SandboxPhase::Provisioning);
18909+
other.metadata.as_mut().unwrap().workspace = "alpha".to_string();
18910+
runtime.store.put_message(&busy).await.unwrap();
18911+
runtime.store.put_message(&other).await.unwrap();
18912+
let now = openshell_core::time::now_ms();
18913+
runtime.reconcile_provisioning_deadlines(now).await.unwrap();
18914+
18915+
let writer = runtime
18916+
.mutation_guard(MutationScope::Workspace("default"))
18917+
.await
18918+
.unwrap();
18919+
// Spawned: the lock-order check forbids nesting under the writer in this task.
18920+
let reconcile = {
18921+
let runtime = runtime.clone();
18922+
tokio::spawn(async move {
18923+
runtime
18924+
.reconcile_provisioning_deadlines(now + 300_000)
18925+
.await
18926+
})
18927+
};
18928+
tokio::time::timeout(Duration::from_secs(5), reconcile)
18929+
.await
18930+
.expect("the scan skips the busy sandbox instead of waiting for the writer")
18931+
.unwrap()
18932+
.unwrap();
18933+
let skipped = runtime
18934+
.store
18935+
.get_message::<Sandbox>("sb-skip-busy")
18936+
.await
18937+
.unwrap()
18938+
.unwrap();
18939+
assert_eq!(skipped.phase(), i32::from(SandboxPhase::Provisioning));
18940+
let expired = runtime
18941+
.store
18942+
.get_message::<Sandbox>("sb-skip-other")
18943+
.await
18944+
.unwrap()
18945+
.unwrap();
18946+
assert_eq!(
18947+
expired.phase(),
18948+
i32::from(SandboxPhase::Error),
18949+
"a skipped sandbox must not end the pass"
1884118950
);
1884218951
drop(writer);
18952+
}
18953+
18954+
#[tokio::test]
18955+
async fn provisioning_reconcile_releases_the_workspace_behind_a_busy_sandbox() {
18956+
use crate::persistence::MutationLockKey;
18957+
let runtime = test_runtime(ControlledDriver::new()).await;
18958+
// With the workspace key first, the scan holds S(workspace) while it
18959+
// waits for X(sandbox), and a provider writer queues behind the scan.
18960+
let workspace_key = MutationLockKey::Workspace("default").advisory_key();
18961+
let sandbox_id = (0..1000)
18962+
.map(|i| format!("sb-busy-{i}"))
18963+
.find(|id| MutationLockKey::Sandbox(id).advisory_key() > workspace_key)
18964+
.expect("a sandbox key that sorts after the workspace key");
18965+
let sandbox_key = MutationLockKey::Sandbox(&sandbox_id).advisory_key();
18966+
runtime
18967+
.store
18968+
.put_message(&sandbox_record(
18969+
&sandbox_id,
18970+
"busy",
18971+
SandboxPhase::Provisioning,
18972+
))
18973+
.await
18974+
.unwrap();
18975+
let now = openshell_core::time::now_ms();
18976+
runtime.reconcile_provisioning_deadlines(now).await.unwrap();
18977+
18978+
let lifecycle = runtime.lock_sandbox_local(&sandbox_id).await;
18979+
let reconcile = {
18980+
let runtime = runtime.clone();
18981+
tokio::spawn(async move {
18982+
runtime
18983+
.reconcile_provisioning_deadlines(now + 300_000)
18984+
.await
18985+
})
18986+
};
18987+
tokio::time::timeout(Duration::from_secs(5), async {
18988+
while runtime.mutation_locks.key_references(workspace_key) == 0
18989+
|| runtime.mutation_locks.key_references(sandbox_key) < 2
18990+
{
18991+
tokio::task::yield_now().await;
18992+
}
18993+
})
18994+
.await
18995+
.expect("the scan holds the workspace key and waits for the sandbox key");
18996+
18997+
let writer = {
18998+
let runtime = runtime.clone();
18999+
tokio::spawn(async move {
19000+
drop(
19001+
runtime
19002+
.mutation_guard(MutationScope::Workspace("default"))
19003+
.await
19004+
.unwrap(),
19005+
);
19006+
})
19007+
};
19008+
tokio::time::timeout(Duration::from_secs(5), writer)
19009+
.await
19010+
.expect("a provider writer proceeds while the sandbox is busy")
19011+
.unwrap();
1884319012
tokio::time::timeout(Duration::from_secs(5), reconcile)
1884419013
.await
18845-
.expect("provisioning reconcile proceeds once the writer releases")
19014+
.expect("the scan skips the busy sandbox instead of waiting for it")
19015+
.unwrap()
19016+
.unwrap();
19017+
drop(lifecycle);
19018+
let skipped = runtime
19019+
.store
19020+
.get_message::<Sandbox>(&sandbox_id)
19021+
.await
1884619022
.unwrap()
1884719023
.unwrap();
19024+
assert_eq!(skipped.phase(), i32::from(SandboxPhase::Provisioning));
1884819025
}
1884919026

1885019027
#[tokio::test]

‎crates/openshell-server/src/compute/mutation_guard.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,11 @@ impl ComputeRuntime {
452452
/// Local S(global) S(workspace) X(sandbox), for provisioning-deadline
453453
/// reconciliation, which re-derives configuration from provider and
454454
/// profile records and must not interleave with their local writers.
455+
///
456+
/// The wait is unbounded. When the sandbox key sorts first, the caller
457+
/// holds X(sandbox) while it waits behind a workspace writer, so a
458+
/// background caller should bound the wait with a timeout; dropping the
459+
/// future releases the keys already taken.
455460
pub(super) async fn lock_sandbox_local_in_workspace(
456461
&self,
457462
workspace: &str,

‎crates/openshell-server/src/compute/provisioning_deadline.rs‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ use serde::{Deserialize, Serialize};
1414

1515
const REPAIR_WINDOW_MS: i64 = 300_000;
1616

17+
/// How long a scan pass waits for one sandbox's local keys before it skips
18+
/// that sandbox until the next pass. Keys are taken in ascending order, so the
19+
/// scan can hold X(sandbox) while it waits for S(workspace) behind a provider
20+
/// or profile writer, and the sandbox's lifecycle operations wait behind the
21+
/// scan. Well under the mutation lock timeout.
22+
const SCAN_LOCK_WAIT: std::time::Duration = std::time::Duration::from_millis(500);
23+
1724
/// Opaque identity of a committed effective configuration change. A -> B -> A
1825
/// must use three different IDs even if the first and last content hashes match.
1926
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -469,9 +476,35 @@ impl super::ComputeRuntime {
469476
// Expiration can fence Starting while its driver RPC owns the
470477
// lifecycle gate. Cleanup waits for that gate; Error never waits
471478
// for compute I/O, matching the existing driver-observation fence.
472-
let local_guard = self
473-
.lock_sandbox_local_in_workspace(candidate.object_workspace(), &record.id)
474-
.await;
479+
// A timed-out acquisition releases the keys it already took.
480+
let Ok(local_guard) = tokio::time::timeout(
481+
SCAN_LOCK_WAIT,
482+
self.lock_sandbox_local_in_workspace(candidate.object_workspace(), &record.id),
483+
)
484+
.await
485+
else {
486+
// Brief contention is routine; a skip past the deadline
487+
// delays expiry, so operators should see it.
488+
let overdue = candidate
489+
.status
490+
.as_ref()
491+
.and_then(|status| status.provisioning.as_ref())
492+
.and_then(|record| record.deadline.as_ref())
493+
.and_then(|deadline| timestamp_to_millis(deadline).ok())
494+
.is_some_and(|deadline| deadline <= now_ms);
495+
if overdue {
496+
tracing::warn!(
497+
sandbox_id = %record.id,
498+
"Provisioning deadline scan skipped an overdue sandbox whose locks are busy"
499+
);
500+
} else {
501+
tracing::debug!(
502+
sandbox_id = %record.id,
503+
"Provisioning deadline scan skipped a sandbox whose locks are busy"
504+
);
505+
}
506+
continue;
507+
};
475508
let Some(mut current) = self
476509
.store
477510
.get_message::<Sandbox>(&record.id)

‎crates/openshell-server/src/persistence/mutation_lock.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
//! | providers and workspace-scoped profiles | S(global) X(workspace) |
1414
//! | one sandbox, admin or supervisor | S(global) S(workspace) X(sandbox) |
1515
//! | lifecycle, driver watch, reconcile (process-local only) | S(global) X(sandbox) |
16-
//! | provisioning-deadline reconcile (process-local only) | S(global) S(workspace) X(sandbox) |
16+
//! | provisioning-deadline reconcile (process-local only, skips a busy sandbox) | S(global) S(workspace) X(sandbox) |
1717
//!
1818
//! Ordering rules, which make the scheme deadlock-free:
1919
//!

0 commit comments

Comments
 (0)