Skip to content

Commit 760396e

Browse files
committed
fix(server): keep completed stops stable across delayed driver watches
Wait for the active lifecycle operation and revalidate delayed bootstrap observations against current driver state. Preserve a completed stop when the read is unavailable or invalid, while retaining progress from an accepted start. Signed-off-by: Shiju <[email protected]>
1 parent cc0642f commit 760396e

1 file changed

Lines changed: 113 additions & 3 deletions

File tree

  • crates/openshell-server/src/compute

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 113 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5581,7 +5581,9 @@ impl ComputeRuntime {
55815581
let ready_container_exit = existing_phase == SandboxPhase::Ready
55825582
&& (driver_snapshot_confirms_stopped(&incoming)
55835583
|| driver_snapshot_reports_runtime_restart(&incoming));
5584-
if existing_phase != SandboxPhase::Starting && !ready_container_exit {
5584+
let stopped_bootstrap =
5585+
existing_phase == SandboxPhase::Stopped && driver_snapshot_confirms_starting(&incoming);
5586+
if existing_phase != SandboxPhase::Starting && !ready_container_exit && !stopped_bootstrap {
55855587
return self.apply_sandbox_update_locked(incoming, existing).await;
55865588
}
55875589

@@ -5591,6 +5593,8 @@ impl ComputeRuntime {
55915593
// stop the new generation before the replacement supervisor connects.
55925594
// The replacement may already be Ready when the old exit arrives, so
55935595
// terminal container snapshots need the same check after readiness.
5596+
// A bootstrap event queued before Stop reached Stopped must also be
5597+
// revalidated so it cannot undo the completed stop.
55945598
// Release this sandbox's local mutation lock, wait for that lifecycle
55955599
// operation, and then reread both the driver and store before applying
55965600
// an authoritative observation. Taking the per-sandbox gate only for
@@ -5620,7 +5624,10 @@ impl ComputeRuntime {
56205624
match observed {
56215625
Ok(Some(live)) if live.id == incoming.id && live.status.is_some() => incoming = live,
56225626
Ok(Some(_) | None) | Err(_)
5623-
if matches!(current_phase, SandboxPhase::Starting | SandboxPhase::Ready) =>
5627+
if matches!(
5628+
current_phase,
5629+
SandboxPhase::Starting | SandboxPhase::Ready | SandboxPhase::Stopped
5630+
) =>
56245631
{
56255632
warn!(
56265633
sandbox_id = %incoming.id,
@@ -14297,9 +14304,109 @@ mod tests {
1429714304
assert_eq!(current.phase(), SandboxPhase::Stopped as i32);
1429814305
}
1429914306

14307+
#[tokio::test]
14308+
async fn stale_bootstrap_snapshot_waits_for_stop_and_revalidates() {
14309+
let sandbox = sandbox_record("sb-stop-race", "sandbox-stop-race", SandboxPhase::Ready);
14310+
let mut stopped = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14311+
stopped.status = Some(make_driver_status(make_driver_condition(
14312+
"ContainerExited",
14313+
"container stopped by request",
14314+
)));
14315+
let mut bootstrapping = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14316+
let mut status = make_driver_status(make_driver_condition(
14317+
"DependenciesNotReady",
14318+
"old bootstrap snapshot queued before stop",
14319+
));
14320+
status.conditions.push(DriverCondition {
14321+
r#type: "Bootstrapping".to_string(),
14322+
status: "True".to_string(),
14323+
reason: "GenerationStarting".to_string(),
14324+
..Default::default()
14325+
});
14326+
bootstrapping.status = Some(status);
14327+
14328+
for (observation, outcome) in [
14329+
(
14330+
"stopped",
14331+
ControlledGetOutcome::Sandbox(Box::new(stopped.clone())),
14332+
),
14333+
("missing", ControlledGetOutcome::Missing),
14334+
(
14335+
"unavailable",
14336+
ControlledGetOutcome::Error("driver unavailable"),
14337+
),
14338+
(
14339+
"wrong sandbox",
14340+
ControlledGetOutcome::Sandbox(Box::new(DriverSandbox {
14341+
id: "other-sandbox".to_string(),
14342+
..stopped.clone()
14343+
})),
14344+
),
14345+
(
14346+
"missing status",
14347+
ControlledGetOutcome::Sandbox(Box::new(DriverSandbox {
14348+
status: None,
14349+
..stopped.clone()
14350+
})),
14351+
),
14352+
] {
14353+
let driver = ControlledDriver::new();
14354+
driver.block_stop();
14355+
driver.set_get_outcome(outcome);
14356+
let runtime = test_runtime(driver.clone()).await;
14357+
runtime.store.put_message(&sandbox).await.unwrap();
14358+
14359+
let stop_runtime = runtime.clone();
14360+
let sandbox_name = sandbox.object_name().to_string();
14361+
let stop =
14362+
tokio::spawn(
14363+
async move { stop_runtime.stop_sandbox("default", &sandbox_name).await },
14364+
);
14365+
tokio::time::timeout(Duration::from_secs(1), driver.stop_started.notified())
14366+
.await
14367+
.expect("stop did not reach the driver");
14368+
runtime.apply_sandbox_update(stopped.clone()).await.unwrap();
14369+
14370+
let gate = runtime.lifecycle_gates.gate_for(sandbox.object_id());
14371+
let update_runtime = runtime.clone();
14372+
let stale_bootstrap = bootstrapping.clone();
14373+
let update =
14374+
tokio::spawn(
14375+
async move { update_runtime.apply_sandbox_update(stale_bootstrap).await },
14376+
);
14377+
tokio::time::timeout(Duration::from_secs(1), async {
14378+
while Arc::strong_count(&gate) < 3 {
14379+
assert!(
14380+
!update.is_finished(),
14381+
"{observation}: queued bootstrap must wait for the active stop"
14382+
);
14383+
tokio::task::yield_now().await;
14384+
}
14385+
})
14386+
.await
14387+
.expect("bootstrap update did not wait on the lifecycle gate");
14388+
14389+
driver.release_stop();
14390+
stop.await.unwrap().unwrap();
14391+
update.await.unwrap().unwrap();
14392+
let current = runtime
14393+
.store
14394+
.get_message::<Sandbox>(sandbox.object_id())
14395+
.await
14396+
.unwrap()
14397+
.unwrap();
14398+
assert_eq!(
14399+
current.phase(),
14400+
SandboxPhase::Stopped as i32,
14401+
"{observation}: stale bootstrap must not revive the stopped sandbox"
14402+
);
14403+
}
14404+
}
14405+
1430014406
#[tokio::test]
1430114407
async fn active_bootstrap_snapshot_recovers_stale_stopped_view() {
14302-
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
14408+
let driver = ControlledDriver::new();
14409+
let runtime = test_runtime(driver.clone()).await;
1430314410
let sandbox = sandbox_record("sb-restart", "sandbox-restart", SandboxPhase::Stopped);
1430414411
runtime.store.put_message(&sandbox).await.unwrap();
1430514412
let mut bootstrapping = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
@@ -14316,6 +14423,9 @@ mod tests {
1431614423
});
1431714424
bootstrapping.status = Some(status);
1431814425

14426+
driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(
14427+
bootstrapping.clone(),
14428+
)));
1431914429
runtime.apply_sandbox_update(bootstrapping).await.unwrap();
1432014430

1432114431
let current = runtime

0 commit comments

Comments
 (0)