@@ -5581,7 +5581,9 @@ impl ComputeRuntime {
55815581 let ready_container_exit = existing_phase == SandboxPhase::Ready
55825582 && (driver_snapshot_confirms_stopped(&incoming)
55835583 || driver_snapshot_reports_runtime_restart(&incoming));
5584- if existing_phase != SandboxPhase::Starting && !ready_container_exit {
5584+ let stopped_bootstrap =
5585+ existing_phase == SandboxPhase::Stopped && driver_snapshot_confirms_starting(&incoming);
5586+ if existing_phase != SandboxPhase::Starting && !ready_container_exit && !stopped_bootstrap {
55855587 return self.apply_sandbox_update_locked(incoming, existing).await;
55865588 }
55875589
@@ -5591,6 +5593,8 @@ impl ComputeRuntime {
55915593 // stop the new generation before the replacement supervisor connects.
55925594 // The replacement may already be Ready when the old exit arrives, so
55935595 // terminal container snapshots need the same check after readiness.
5596+ // A bootstrap event queued before Stop reached Stopped must also be
5597+ // revalidated so it cannot undo the completed stop.
55945598 // Release this sandbox's local mutation lock, wait for that lifecycle
55955599 // operation, and then reread both the driver and store before applying
55965600 // an authoritative observation. Taking the per-sandbox gate only for
@@ -5620,7 +5624,10 @@ impl ComputeRuntime {
56205624 match observed {
56215625 Ok(Some(live)) if live.id == incoming.id && live.status.is_some() => incoming = live,
56225626 Ok(Some(_) | None) | Err(_)
5623- if matches!(current_phase, SandboxPhase::Starting | SandboxPhase::Ready) =>
5627+ if matches!(
5628+ current_phase,
5629+ SandboxPhase::Starting | SandboxPhase::Ready | SandboxPhase::Stopped
5630+ ) =>
56245631 {
56255632 warn!(
56265633 sandbox_id = %incoming.id,
@@ -14297,9 +14304,109 @@ mod tests {
1429714304 assert_eq!(current.phase(), SandboxPhase::Stopped as i32);
1429814305 }
1429914306
14307+ #[tokio::test]
14308+ async fn stale_bootstrap_snapshot_waits_for_stop_and_revalidates() {
14309+ let sandbox = sandbox_record("sb-stop-race", "sandbox-stop-race", SandboxPhase::Ready);
14310+ let mut stopped = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14311+ stopped.status = Some(make_driver_status(make_driver_condition(
14312+ "ContainerExited",
14313+ "container stopped by request",
14314+ )));
14315+ let mut bootstrapping = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14316+ let mut status = make_driver_status(make_driver_condition(
14317+ "DependenciesNotReady",
14318+ "old bootstrap snapshot queued before stop",
14319+ ));
14320+ status.conditions.push(DriverCondition {
14321+ r#type: "Bootstrapping".to_string(),
14322+ status: "True".to_string(),
14323+ reason: "GenerationStarting".to_string(),
14324+ ..Default::default()
14325+ });
14326+ bootstrapping.status = Some(status);
14327+
14328+ for (observation, outcome) in [
14329+ (
14330+ "stopped",
14331+ ControlledGetOutcome::Sandbox(Box::new(stopped.clone())),
14332+ ),
14333+ ("missing", ControlledGetOutcome::Missing),
14334+ (
14335+ "unavailable",
14336+ ControlledGetOutcome::Error("driver unavailable"),
14337+ ),
14338+ (
14339+ "wrong sandbox",
14340+ ControlledGetOutcome::Sandbox(Box::new(DriverSandbox {
14341+ id: "other-sandbox".to_string(),
14342+ ..stopped.clone()
14343+ })),
14344+ ),
14345+ (
14346+ "missing status",
14347+ ControlledGetOutcome::Sandbox(Box::new(DriverSandbox {
14348+ status: None,
14349+ ..stopped.clone()
14350+ })),
14351+ ),
14352+ ] {
14353+ let driver = ControlledDriver::new();
14354+ driver.block_stop();
14355+ driver.set_get_outcome(outcome);
14356+ let runtime = test_runtime(driver.clone()).await;
14357+ runtime.store.put_message(&sandbox).await.unwrap();
14358+
14359+ let stop_runtime = runtime.clone();
14360+ let sandbox_name = sandbox.object_name().to_string();
14361+ let stop =
14362+ tokio::spawn(
14363+ async move { stop_runtime.stop_sandbox("default", &sandbox_name).await },
14364+ );
14365+ tokio::time::timeout(Duration::from_secs(1), driver.stop_started.notified())
14366+ .await
14367+ .expect("stop did not reach the driver");
14368+ runtime.apply_sandbox_update(stopped.clone()).await.unwrap();
14369+
14370+ let gate = runtime.lifecycle_gates.gate_for(sandbox.object_id());
14371+ let update_runtime = runtime.clone();
14372+ let stale_bootstrap = bootstrapping.clone();
14373+ let update =
14374+ tokio::spawn(
14375+ async move { update_runtime.apply_sandbox_update(stale_bootstrap).await },
14376+ );
14377+ tokio::time::timeout(Duration::from_secs(1), async {
14378+ while Arc::strong_count(&gate) < 3 {
14379+ assert!(
14380+ !update.is_finished(),
14381+ "{observation}: queued bootstrap must wait for the active stop"
14382+ );
14383+ tokio::task::yield_now().await;
14384+ }
14385+ })
14386+ .await
14387+ .expect("bootstrap update did not wait on the lifecycle gate");
14388+
14389+ driver.release_stop();
14390+ stop.await.unwrap().unwrap();
14391+ update.await.unwrap().unwrap();
14392+ let current = runtime
14393+ .store
14394+ .get_message::<Sandbox>(sandbox.object_id())
14395+ .await
14396+ .unwrap()
14397+ .unwrap();
14398+ assert_eq!(
14399+ current.phase(),
14400+ SandboxPhase::Stopped as i32,
14401+ "{observation}: stale bootstrap must not revive the stopped sandbox"
14402+ );
14403+ }
14404+ }
14405+
1430014406 #[tokio::test]
1430114407 async fn active_bootstrap_snapshot_recovers_stale_stopped_view() {
14302- let runtime = test_runtime(Arc::new(TestDriver::default())).await;
14408+ let driver = ControlledDriver::new();
14409+ let runtime = test_runtime(driver.clone()).await;
1430314410 let sandbox = sandbox_record("sb-restart", "sandbox-restart", SandboxPhase::Stopped);
1430414411 runtime.store.put_message(&sandbox).await.unwrap();
1430514412 let mut bootstrapping = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
@@ -14316,6 +14423,9 @@ mod tests {
1431614423 });
1431714424 bootstrapping.status = Some(status);
1431814425
14426+ driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(
14427+ bootstrapping.clone(),
14428+ )));
1431914429 runtime.apply_sandbox_update(bootstrapping).await.unwrap();
1432014430
1432114431 let current = runtime
0 commit comments