Skip to content

Commit d397a4a

Browse files
fix(kubernetes): keep a late stop snapshot from reviving a stopped sandbox
A stop keeps the sandbox-runtime bootstrap annotations, with operation "stop", until its completion patch. The watch converter marked every object with those annotations as a starting generation: it added Bootstrapping=True and dropped Suspended. The gateway treats that condition as an accepted start that may replace a settled Stopped phase, so a snapshot taken mid-stop and delivered after the stop settled moved the sandbox from Stopped to Provisioning and the next stop was rejected. For operation "stop", withdraw readiness and withhold the suspension until completion as before, but do not report Bootstrapping. Create and restart keep their existing marking, so a real start still recovers a stale Stopped view. Closes #5195 Signed-off-by: Matthew Grossman <[email protected]>
1 parent cc0642f commit d397a4a

2 files changed

Lines changed: 152 additions & 5 deletions

File tree

  • crates

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 127 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6923,17 +6923,13 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
69236923
namespace: &str,
69246924
obj: DynamicObject,
69256925
) -> Result<(String, Sandbox), String> {
6926-
let bootstrapping = sandbox_runtime_bootstrap_in_progress(&obj);
69276926
let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default();
69286927
let object_namespace = obj
69296928
.metadata
69306929
.namespace
69316930
.clone()
69326931
.unwrap_or_else(|| namespace.to_string());
6933-
let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?;
6934-
if bootstrapping {
6935-
mark_sandbox_runtime_bootstrapping(&mut sandbox);
6936-
}
6932+
let (name, mut sandbox) = sandbox_from_object_with_bootstrap_state(namespace, &obj)?;
69376933
if !sandbox_id.is_empty() {
69386934
let dependencies = Box::pin(generation_control_availability(
69396935
client,
@@ -6972,6 +6968,34 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
69726968
Ok((name, sandbox))
69736969
}
69746970

6971+
fn sandbox_from_object_with_bootstrap_state(
6972+
namespace: &str,
6973+
obj: &DynamicObject,
6974+
) -> Result<(String, Sandbox), String> {
6975+
let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?;
6976+
if sandbox_runtime_bootstrap_in_progress(obj) {
6977+
if sandbox_runtime_bootstrap_operation(obj) == Some("stop") {
6978+
mark_sandbox_runtime_stopping(&mut sandbox);
6979+
} else {
6980+
mark_sandbox_runtime_bootstrapping(&mut sandbox);
6981+
}
6982+
}
6983+
Ok((name, sandbox))
6984+
}
6985+
6986+
/// A stop holds the bootstrap annotations until its completion patch, but it
6987+
/// is not a start: the gateway reads `Bootstrapping=True` as an accepted
6988+
/// start that may replace a settled Stopped phase. Withhold the suspension
6989+
/// until the stop completes, as a start does.
6990+
fn mark_sandbox_runtime_stopping(sandbox: &mut Sandbox) {
6991+
if let Some(status) = sandbox.status.as_mut() {
6992+
status.conditions.retain(|condition| {
6993+
condition.r#type != SANDBOX_SUSPENDED_CONDITION && condition.r#type != "Bootstrapping"
6994+
});
6995+
}
6996+
mark_sandbox_runtime_control_unavailable(sandbox);
6997+
}
6998+
69756999
fn mark_sandbox_runtime_bootstrapping(sandbox: &mut Sandbox) {
69767000
if let Some(status) = sandbox.status.as_mut() {
69777001
status.conditions.retain(|condition| {
@@ -14451,6 +14475,104 @@ mod tests {
1445114475
}));
1445214476
}
1445314477

14478+
fn sandbox_cr_mid_operation(operation: Option<&str>) -> DynamicObject {
14479+
let mut annotations = BTreeMap::from([
14480+
(
14481+
ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(),
14482+
"true".to_string(),
14483+
),
14484+
(
14485+
ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE.to_string(),
14486+
SandboxRuntimeBootstrapPhase::Suspending
14487+
.as_str()
14488+
.to_string(),
14489+
),
14490+
]);
14491+
if let Some(operation) = operation {
14492+
annotations.insert(
14493+
ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION.to_string(),
14494+
operation.to_string(),
14495+
);
14496+
}
14497+
DynamicObject {
14498+
types: None,
14499+
metadata: ObjectMeta {
14500+
name: Some("alpha--work".to_string()),
14501+
namespace: Some("default".to_string()),
14502+
annotations: Some(annotations),
14503+
labels: Some(BTreeMap::from([
14504+
(LABEL_SANDBOX_ID.to_string(), "uuid-123".to_string()),
14505+
(LABEL_SANDBOX_NAME.to_string(), "work".to_string()),
14506+
(LABEL_SANDBOX_WORKSPACE.to_string(), "alpha".to_string()),
14507+
(
14508+
LABEL_MANAGED_BY.to_string(),
14509+
LABEL_MANAGED_BY_VALUE.to_string(),
14510+
),
14511+
])),
14512+
..Default::default()
14513+
},
14514+
data: serde_json::json!({
14515+
"status": {
14516+
"conditions": [
14517+
{"type": "Ready", "status": "False", "reason": "SandboxSuspended"},
14518+
{"type": "Suspended", "status": "True", "reason": "PodTerminated"},
14519+
]
14520+
}
14521+
}),
14522+
}
14523+
}
14524+
14525+
fn has_condition(sandbox: &Sandbox, r#type: &str, status: &str) -> bool {
14526+
sandbox.status.as_ref().is_some_and(|sandbox_status| {
14527+
sandbox_status
14528+
.conditions
14529+
.iter()
14530+
.any(|condition| condition.r#type == r#type && condition.status == status)
14531+
})
14532+
}
14533+
14534+
// The gateway treats Bootstrapping=True as an accepted start and lets it
14535+
// replace a settled Stopped phase. A snapshot taken while a stop still
14536+
// holds its bootstrap annotations can arrive after that stop settled
14537+
// (#5195), so it must not claim a starting generation.
14538+
#[test]
14539+
fn stop_snapshot_does_not_report_a_starting_generation() {
14540+
let (_, sandbox) = sandbox_from_object_with_bootstrap_state(
14541+
"default",
14542+
&sandbox_cr_mid_operation(Some("stop")),
14543+
)
14544+
.unwrap();
14545+
14546+
assert!(!has_condition(&sandbox, "Bootstrapping", "True"));
14547+
// Suspension is still published only once the stop completes.
14548+
assert!(!has_condition(
14549+
&sandbox,
14550+
SANDBOX_SUSPENDED_CONDITION,
14551+
"True"
14552+
));
14553+
assert!(!has_condition(&sandbox, "Ready", "True"));
14554+
}
14555+
14556+
#[test]
14557+
fn start_snapshots_report_a_starting_generation() {
14558+
for operation in [Some("create"), Some("restart"), None] {
14559+
let (_, sandbox) = sandbox_from_object_with_bootstrap_state(
14560+
"default",
14561+
&sandbox_cr_mid_operation(operation),
14562+
)
14563+
.unwrap();
14564+
14565+
assert!(
14566+
has_condition(&sandbox, "Bootstrapping", "True"),
14567+
"{operation:?}"
14568+
);
14569+
assert!(
14570+
!has_condition(&sandbox, SANDBOX_SUSPENDED_CONDITION, "True"),
14571+
"{operation:?}"
14572+
);
14573+
}
14574+
}
14575+
1445414576
#[test]
1445514577
fn completed_sandbox_runtime_bootstrap_preserves_real_suspension() {
1445614578
let sandbox = Sandbox {

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14327,6 +14327,31 @@ mod tests {
1432714327
assert_eq!(current.phase(), SandboxPhase::Provisioning as i32);
1432814328
}
1432914329

14330+
// A Kubernetes snapshot taken while a stop still holds its bootstrap
14331+
// annotations can arrive after the stop settled (#5195). It withdraws
14332+
// readiness without claiming a starting generation.
14333+
#[tokio::test]
14334+
async fn late_stop_snapshot_keeps_stopped_sandbox_stopped() {
14335+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
14336+
let sandbox = sandbox_record("sb-stopped", "sandbox-stopped", SandboxPhase::Stopped);
14337+
runtime.store.put_message(&sandbox).await.unwrap();
14338+
let mut stopping = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14339+
stopping.status = Some(make_driver_status(make_driver_condition(
14340+
"DependenciesNotReady",
14341+
"sandbox-runtime enforcement dependencies are not ready",
14342+
)));
14343+
14344+
runtime.apply_sandbox_update(stopping).await.unwrap();
14345+
14346+
let current = runtime
14347+
.store
14348+
.get_message::<Sandbox>(sandbox.object_id())
14349+
.await
14350+
.unwrap()
14351+
.unwrap();
14352+
assert_eq!(current.phase(), SandboxPhase::Stopped as i32);
14353+
}
14354+
1433014355
#[tokio::test]
1433114356
async fn suspended_bootstrap_snapshot_does_not_revive_stopped_sandbox() {
1433214357
let runtime = test_runtime(Arc::new(TestDriver::default())).await;

0 commit comments

Comments
 (0)