Skip to content

Commit f4a6137

Browse files
fix(kubernetes): keep a late stop snapshot from reviving a stopped sandbox (NVIDIA#5196)
* fix(kubernetes): keep a late stop snapshot from reviving a stopped sandbox A stop keeps the sandbox-runtime bootstrap annotations, with operation "stop", until its completion patch. The watch converter marked every object with those annotations as a starting generation: it added Bootstrapping=True and dropped Suspended. The gateway treats that condition as an accepted start that may replace a settled Stopped phase, so a snapshot taken mid-stop and delivered after the stop settled moved the sandbox from Stopped to Provisioning and the next stop was rejected. For operation "stop", withdraw readiness and withhold the suspension until completion as before, but do not report Bootstrapping. Create and restart keep their existing marking, so a real start still recovers a stale Stopped view. Closes NVIDIA#5195 Signed-off-by: Matthew Grossman <[email protected]> * test(kubernetes): cover the full stop sequence for stop snapshots Replace the late-stop gateway test with one that walks a stop from Stopping through completion and then replays the in-flight snapshot, with and without a connected supervisor session. An in-flight stop snapshot, which now withdraws readiness without Bootstrapping or Suspended, keeps the sandbox Stopping rather than Provisioning or Error; completion settles it Stopped; and the late replay leaves it Stopped. The test documents that it guards the gateway contract and that the driver test covers the regression itself. Share the condition reset between the stopping and bootstrapping markers in the Kubernetes driver. Refs NVIDIA#5195 Signed-off-by: Matthew Grossman <[email protected]> --------- Signed-off-by: Matthew Grossman <[email protected]>
1 parent fab1404 commit f4a6137

2 files changed

Lines changed: 192 additions & 8 deletions

File tree

  • crates

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 132 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6923,17 +6923,13 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
69236923
namespace: &str,
69246924
obj: DynamicObject,
69256925
) -> Result<(String, Sandbox), String> {
6926-
let bootstrapping = sandbox_runtime_bootstrap_in_progress(&obj);
69276926
let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default();
69286927
let object_namespace = obj
69296928
.metadata
69306929
.namespace
69316930
.clone()
69326931
.unwrap_or_else(|| namespace.to_string());
6933-
let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?;
6934-
if bootstrapping {
6935-
mark_sandbox_runtime_bootstrapping(&mut sandbox);
6936-
}
6932+
let (name, mut sandbox) = sandbox_from_object_with_bootstrap_state(namespace, &obj)?;
69376933
if !sandbox_id.is_empty() {
69386934
let dependencies = Box::pin(generation_control_availability(
69396935
client,
@@ -6972,11 +6968,33 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
69726968
Ok((name, sandbox))
69736969
}
69746970

6971+
fn sandbox_from_object_with_bootstrap_state(
6972+
namespace: &str,
6973+
obj: &DynamicObject,
6974+
) -> Result<(String, Sandbox), String> {
6975+
let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?;
6976+
if sandbox_runtime_bootstrap_in_progress(obj) {
6977+
if sandbox_runtime_bootstrap_operation(obj) == Some("stop") {
6978+
mark_sandbox_runtime_stopping(&mut sandbox);
6979+
} else {
6980+
mark_sandbox_runtime_bootstrapping(&mut sandbox);
6981+
}
6982+
}
6983+
Ok((name, sandbox))
6984+
}
6985+
6986+
/// A stop holds the bootstrap annotations until its completion patch, but it
6987+
/// is not a start: the gateway reads `Bootstrapping=True` as an accepted
6988+
/// start that may replace a settled Stopped phase. Withhold the suspension
6989+
/// until the stop completes, as a start does.
6990+
fn mark_sandbox_runtime_stopping(sandbox: &mut Sandbox) {
6991+
clear_sandbox_runtime_operation_conditions(sandbox);
6992+
mark_sandbox_runtime_control_unavailable(sandbox);
6993+
}
6994+
69756995
fn mark_sandbox_runtime_bootstrapping(sandbox: &mut Sandbox) {
6996+
clear_sandbox_runtime_operation_conditions(sandbox);
69766997
if let Some(status) = sandbox.status.as_mut() {
6977-
status.conditions.retain(|condition| {
6978-
condition.r#type != SANDBOX_SUSPENDED_CONDITION && condition.r#type != "Bootstrapping"
6979-
});
69806998
status.conditions.push(SandboxCondition {
69816999
r#type: "Bootstrapping".to_string(),
69827000
status: "True".to_string(),
@@ -6988,6 +7006,14 @@ fn mark_sandbox_runtime_bootstrapping(sandbox: &mut Sandbox) {
69887006
mark_sandbox_runtime_control_unavailable(sandbox);
69897007
}
69907008

7009+
fn clear_sandbox_runtime_operation_conditions(sandbox: &mut Sandbox) {
7010+
if let Some(status) = sandbox.status.as_mut() {
7011+
status.conditions.retain(|condition| {
7012+
condition.r#type != SANDBOX_SUSPENDED_CONDITION && condition.r#type != "Bootstrapping"
7013+
});
7014+
}
7015+
}
7016+
69917017
fn mark_sandbox_runtime_control_unavailable(sandbox: &mut Sandbox) {
69927018
const REASON: &str = "DependenciesNotReady";
69937019
const MESSAGE: &str = "sandbox-runtime enforcement dependencies are not ready";
@@ -14451,6 +14477,104 @@ mod tests {
1445114477
}));
1445214478
}
1445314479

14480+
fn sandbox_cr_mid_operation(operation: Option<&str>) -> DynamicObject {
14481+
let mut annotations = BTreeMap::from([
14482+
(
14483+
ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(),
14484+
"true".to_string(),
14485+
),
14486+
(
14487+
ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE.to_string(),
14488+
SandboxRuntimeBootstrapPhase::Suspending
14489+
.as_str()
14490+
.to_string(),
14491+
),
14492+
]);
14493+
if let Some(operation) = operation {
14494+
annotations.insert(
14495+
ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION.to_string(),
14496+
operation.to_string(),
14497+
);
14498+
}
14499+
DynamicObject {
14500+
types: None,
14501+
metadata: ObjectMeta {
14502+
name: Some("alpha--work".to_string()),
14503+
namespace: Some("default".to_string()),
14504+
annotations: Some(annotations),
14505+
labels: Some(BTreeMap::from([
14506+
(LABEL_SANDBOX_ID.to_string(), "uuid-123".to_string()),
14507+
(LABEL_SANDBOX_NAME.to_string(), "work".to_string()),
14508+
(LABEL_SANDBOX_WORKSPACE.to_string(), "alpha".to_string()),
14509+
(
14510+
LABEL_MANAGED_BY.to_string(),
14511+
LABEL_MANAGED_BY_VALUE.to_string(),
14512+
),
14513+
])),
14514+
..Default::default()
14515+
},
14516+
data: serde_json::json!({
14517+
"status": {
14518+
"conditions": [
14519+
{"type": "Ready", "status": "False", "reason": "SandboxSuspended"},
14520+
{"type": "Suspended", "status": "True", "reason": "PodTerminated"},
14521+
]
14522+
}
14523+
}),
14524+
}
14525+
}
14526+
14527+
fn has_condition(sandbox: &Sandbox, r#type: &str, status: &str) -> bool {
14528+
sandbox.status.as_ref().is_some_and(|sandbox_status| {
14529+
sandbox_status
14530+
.conditions
14531+
.iter()
14532+
.any(|condition| condition.r#type == r#type && condition.status == status)
14533+
})
14534+
}
14535+
14536+
// The gateway treats Bootstrapping=True as an accepted start and lets it
14537+
// replace a settled Stopped phase. A snapshot taken while a stop still
14538+
// holds its bootstrap annotations can arrive after that stop settled
14539+
// (#5195), so it must not claim a starting generation.
14540+
#[test]
14541+
fn stop_snapshot_does_not_report_a_starting_generation() {
14542+
let (_, sandbox) = sandbox_from_object_with_bootstrap_state(
14543+
"default",
14544+
&sandbox_cr_mid_operation(Some("stop")),
14545+
)
14546+
.unwrap();
14547+
14548+
assert!(!has_condition(&sandbox, "Bootstrapping", "True"));
14549+
// Suspension is still published only once the stop completes.
14550+
assert!(!has_condition(
14551+
&sandbox,
14552+
SANDBOX_SUSPENDED_CONDITION,
14553+
"True"
14554+
));
14555+
assert!(!has_condition(&sandbox, "Ready", "True"));
14556+
}
14557+
14558+
#[test]
14559+
fn start_snapshots_report_a_starting_generation() {
14560+
for operation in [Some("create"), Some("restart"), None] {
14561+
let (_, sandbox) = sandbox_from_object_with_bootstrap_state(
14562+
"default",
14563+
&sandbox_cr_mid_operation(operation),
14564+
)
14565+
.unwrap();
14566+
14567+
assert!(
14568+
has_condition(&sandbox, "Bootstrapping", "True"),
14569+
"{operation:?}"
14570+
);
14571+
assert!(
14572+
!has_condition(&sandbox, SANDBOX_SUSPENDED_CONDITION, "True"),
14573+
"{operation:?}"
14574+
);
14575+
}
14576+
}
14577+
1445414578
#[test]
1445514579
fn completed_sandbox_runtime_bootstrap_preserves_real_suspension() {
1445614580
let sandbox = Sandbox {

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14327,6 +14327,66 @@ mod tests {
1432714327
assert_eq!(current.phase(), SandboxPhase::Provisioning as i32);
1432814328
}
1432914329

14330+
// Gateway side of the Kubernetes stop contract. While a stop holds its
14331+
// bootstrap annotations, the driver withdraws readiness and reports
14332+
// neither Bootstrapping nor Suspended. Such a snapshot keeps the stop in
14333+
// Stopping, never Provisioning or Error, and cannot revive the sandbox
14334+
// after the stop settles. This passes without the #5195 fix; the driver
14335+
// test stop_snapshot_does_not_report_a_starting_generation covers the
14336+
// regression.
14337+
#[tokio::test]
14338+
async fn kubernetes_stop_snapshots_settle_without_reviving_the_sandbox() {
14339+
for session_connected in [false, true] {
14340+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
14341+
let sandbox = sandbox_record("sb-stopping", "sandbox-stopping", SandboxPhase::Stopping);
14342+
runtime.store.put_message(&sandbox).await.unwrap();
14343+
if session_connected {
14344+
register_test_supervisor_session(&runtime, sandbox.object_id());
14345+
}
14346+
14347+
let mut in_flight = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14348+
in_flight.status = Some(make_driver_status(make_driver_condition(
14349+
"DependenciesNotReady",
14350+
"sandbox-runtime enforcement dependencies are not ready",
14351+
)));
14352+
let mut settled_status = make_driver_status(make_driver_condition(
14353+
"SandboxSuspended",
14354+
"Sandbox is suspended",
14355+
));
14356+
settled_status.conditions.push(DriverCondition {
14357+
r#type: "Suspended".to_string(),
14358+
status: "True".to_string(),
14359+
reason: "PodTerminated".to_string(),
14360+
message: "Pod terminated".to_string(),
14361+
transition_time: None,
14362+
});
14363+
let mut settled = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
14364+
settled.status = Some(settled_status);
14365+
14366+
for (step, snapshot, expected) in [
14367+
("in-flight stop", &in_flight, SandboxPhase::Stopping),
14368+
("completed stop", &settled, SandboxPhase::Stopped),
14369+
("late in-flight stop", &in_flight, SandboxPhase::Stopped),
14370+
] {
14371+
runtime
14372+
.apply_sandbox_update(snapshot.clone())
14373+
.await
14374+
.unwrap();
14375+
let current = runtime
14376+
.store
14377+
.get_message::<Sandbox>(sandbox.object_id())
14378+
.await
14379+
.unwrap()
14380+
.unwrap();
14381+
assert_eq!(
14382+
current.phase(),
14383+
expected as i32,
14384+
"{step} (session_connected={session_connected})"
14385+
);
14386+
}
14387+
}
14388+
}
14389+
1433014390
#[tokio::test]
1433114391
async fn suspended_bootstrap_snapshot_does_not_revive_stopped_sandbox() {
1433214392
let runtime = test_runtime(Arc::new(TestDriver::default())).await;

0 commit comments

Comments
 (0)