Skip to content

Commit 487022f

Browse files
committed
fix(gateway): let Ready win over stale Suspended in derive_phase
On Agent Sandbox v1beta1, a `sandbox stop` then `sandbox start` left the sandbox stuck at `Starting` and `start` timed out after 300s, even though the workload pod was `Running 1/1` and the CR reported `Ready=True`. The v1beta1 controller (v0.5.0) sets `Suspended=True (PodTerminated)` on stop and does not clear it on resume, so a resumed CR carries both `Ready=True (DependenciesReady)` and a stale `Suspended=True`. The gateway's `derive_phase` treated any `Suspended=True` condition as `Stopped` before it consulted `Ready`, so a resumed-but-Ready sandbox was read as not-running and the `Starting` phase never advanced to `Ready`. `Ready=True` means the sandbox is usable through this gateway and must win over a `Suspended=True` condition. Only resolve `Stopped` from `Suspended` when `Ready` is not `True`; a genuine stop leaves `Ready` unset or `False`, so it still resolves to `Stopped`. v1alpha1 (which encodes desired state as `spec.replicas` and emits no lingering `Suspended`) is unaffected. The underlying stale condition is an upstream bug in the agent-sandbox controller, fixed by kubernetes-sigs/agent-sandbox#1150 (in v0.5.6). This gateway change keeps OpenShell resilient regardless of controller version. Fixes #2932 Signed-off-by: Russell Bryant <[email protected]>
1 parent fb6610d commit 487022f

1 file changed

Lines changed: 67 additions & 3 deletions

File tree

  • crates/openshell-server/src/compute

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 67 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3980,10 +3980,24 @@ fn derive_phase(status: Option<&DriverSandboxStatus>) -> SandboxPhase {
39803980
return SandboxPhase::Deleting;
39813981
}
39823982

3983-
if status.conditions.iter().any(|condition| {
3984-
condition.r#type.eq_ignore_ascii_case("Suspended")
3983+
// `Ready=True` means the sandbox is usable through this gateway and must
3984+
// win over a `Suspended=True` condition. Agent Sandbox v1beta1 sets
3985+
// `Suspended=True (PodTerminated)` on stop and does not clear it on resume,
3986+
// so a resumed CR carries both `Ready=True` and a stale `Suspended=True`.
3987+
// Treating any `Suspended=True` as Stopped would pin the resumed sandbox at
3988+
// Starting forever (issue #2932). A genuine stop leaves `Ready` unset or
3989+
// False, so `Suspended` still resolves to Stopped in that case.
3990+
let ready = status.conditions.iter().any(|condition| {
3991+
condition.r#type.eq_ignore_ascii_case("Ready")
39853992
&& condition.status.eq_ignore_ascii_case("true")
3986-
}) {
3993+
});
3994+
3995+
if !ready
3996+
&& status.conditions.iter().any(|condition| {
3997+
condition.r#type.eq_ignore_ascii_case("Suspended")
3998+
&& condition.status.eq_ignore_ascii_case("true")
3999+
})
4000+
{
39874001
return SandboxPhase::Stopped;
39884002
}
39894003

@@ -6458,6 +6472,56 @@ mod tests {
64586472
assert_eq!(current.phase(), SandboxPhase::Stopped as i32);
64596473
}
64606474

6475+
#[tokio::test]
6476+
async fn resumed_v1beta1_snapshot_with_stale_suspended_reaches_ready() {
6477+
// Reproduces issue #2932: on Agent Sandbox v1beta1 a resumed CR reports
6478+
// Ready=True (DependenciesReady) alongside a stale Suspended=True
6479+
// (PodTerminated). Starting from the Starting phase that `start` sets, the
6480+
// reconciled sandbox must advance to Ready rather than being pinned at
6481+
// Starting by the stale Suspended condition.
6482+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
6483+
let sandbox = sandbox_record("sb-resumed", "sandbox-resumed", SandboxPhase::Starting);
6484+
runtime.store.put_message(&sandbox).await.unwrap();
6485+
register_test_supervisor_session(&runtime, sandbox.object_id());
6486+
6487+
let mut resumed = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name());
6488+
resumed.status = Some(DriverSandboxStatus {
6489+
sandbox_name: sandbox.object_name().to_string(),
6490+
instance_id: format!("{}-pod", sandbox.object_name()),
6491+
conditions: vec![
6492+
DriverCondition {
6493+
r#type: "Ready".to_string(),
6494+
status: "True".to_string(),
6495+
reason: "DependenciesReady".to_string(),
6496+
message: "Sandbox is ready".to_string(),
6497+
last_transition_time: String::new(),
6498+
},
6499+
DriverCondition {
6500+
r#type: "Suspended".to_string(),
6501+
status: "True".to_string(),
6502+
reason: "PodTerminated".to_string(),
6503+
message: "Pod terminated".to_string(),
6504+
last_transition_time: String::new(),
6505+
},
6506+
],
6507+
..Default::default()
6508+
});
6509+
6510+
runtime.apply_sandbox_update(resumed).await.unwrap();
6511+
6512+
let current = runtime
6513+
.store
6514+
.get_message::<Sandbox>(sandbox.object_id())
6515+
.await
6516+
.unwrap()
6517+
.unwrap();
6518+
assert_eq!(
6519+
current.phase(),
6520+
SandboxPhase::Ready as i32,
6521+
"a resumed, Ready sandbox must not stay Starting because of a stale Suspended condition"
6522+
);
6523+
}
6524+
64616525
#[tokio::test]
64626526
async fn stopped_container_snapshot_cannot_error_stopped_sandbox() {
64636527
let runtime = test_runtime(Arc::new(TestDriver::default())).await;

0 commit comments

Comments
 (0)