Skip to content

Commit fc533b0

Browse files
committed
fix(server): recover a sandbox to Ready after its container restarts
A supervisor that reconnects after a container restart is a fresh instance, but the sandbox stays stuck in Error forever. The main-process exit report sent when the container is restarted leaves the sandbox in Error, and the subsequent ConnectSupervisor from the restarted container is ignored because Error is treated as terminal. Every lifecycle operation (exec, stop, start, recover, rebuild) is then rejected, and the sandbox can only be recovered by destroying and recreating it. Accept a supervisor session while the sandbox is in Error as long as the connecting instance is fresh (a container restart), clearing the failed main-process result and restoring Ready so exec and lifecycle operations work again without user action. A reconnect from the same instance, and a Completed sandbox, remain terminal. Fixes #4828 Signed-off-by: Chengjie Wang <[email protected]>
1 parent 4d4c15c commit fc533b0

1 file changed

Lines changed: 135 additions & 10 deletions

File tree

  • crates/openshell-server/src/compute

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 135 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5725,16 +5725,32 @@ impl ComputeRuntime {
57255725
self.schedule_ephemeral_sandbox_delete(&current);
57265726
return Ok(());
57275727
}
5728-
if matches!(
5729-
current_phase,
5730-
SandboxPhase::Deleting
5731-
| SandboxPhase::Error
5732-
| SandboxPhase::Stopping
5733-
| SandboxPhase::Stopped
5734-
| SandboxPhase::Suspending
5735-
| SandboxPhase::Suspended
5736-
| SandboxPhase::Completed
5737-
) {
5728+
// A container restart kills the supervisor together with the main
5729+
// process, so the supervisor that reconnects afterwards is a
5730+
// *fresh* instance. Its connect outlives the main-process exit
5731+
// report from the instance it replaces: accept the session and
5732+
// restore Ready instead of leaving the sandbox stuck in Error.
5733+
// A reconnect from the same instance (e.g. a main-process crash
5734+
// that is not being restarted) is still rejected, and Completed
5735+
// stays terminal.
5736+
let recovering_from_container_restart = connected
5737+
&& current_phase == SandboxPhase::Error
5738+
&& current.status.as_ref().is_some_and(|status| {
5739+
!status.main_process_instance_id.is_empty()
5740+
&& Some(status.main_process_instance_id.as_str()) != instance_id
5741+
});
5742+
if !recovering_from_container_restart
5743+
&& matches!(
5744+
current_phase,
5745+
SandboxPhase::Deleting
5746+
| SandboxPhase::Error
5747+
| SandboxPhase::Stopping
5748+
| SandboxPhase::Stopped
5749+
| SandboxPhase::Suspending
5750+
| SandboxPhase::Suspended
5751+
| SandboxPhase::Completed
5752+
)
5753+
{
57385754
return Ok(());
57395755
}
57405756
if !connected && current_phase != SandboxPhase::Ready {
@@ -15875,6 +15891,115 @@ mod tests {
1587515891
);
1587615892
}
1587715893

15894+
#[tokio::test]
15895+
async fn container_restart_fresh_supervisor_restores_ready_from_main_process_error() {
15896+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
15897+
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Error);
15898+
sandbox.status = Some(SandboxStatus {
15899+
phase: SandboxPhase::Error as i32,
15900+
main_process_instance_id: "instance-1".into(),
15901+
exit_code: Some(143),
15902+
conditions: vec![SandboxCondition {
15903+
r#type: "Ready".into(),
15904+
status: "False".into(),
15905+
reason: "MainProcessFailed".into(),
15906+
message: "Canonical main process exited with status 143".into(),
15907+
..Default::default()
15908+
}],
15909+
..Default::default()
15910+
});
15911+
runtime.store.put_message(&sandbox).await.unwrap();
15912+
15913+
// After the container restarts, a fresh supervisor instance connects.
15914+
runtime
15915+
.supervisor_session_connected("sb-1", "instance-2")
15916+
.await
15917+
.unwrap();
15918+
15919+
let stored = runtime
15920+
.store
15921+
.get_message::<Sandbox>("sb-1")
15922+
.await
15923+
.unwrap()
15924+
.unwrap();
15925+
assert_eq!(
15926+
SandboxPhase::try_from(stored.phase()).unwrap(),
15927+
SandboxPhase::Ready
15928+
);
15929+
let status = stored.status.unwrap();
15930+
assert_eq!(status.main_process_instance_id, "instance-2");
15931+
assert_eq!(status.exit_code, None);
15932+
assert!(status.conditions.iter().any(|condition| {
15933+
condition.r#type == "Ready"
15934+
&& condition.status == "True"
15935+
&& condition.reason == "DependenciesReady"
15936+
}));
15937+
}
15938+
15939+
#[tokio::test]
15940+
async fn container_restart_same_instance_reconnect_keeps_sandbox_in_error() {
15941+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
15942+
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Error);
15943+
sandbox.status = Some(SandboxStatus {
15944+
phase: SandboxPhase::Error as i32,
15945+
main_process_instance_id: "instance-1".into(),
15946+
exit_code: Some(9),
15947+
conditions: vec![SandboxCondition {
15948+
r#type: "Ready".into(),
15949+
status: "False".into(),
15950+
reason: "MainProcessFailed".into(),
15951+
message: "Canonical main process exited with status 9".into(),
15952+
..Default::default()
15953+
}],
15954+
..Default::default()
15955+
});
15956+
runtime.store.put_message(&sandbox).await.unwrap();
15957+
15958+
// The same supervisor instance reconnecting (e.g. a gateway restart
15959+
// after a main-process crash that is not being restarted) must not
15960+
// resurrect a failed sandbox.
15961+
runtime
15962+
.supervisor_session_connected("sb-1", "instance-1")
15963+
.await
15964+
.unwrap();
15965+
15966+
let stored = runtime
15967+
.store
15968+
.get_message::<Sandbox>("sb-1")
15969+
.await
15970+
.unwrap()
15971+
.unwrap();
15972+
assert_eq!(stored.phase(), SandboxPhase::Error as i32);
15973+
assert_eq!(stored.status.unwrap().exit_code, Some(9));
15974+
}
15975+
15976+
#[tokio::test]
15977+
async fn container_restart_reconnect_does_not_revive_completed_sandbox() {
15978+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
15979+
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Completed);
15980+
sandbox.status = Some(SandboxStatus {
15981+
phase: SandboxPhase::Completed as i32,
15982+
main_process_instance_id: "instance-1".into(),
15983+
exit_code: Some(0),
15984+
..Default::default()
15985+
});
15986+
runtime.store.put_message(&sandbox).await.unwrap();
15987+
15988+
runtime
15989+
.supervisor_session_connected("sb-1", "instance-2")
15990+
.await
15991+
.unwrap();
15992+
15993+
let stored = runtime
15994+
.store
15995+
.get_message::<Sandbox>("sb-1")
15996+
.await
15997+
.unwrap()
15998+
.unwrap();
15999+
assert_eq!(stored.phase(), SandboxPhase::Completed as i32);
16000+
assert_eq!(stored.status.unwrap().exit_code, Some(0));
16001+
}
16002+
1587816003
#[tokio::test]
1587916004
async fn supervisor_session_connected_rejects_stopped_sandbox() {
1588016005
let runtime = test_runtime(Arc::new(TestDriver::default())).await;

0 commit comments

Comments
 (0)