Skip to content

Commit 4709871

Browse files
committed
fix(server): recover a sandbox to Ready after its container restarts
A supervisor that reconnects after a container restart is a fresh instance, but the sandbox stays stuck in Error forever. The main-process exit report sent when the container is restarted leaves the sandbox in Error, and the subsequent ConnectSupervisor from the restarted container is ignored because Error is treated as terminal. Every lifecycle operation (exec, stop, start, recover, rebuild) is then rejected, and the sandbox can only be recovered by destroying and recreating it. Accept a supervisor session while the sandbox is in Error as long as the connecting instance is fresh (a container restart), clearing the failed main-process result and restoring Ready so exec and lifecycle operations work again without user action. A reconnect from the same instance, and a Completed sandbox, remain terminal. Fixes #4828 Signed-off-by: Chengjie Wang <[email protected]>
1 parent 9a85667 commit 4709871

1 file changed

Lines changed: 133 additions & 8 deletions

File tree

  • crates/openshell-server/src/compute

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 133 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5292,14 +5292,30 @@ impl ComputeRuntime {
52925292
self.schedule_ephemeral_sandbox_delete(&current);
52935293
return Ok(());
52945294
}
5295-
if matches!(
5296-
current_phase,
5297-
SandboxPhase::Deleting
5298-
| SandboxPhase::Error
5299-
| SandboxPhase::Stopping
5300-
| SandboxPhase::Stopped
5301-
| SandboxPhase::Completed
5302-
) {
5295+
// A container restart kills the supervisor together with the main
5296+
// process, so the supervisor that reconnects afterwards is a
5297+
// *fresh* instance. Its connect outlives the main-process exit
5298+
// report from the instance it replaces: accept the session and
5299+
// restore Ready instead of leaving the sandbox stuck in Error.
5300+
// A reconnect from the same instance (e.g. a main-process crash
5301+
// that is not being restarted) is still rejected, and Completed
5302+
// stays terminal.
5303+
let recovering_from_container_restart = connected
5304+
&& current_phase == SandboxPhase::Error
5305+
&& current.status.as_ref().is_some_and(|status| {
5306+
!status.main_process_instance_id.is_empty()
5307+
&& Some(status.main_process_instance_id.as_str()) != instance_id
5308+
});
5309+
if !recovering_from_container_restart
5310+
&& matches!(
5311+
current_phase,
5312+
SandboxPhase::Deleting
5313+
| SandboxPhase::Error
5314+
| SandboxPhase::Stopping
5315+
| SandboxPhase::Stopped
5316+
| SandboxPhase::Completed
5317+
)
5318+
{
53035319
return Ok(());
53045320
}
53055321
if !connected && current_phase != SandboxPhase::Ready {
@@ -15248,6 +15264,115 @@ mod tests {
1524815264
);
1524915265
}
1525015266

15267+
#[tokio::test]
15268+
async fn container_restart_fresh_supervisor_restores_ready_from_main_process_error() {
15269+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
15270+
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Error);
15271+
sandbox.status = Some(SandboxStatus {
15272+
phase: SandboxPhase::Error as i32,
15273+
main_process_instance_id: "instance-1".into(),
15274+
exit_code: Some(143),
15275+
conditions: vec![SandboxCondition {
15276+
r#type: "Ready".into(),
15277+
status: "False".into(),
15278+
reason: "MainProcessFailed".into(),
15279+
message: "Canonical main process exited with status 143".into(),
15280+
..Default::default()
15281+
}],
15282+
..Default::default()
15283+
});
15284+
runtime.store.put_message(&sandbox).await.unwrap();
15285+
15286+
// After the container restarts, a fresh supervisor instance connects.
15287+
runtime
15288+
.supervisor_session_connected("sb-1", "instance-2")
15289+
.await
15290+
.unwrap();
15291+
15292+
let stored = runtime
15293+
.store
15294+
.get_message::<Sandbox>("sb-1")
15295+
.await
15296+
.unwrap()
15297+
.unwrap();
15298+
assert_eq!(
15299+
SandboxPhase::try_from(stored.phase()).unwrap(),
15300+
SandboxPhase::Ready
15301+
);
15302+
let status = stored.status.unwrap();
15303+
assert_eq!(status.main_process_instance_id, "instance-2");
15304+
assert_eq!(status.exit_code, None);
15305+
assert!(status.conditions.iter().any(|condition| {
15306+
condition.r#type == "Ready"
15307+
&& condition.status == "True"
15308+
&& condition.reason == "DependenciesReady"
15309+
}));
15310+
}
15311+
15312+
#[tokio::test]
15313+
async fn container_restart_same_instance_reconnect_keeps_sandbox_in_error() {
15314+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
15315+
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Error);
15316+
sandbox.status = Some(SandboxStatus {
15317+
phase: SandboxPhase::Error as i32,
15318+
main_process_instance_id: "instance-1".into(),
15319+
exit_code: Some(9),
15320+
conditions: vec![SandboxCondition {
15321+
r#type: "Ready".into(),
15322+
status: "False".into(),
15323+
reason: "MainProcessFailed".into(),
15324+
message: "Canonical main process exited with status 9".into(),
15325+
..Default::default()
15326+
}],
15327+
..Default::default()
15328+
});
15329+
runtime.store.put_message(&sandbox).await.unwrap();
15330+
15331+
// The same supervisor instance reconnecting (e.g. a gateway restart
15332+
// after a main-process crash that is not being restarted) must not
15333+
// resurrect a failed sandbox.
15334+
runtime
15335+
.supervisor_session_connected("sb-1", "instance-1")
15336+
.await
15337+
.unwrap();
15338+
15339+
let stored = runtime
15340+
.store
15341+
.get_message::<Sandbox>("sb-1")
15342+
.await
15343+
.unwrap()
15344+
.unwrap();
15345+
assert_eq!(stored.phase(), SandboxPhase::Error as i32);
15346+
assert_eq!(stored.status.unwrap().exit_code, Some(9));
15347+
}
15348+
15349+
#[tokio::test]
15350+
async fn container_restart_reconnect_does_not_revive_completed_sandbox() {
15351+
let runtime = test_runtime(Arc::new(TestDriver::default())).await;
15352+
let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Completed);
15353+
sandbox.status = Some(SandboxStatus {
15354+
phase: SandboxPhase::Completed as i32,
15355+
main_process_instance_id: "instance-1".into(),
15356+
exit_code: Some(0),
15357+
..Default::default()
15358+
});
15359+
runtime.store.put_message(&sandbox).await.unwrap();
15360+
15361+
runtime
15362+
.supervisor_session_connected("sb-1", "instance-2")
15363+
.await
15364+
.unwrap();
15365+
15366+
let stored = runtime
15367+
.store
15368+
.get_message::<Sandbox>("sb-1")
15369+
.await
15370+
.unwrap()
15371+
.unwrap();
15372+
assert_eq!(stored.phase(), SandboxPhase::Completed as i32);
15373+
assert_eq!(stored.status.unwrap().exit_code, Some(0));
15374+
}
15375+
1525115376
#[tokio::test]
1525215377
async fn supervisor_session_connected_rejects_stopped_sandbox() {
1525315378
let runtime = test_runtime(Arc::new(TestDriver::default())).await;

0 commit comments

Comments
 (0)