Skip to content

Commit 4101b12

Browse files
committed
fix(compute): recover Error-phase sandboxes on gateway startup (NVIDIA#2179)
After a Podman/Docker machine restart, all sandbox containers exit with SIGTERM (code 143). The gateway marks them Error, and on next startup the resume sweep skipped Error-phase sandboxes entirely — leaving them stuck until the user deleted and recreated them. Now resume_persisted_sandboxes() attempts Error-phase sandboxes too: - Ok(true): container exists and restarted — clear error, set phase to Provisioning so the watch loop promotes to Ready - Ok(false): container gone — leave as Error (no overwrite) - Err: driver failure — leave as Error (no overwrite) Also wires StartupResume for PodmanComputeDriver so the resume sweep runs on Podman-backed gateways (previously Docker-only). Signed-off-by: Ian Miller <[email protected]>
1 parent 265b74e commit 4101b12

1 file changed

Lines changed: 167 additions & 40 deletions

File tree

  • crates/openshell-server/src/compute

‎crates/openshell-server/src/compute/mod.rs‎

Lines changed: 167 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,15 @@ impl StartupResume for DockerComputeDriver {
113113
.map_err(|err| err.to_string())
114114
}
115115
}
116+
#[tonic::async_trait]
117+
impl StartupResume for PodmanComputeDriver {
118+
async fn resume_sandbox(&self, _sandbox_id: &str, sandbox_name: &str) -> Result<bool, String> {
119+
Self::resume_sandbox(self, sandbox_name)
120+
.await
121+
.map_err(|err| err.to_string())
122+
}
123+
}
124+
116125
/// Interval between store-vs-backend reconciliation sweeps.
117126
const RECONCILE_INTERVAL: Duration = Duration::from_secs(60);
118127

@@ -454,12 +463,13 @@ impl ComputeRuntime {
454463
let driver = PodmanComputeDriver::new(config)
455464
.await
456465
.map_err(|err| ComputeError::Message(err.to_string()))?;
466+
let startup_resume: Arc<dyn StartupResume> = Arc::new(driver.clone());
457467
let driver: SharedComputeDriver = Arc::new(PodmanDriverService::new(driver));
458468
Self::from_driver(
459469
ComputeDriverKind::Podman.as_str().to_string(),
460470
driver,
461471
None,
462-
None,
472+
Some(startup_resume),
463473
None,
464474
store,
465475
sandbox_index,
@@ -779,11 +789,12 @@ impl ComputeRuntime {
779789
/// Resume sandboxes whose store records say they should be running.
780790
/// Drivers that do not auto-restart compute resources across gateway
781791
/// restarts (currently only Docker) implement `StartupResume`. For
782-
/// each sandbox in the store whose phase is not `Deleting` or
783-
/// `Error`, we ask the driver to resume the underlying resource. If
784-
/// the driver reports that the resource no longer exists or fails to
785-
/// start, the sandbox is moved to the `Error` phase so the failure
786-
/// surfaces in the UI.
792+
/// each sandbox in the store whose phase is not `Deleting`, we ask
793+
/// the driver to resume the underlying resource. Error-phase
794+
/// sandboxes are included: if the container still exists (e.g. after
795+
/// a Podman machine restart), it is restarted and the sandbox is
796+
/// moved back to `Provisioning`. If the container is gone or the
797+
/// driver fails, the sandbox stays in `Error`.
787798
///
788799
/// Should be called once at gateway startup, before watchers spawn,
789800
/// so the watch loop sees the post-resume state on its first poll.
@@ -799,6 +810,7 @@ impl ComputeRuntime {
799810
.map_err(|e| e.to_string())?;
800811

801812
let mut resumed = 0usize;
813+
let mut recovered = 0usize;
802814
let mut missing = 0usize;
803815
let mut failed = 0usize;
804816

@@ -812,40 +824,45 @@ impl ComputeRuntime {
812824
};
813825

814826
let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown);
815-
if !sandbox_phase_should_be_running(phase) {
827+
if phase == SandboxPhase::Deleting {
816828
continue;
817829
}
830+
let sandbox_error = phase == SandboxPhase::Error;
818831

819832
match resume
820833
.resume_sandbox(sandbox.object_id(), sandbox.object_name())
821834
.await
822835
{
823836
Ok(true) => {
837+
if sandbox_error {
838+
self.clear_sandbox_error(&sandbox).await;
839+
}
824840
info!(
825841
sandbox_id = %sandbox.object_id(),
826842
sandbox_name = %sandbox.object_name(),
827843
?phase,
844+
recovered = sandbox_error,
828845
"Resumed sandbox during gateway startup"
829846
);
830847
resumed += 1;
848+
if sandbox_error {
849+
recovered += 1;
850+
}
831851
}
832852
Ok(false) => {
833-
// Backend resource is gone but the store still
834-
// remembers the sandbox. Mark Error so the UI
835-
// surfaces the inconsistency; the reconcile loop
836-
// will eventually prune it after the orphan grace
837-
// period.
838853
warn!(
839854
sandbox_id = %sandbox.object_id(),
840855
sandbox_name = %sandbox.object_name(),
841856
"Cannot resume sandbox: backend resource is missing"
842857
);
843-
self.mark_sandbox_error(
844-
&sandbox,
845-
"BackendResourceMissing",
846-
"Sandbox container disappeared while the gateway was offline",
847-
)
848-
.await;
858+
if !sandbox_error {
859+
self.mark_sandbox_error(
860+
&sandbox,
861+
"BackendResourceMissing",
862+
"Sandbox container disappeared while the gateway was offline",
863+
)
864+
.await;
865+
}
849866
missing += 1;
850867
}
851868
Err(err) => {
@@ -855,12 +872,14 @@ impl ComputeRuntime {
855872
error = %err,
856873
"Failed to resume sandbox during gateway startup"
857874
);
858-
self.mark_sandbox_error(
859-
&sandbox,
860-
"ResumeFailed",
861-
&format!("Failed to resume sandbox during gateway startup: {err}"),
862-
)
863-
.await;
875+
if !sandbox_error {
876+
self.mark_sandbox_error(
877+
&sandbox,
878+
"ResumeFailed",
879+
&format!("Failed to resume sandbox during gateway startup: {err}"),
880+
)
881+
.await;
882+
}
864883
failed += 1;
865884
}
866885
}
@@ -869,6 +888,7 @@ impl ComputeRuntime {
869888
if resumed > 0 || missing > 0 || failed > 0 {
870889
info!(
871890
resumed,
891+
recovered,
872892
missing_backend = missing,
873893
failed,
874894
"Sandbox resume sweep complete"
@@ -915,6 +935,42 @@ impl ComputeRuntime {
915935
}
916936
}
917937

938+
async fn clear_sandbox_error(&self, sandbox: &Sandbox) {
939+
let _guard = self.sync_lock.lock().await;
940+
let sandbox_id = sandbox.object_id().to_string();
941+
match self
942+
.store
943+
.update_message_cas::<Sandbox, _>(&sandbox_id, 0, |s| {
944+
s.set_phase(SandboxPhase::Provisioning as i32);
945+
let name = s.object_name().to_string();
946+
upsert_ready_condition(
947+
&mut s.status,
948+
&name,
949+
SandboxCondition {
950+
r#type: "Ready".to_string(),
951+
status: "False".to_string(),
952+
reason: "Resumed".to_string(),
953+
message: "Sandbox recovered during gateway startup".to_string(),
954+
last_transition_time: String::new(),
955+
},
956+
);
957+
})
958+
.await
959+
{
960+
Ok(updated) => {
961+
self.sandbox_index.update_from_sandbox(&updated);
962+
self.sandbox_watch_bus.notify(&sandbox_id);
963+
}
964+
Err(err) => {
965+
warn!(
966+
sandbox_id = %sandbox_id,
967+
error = %err,
968+
"Failed to clear sandbox error state during startup resume"
969+
);
970+
}
971+
}
972+
}
973+
918974
async fn lease_coordinator(self: Arc<Self>, mut shutdown_rx: watch::Receiver<bool>) {
919975
use lease::{LEASE_ACQUIRE_INTERVAL, LEASE_TTL, ReconcilerLease};
920976

@@ -2070,22 +2126,6 @@ fn rewrite_user_facing_conditions(status: &mut Option<SandboxStatus>, spec: Opti
20702126
}
20712127
}
20722128

2073-
/// Phases for which a sandbox should have a running compute resource.
2074-
/// `Deleting` and `Error` are intentionally excluded: deletion is in
2075-
/// progress, or the sandbox has already failed and should not be
2076-
/// silently revived. `Unspecified` is included because it is the proto
2077-
/// default value; persisted rows with that value should be reconciled
2078-
/// from the live driver state rather than skipped forever.
2079-
fn sandbox_phase_should_be_running(phase: SandboxPhase) -> bool {
2080-
matches!(
2081-
phase,
2082-
SandboxPhase::Unspecified
2083-
| SandboxPhase::Provisioning
2084-
| SandboxPhase::Ready
2085-
| SandboxPhase::Unknown
2086-
)
2087-
}
2088-
20892129
fn is_terminal_failure_reason(reason: &str) -> bool {
20902130
let reason = reason.to_ascii_lowercase();
20912131
let transient_reasons = [
@@ -3472,6 +3512,7 @@ mod tests {
34723512
assert_eq!(
34733513
called_ids,
34743514
vec![
3515+
"sb-error".to_string(),
34753516
"sb-prov".to_string(),
34763517
"sb-ready".to_string(),
34773518
"sb-unknown".to_string(),
@@ -3563,6 +3604,92 @@ mod tests {
35633604
);
35643605
}
35653606

3607+
#[tokio::test]
3608+
async fn resume_persisted_sandboxes_recovers_error_phase_when_container_exists() {
3609+
let resume = Arc::new(RecordingResume::default());
3610+
resume.set_result("sb-err-recover", Ok(true)).await;
3611+
let runtime =
3612+
test_runtime_with_resume(Arc::new(TestDriver::default()), Some(resume.clone())).await;
3613+
3614+
let sandbox = sandbox_record("sb-err-recover", "recover", SandboxPhase::Error);
3615+
runtime.store.put_message(&sandbox).await.unwrap();
3616+
3617+
runtime.resume_persisted_sandboxes().await.unwrap();
3618+
3619+
assert_eq!(resume.calls().await.len(), 1);
3620+
3621+
let stored = runtime
3622+
.store
3623+
.get_message::<Sandbox>("sb-err-recover")
3624+
.await
3625+
.unwrap()
3626+
.unwrap();
3627+
assert_eq!(
3628+
SandboxPhase::try_from(stored.phase()).unwrap(),
3629+
SandboxPhase::Provisioning
3630+
);
3631+
let ready = stored
3632+
.status
3633+
.as_ref()
3634+
.and_then(|s| s.conditions.iter().find(|c| c.r#type == "Ready"))
3635+
.expect("Ready condition present");
3636+
assert_eq!(ready.reason, "Resumed");
3637+
}
3638+
3639+
#[tokio::test]
3640+
async fn resume_persisted_sandboxes_leaves_error_when_container_missing() {
3641+
let resume = Arc::new(RecordingResume::default());
3642+
resume.set_result("sb-err-gone", Ok(false)).await;
3643+
let runtime =
3644+
test_runtime_with_resume(Arc::new(TestDriver::default()), Some(resume.clone())).await;
3645+
3646+
let sandbox = sandbox_record("sb-err-gone", "gone", SandboxPhase::Error);
3647+
runtime.store.put_message(&sandbox).await.unwrap();
3648+
3649+
runtime.resume_persisted_sandboxes().await.unwrap();
3650+
3651+
assert_eq!(resume.calls().await.len(), 1);
3652+
3653+
let stored = runtime
3654+
.store
3655+
.get_message::<Sandbox>("sb-err-gone")
3656+
.await
3657+
.unwrap()
3658+
.unwrap();
3659+
assert_eq!(
3660+
SandboxPhase::try_from(stored.phase()).unwrap(),
3661+
SandboxPhase::Error
3662+
);
3663+
}
3664+
3665+
#[tokio::test]
3666+
async fn resume_persisted_sandboxes_leaves_error_when_resume_fails() {
3667+
let resume = Arc::new(RecordingResume::default());
3668+
resume
3669+
.set_result("sb-err-fail", Err("docker broke".to_string()))
3670+
.await;
3671+
let runtime =
3672+
test_runtime_with_resume(Arc::new(TestDriver::default()), Some(resume.clone())).await;
3673+
3674+
let sandbox = sandbox_record("sb-err-fail", "fail", SandboxPhase::Error);
3675+
runtime.store.put_message(&sandbox).await.unwrap();
3676+
3677+
runtime.resume_persisted_sandboxes().await.unwrap();
3678+
3679+
assert_eq!(resume.calls().await.len(), 1);
3680+
3681+
let stored = runtime
3682+
.store
3683+
.get_message::<Sandbox>("sb-err-fail")
3684+
.await
3685+
.unwrap()
3686+
.unwrap();
3687+
assert_eq!(
3688+
SandboxPhase::try_from(stored.phase()).unwrap(),
3689+
SandboxPhase::Error
3690+
);
3691+
}
3692+
35663693
#[test]
35673694
fn build_platform_config_inverts_user_namespaces_to_host_users() {
35683695
use prost_types::value::Kind;

0 commit comments

Comments
 (0)