@@ -113,6 +113,15 @@ impl StartupResume for DockerComputeDriver {
113113 . map_err ( |err| err. to_string ( ) )
114114 }
115115}
116+ #[ tonic:: async_trait]
117+ impl StartupResume for PodmanComputeDriver {
118+ async fn resume_sandbox ( & self , _sandbox_id : & str , sandbox_name : & str ) -> Result < bool , String > {
119+ Self :: resume_sandbox ( self , sandbox_name)
120+ . await
121+ . map_err ( |err| err. to_string ( ) )
122+ }
123+ }
124+
116125/// Interval between store-vs-backend reconciliation sweeps.
117126const RECONCILE_INTERVAL : Duration = Duration :: from_secs ( 60 ) ;
118127
@@ -454,12 +463,13 @@ impl ComputeRuntime {
454463 let driver = PodmanComputeDriver :: new ( config)
455464 . await
456465 . map_err ( |err| ComputeError :: Message ( err. to_string ( ) ) ) ?;
466+ let startup_resume: Arc < dyn StartupResume > = Arc :: new ( driver. clone ( ) ) ;
457467 let driver: SharedComputeDriver = Arc :: new ( PodmanDriverService :: new ( driver) ) ;
458468 Self :: from_driver (
459469 ComputeDriverKind :: Podman . as_str ( ) . to_string ( ) ,
460470 driver,
461471 None ,
462- None ,
472+ Some ( startup_resume ) ,
463473 None ,
464474 store,
465475 sandbox_index,
@@ -779,11 +789,12 @@ impl ComputeRuntime {
779789 /// Resume sandboxes whose store records say they should be running.
780790 /// Drivers that do not auto-restart compute resources across gateway
781791 /// restarts (currently only Docker) implement `StartupResume`. For
782- /// each sandbox in the store whose phase is not `Deleting` or
783- /// `Error`, we ask the driver to resume the underlying resource. If
784- /// the driver reports that the resource no longer exists or fails to
785- /// start, the sandbox is moved to the `Error` phase so the failure
786- /// surfaces in the UI.
792+ /// each sandbox in the store whose phase is not `Deleting`, we ask
793+ /// the driver to resume the underlying resource. Error-phase
794+ /// sandboxes are included: if the container still exists (e.g. after
795+ /// a Podman machine restart), it is restarted and the sandbox is
796+ /// moved back to `Provisioning`. If the container is gone or the
797+ /// driver fails, the sandbox stays in `Error`.
787798 ///
788799 /// Should be called once at gateway startup, before watchers spawn,
789800 /// so the watch loop sees the post-resume state on its first poll.
@@ -799,6 +810,7 @@ impl ComputeRuntime {
799810 . map_err ( |e| e. to_string ( ) ) ?;
800811
801812 let mut resumed = 0usize ;
813+ let mut recovered = 0usize ;
802814 let mut missing = 0usize ;
803815 let mut failed = 0usize ;
804816
@@ -812,40 +824,45 @@ impl ComputeRuntime {
812824 } ;
813825
814826 let phase = SandboxPhase :: try_from ( sandbox. phase ( ) ) . unwrap_or ( SandboxPhase :: Unknown ) ;
815- if ! sandbox_phase_should_be_running ( phase) {
827+ if phase == SandboxPhase :: Deleting {
816828 continue ;
817829 }
830+ let sandbox_error = phase == SandboxPhase :: Error ;
818831
819832 match resume
820833 . resume_sandbox ( sandbox. object_id ( ) , sandbox. object_name ( ) )
821834 . await
822835 {
823836 Ok ( true ) => {
837+ if sandbox_error {
838+ self . clear_sandbox_error ( & sandbox) . await ;
839+ }
824840 info ! (
825841 sandbox_id = %sandbox. object_id( ) ,
826842 sandbox_name = %sandbox. object_name( ) ,
827843 ?phase,
844+ recovered = sandbox_error,
828845 "Resumed sandbox during gateway startup"
829846 ) ;
830847 resumed += 1 ;
848+ if sandbox_error {
849+ recovered += 1 ;
850+ }
831851 }
832852 Ok ( false ) => {
833- // Backend resource is gone but the store still
834- // remembers the sandbox. Mark Error so the UI
835- // surfaces the inconsistency; the reconcile loop
836- // will eventually prune it after the orphan grace
837- // period.
838853 warn ! (
839854 sandbox_id = %sandbox. object_id( ) ,
840855 sandbox_name = %sandbox. object_name( ) ,
841856 "Cannot resume sandbox: backend resource is missing"
842857 ) ;
843- self . mark_sandbox_error (
844- & sandbox,
845- "BackendResourceMissing" ,
846- "Sandbox container disappeared while the gateway was offline" ,
847- )
848- . await ;
858+ if !sandbox_error {
859+ self . mark_sandbox_error (
860+ & sandbox,
861+ "BackendResourceMissing" ,
862+ "Sandbox container disappeared while the gateway was offline" ,
863+ )
864+ . await ;
865+ }
849866 missing += 1 ;
850867 }
851868 Err ( err) => {
@@ -855,12 +872,14 @@ impl ComputeRuntime {
855872 error = %err,
856873 "Failed to resume sandbox during gateway startup"
857874 ) ;
858- self . mark_sandbox_error (
859- & sandbox,
860- "ResumeFailed" ,
861- & format ! ( "Failed to resume sandbox during gateway startup: {err}" ) ,
862- )
863- . await ;
875+ if !sandbox_error {
876+ self . mark_sandbox_error (
877+ & sandbox,
878+ "ResumeFailed" ,
879+ & format ! ( "Failed to resume sandbox during gateway startup: {err}" ) ,
880+ )
881+ . await ;
882+ }
864883 failed += 1 ;
865884 }
866885 }
@@ -869,6 +888,7 @@ impl ComputeRuntime {
869888 if resumed > 0 || missing > 0 || failed > 0 {
870889 info ! (
871890 resumed,
891+ recovered,
872892 missing_backend = missing,
873893 failed,
874894 "Sandbox resume sweep complete"
@@ -915,6 +935,42 @@ impl ComputeRuntime {
915935 }
916936 }
917937
938+ async fn clear_sandbox_error ( & self , sandbox : & Sandbox ) {
939+ let _guard = self . sync_lock . lock ( ) . await ;
940+ let sandbox_id = sandbox. object_id ( ) . to_string ( ) ;
941+ match self
942+ . store
943+ . update_message_cas :: < Sandbox , _ > ( & sandbox_id, 0 , |s| {
944+ s. set_phase ( SandboxPhase :: Provisioning as i32 ) ;
945+ let name = s. object_name ( ) . to_string ( ) ;
946+ upsert_ready_condition (
947+ & mut s. status ,
948+ & name,
949+ SandboxCondition {
950+ r#type : "Ready" . to_string ( ) ,
951+ status : "False" . to_string ( ) ,
952+ reason : "Resumed" . to_string ( ) ,
953+ message : "Sandbox recovered during gateway startup" . to_string ( ) ,
954+ last_transition_time : String :: new ( ) ,
955+ } ,
956+ ) ;
957+ } )
958+ . await
959+ {
960+ Ok ( updated) => {
961+ self . sandbox_index . update_from_sandbox ( & updated) ;
962+ self . sandbox_watch_bus . notify ( & sandbox_id) ;
963+ }
964+ Err ( err) => {
965+ warn ! (
966+ sandbox_id = %sandbox_id,
967+ error = %err,
968+ "Failed to clear sandbox error state during startup resume"
969+ ) ;
970+ }
971+ }
972+ }
973+
918974 async fn lease_coordinator ( self : Arc < Self > , mut shutdown_rx : watch:: Receiver < bool > ) {
919975 use lease:: { LEASE_ACQUIRE_INTERVAL , LEASE_TTL , ReconcilerLease } ;
920976
@@ -2070,22 +2126,6 @@ fn rewrite_user_facing_conditions(status: &mut Option<SandboxStatus>, spec: Opti
20702126 }
20712127}
20722128
2073- /// Phases for which a sandbox should have a running compute resource.
2074- /// `Deleting` and `Error` are intentionally excluded: deletion is in
2075- /// progress, or the sandbox has already failed and should not be
2076- /// silently revived. `Unspecified` is included because it is the proto
2077- /// default value; persisted rows with that value should be reconciled
2078- /// from the live driver state rather than skipped forever.
2079- fn sandbox_phase_should_be_running ( phase : SandboxPhase ) -> bool {
2080- matches ! (
2081- phase,
2082- SandboxPhase :: Unspecified
2083- | SandboxPhase :: Provisioning
2084- | SandboxPhase :: Ready
2085- | SandboxPhase :: Unknown
2086- )
2087- }
2088-
20892129fn is_terminal_failure_reason ( reason : & str ) -> bool {
20902130 let reason = reason. to_ascii_lowercase ( ) ;
20912131 let transient_reasons = [
@@ -3472,6 +3512,7 @@ mod tests {
34723512 assert_eq ! (
34733513 called_ids,
34743514 vec![
3515+ "sb-error" . to_string( ) ,
34753516 "sb-prov" . to_string( ) ,
34763517 "sb-ready" . to_string( ) ,
34773518 "sb-unknown" . to_string( ) ,
@@ -3563,6 +3604,92 @@ mod tests {
35633604 ) ;
35643605 }
35653606
3607+ #[ tokio:: test]
3608+ async fn resume_persisted_sandboxes_recovers_error_phase_when_container_exists ( ) {
3609+ let resume = Arc :: new ( RecordingResume :: default ( ) ) ;
3610+ resume. set_result ( "sb-err-recover" , Ok ( true ) ) . await ;
3611+ let runtime =
3612+ test_runtime_with_resume ( Arc :: new ( TestDriver :: default ( ) ) , Some ( resume. clone ( ) ) ) . await ;
3613+
3614+ let sandbox = sandbox_record ( "sb-err-recover" , "recover" , SandboxPhase :: Error ) ;
3615+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
3616+
3617+ runtime. resume_persisted_sandboxes ( ) . await . unwrap ( ) ;
3618+
3619+ assert_eq ! ( resume. calls( ) . await . len( ) , 1 ) ;
3620+
3621+ let stored = runtime
3622+ . store
3623+ . get_message :: < Sandbox > ( "sb-err-recover" )
3624+ . await
3625+ . unwrap ( )
3626+ . unwrap ( ) ;
3627+ assert_eq ! (
3628+ SandboxPhase :: try_from( stored. phase( ) ) . unwrap( ) ,
3629+ SandboxPhase :: Provisioning
3630+ ) ;
3631+ let ready = stored
3632+ . status
3633+ . as_ref ( )
3634+ . and_then ( |s| s. conditions . iter ( ) . find ( |c| c. r#type == "Ready" ) )
3635+ . expect ( "Ready condition present" ) ;
3636+ assert_eq ! ( ready. reason, "Resumed" ) ;
3637+ }
3638+
3639+ #[ tokio:: test]
3640+ async fn resume_persisted_sandboxes_leaves_error_when_container_missing ( ) {
3641+ let resume = Arc :: new ( RecordingResume :: default ( ) ) ;
3642+ resume. set_result ( "sb-err-gone" , Ok ( false ) ) . await ;
3643+ let runtime =
3644+ test_runtime_with_resume ( Arc :: new ( TestDriver :: default ( ) ) , Some ( resume. clone ( ) ) ) . await ;
3645+
3646+ let sandbox = sandbox_record ( "sb-err-gone" , "gone" , SandboxPhase :: Error ) ;
3647+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
3648+
3649+ runtime. resume_persisted_sandboxes ( ) . await . unwrap ( ) ;
3650+
3651+ assert_eq ! ( resume. calls( ) . await . len( ) , 1 ) ;
3652+
3653+ let stored = runtime
3654+ . store
3655+ . get_message :: < Sandbox > ( "sb-err-gone" )
3656+ . await
3657+ . unwrap ( )
3658+ . unwrap ( ) ;
3659+ assert_eq ! (
3660+ SandboxPhase :: try_from( stored. phase( ) ) . unwrap( ) ,
3661+ SandboxPhase :: Error
3662+ ) ;
3663+ }
3664+
3665+ #[ tokio:: test]
3666+ async fn resume_persisted_sandboxes_leaves_error_when_resume_fails ( ) {
3667+ let resume = Arc :: new ( RecordingResume :: default ( ) ) ;
3668+ resume
3669+ . set_result ( "sb-err-fail" , Err ( "docker broke" . to_string ( ) ) )
3670+ . await ;
3671+ let runtime =
3672+ test_runtime_with_resume ( Arc :: new ( TestDriver :: default ( ) ) , Some ( resume. clone ( ) ) ) . await ;
3673+
3674+ let sandbox = sandbox_record ( "sb-err-fail" , "fail" , SandboxPhase :: Error ) ;
3675+ runtime. store . put_message ( & sandbox) . await . unwrap ( ) ;
3676+
3677+ runtime. resume_persisted_sandboxes ( ) . await . unwrap ( ) ;
3678+
3679+ assert_eq ! ( resume. calls( ) . await . len( ) , 1 ) ;
3680+
3681+ let stored = runtime
3682+ . store
3683+ . get_message :: < Sandbox > ( "sb-err-fail" )
3684+ . await
3685+ . unwrap ( )
3686+ . unwrap ( ) ;
3687+ assert_eq ! (
3688+ SandboxPhase :: try_from( stored. phase( ) ) . unwrap( ) ,
3689+ SandboxPhase :: Error
3690+ ) ;
3691+ }
3692+
35663693 #[ test]
35673694 fn build_platform_config_inverts_user_namespaces_to_host_users ( ) {
35683695 use prost_types:: value:: Kind ;
0 commit comments