@@ -2013,11 +2013,117 @@ impl ComputeRuntime {
20132013 }
20142014
20152015 pub async fn cleanup_on_shutdown ( & self ) -> Result < ( ) , String > {
2016+ let stop_result = self . stop_persisted_sandboxes_on_shutdown ( ) . await ;
2017+
20162018 #[ cfg( unix) ]
2017- if let Some ( process) = & self . driver_process {
2018- process. shutdown ( ) . await ?;
2019+ let process_result = if let Some ( process) = & self . driver_process {
2020+ process. shutdown ( ) . await
2021+ } else {
2022+ Ok ( ( ) )
2023+ } ;
2024+
2025+ #[ cfg( not( unix) ) ]
2026+ let process_result: Result < ( ) , String > = Ok ( ( ) ) ;
2027+
2028+ match ( stop_result, process_result) {
2029+ ( Ok ( ( ) ) , Ok ( ( ) ) ) => Ok ( ( ) ) ,
2030+ ( Err ( stop_err) , Ok ( ( ) ) ) => Err ( stop_err) ,
2031+ ( Ok ( ( ) ) , Err ( process_err) ) => Err ( process_err) ,
2032+ ( Err ( stop_err) , Err ( process_err) ) => Err ( format ! (
2033+ "{stop_err}; managed driver process shutdown failed: {process_err}"
2034+ ) ) ,
2035+ }
2036+ }
2037+
2038+ /// Stop local compute during graceful gateway shutdown without changing
2039+ /// persisted lifecycle intent.
2040+ ///
2041+ /// An explicit sandbox stop persists `Stopped`; gateway shutdown does not.
2042+ /// Docker, Podman, and VM compute is stopped through the same public driver
2043+ /// RPC and restarted from the retained running-intent phase on gateway
2044+ /// startup. Kubernetes compute remains cluster-owned and is excluded.
2045+ async fn stop_persisted_sandboxes_on_shutdown ( & self ) -> Result < ( ) , String > {
2046+ if !matches ! (
2047+ self . driver_kind( ) ,
2048+ Some ( ComputeDriverKind :: Docker | ComputeDriverKind :: Podman | ComputeDriverKind :: Vm )
2049+ ) {
2050+ return Ok ( ( ) ) ;
2051+ }
2052+
2053+ let records = self
2054+ . store
2055+ . list_by_type ( Sandbox :: object_type ( ) , 1000 , 0 )
2056+ . await
2057+ . map_err ( |err| format ! ( "failed to list sandboxes for shutdown: {err}" ) ) ?;
2058+
2059+ let mut stopped = 0usize ;
2060+ let mut failed = 0usize ;
2061+
2062+ for record in records {
2063+ let sandbox = match Sandbox :: decode ( record. payload . as_slice ( ) ) {
2064+ Ok ( sandbox) => sandbox,
2065+ Err ( err) => {
2066+ warn ! ( error = %err, "Failed to decode sandbox record during gateway shutdown" ) ;
2067+ failed += 1 ;
2068+ continue ;
2069+ }
2070+ } ;
2071+
2072+ let phase = SandboxPhase :: try_from ( sandbox. phase ( ) ) . unwrap_or ( SandboxPhase :: Unknown ) ;
2073+ if !sandbox_phase_should_be_running ( phase) {
2074+ continue ;
2075+ }
2076+
2077+ let sandbox_id = sandbox. object_id ( ) . to_string ( ) ;
2078+ let sandbox_name = sandbox. object_name ( ) . to_string ( ) ;
2079+ let _lifecycle_guard = self . lifecycle_gates . lock_for ( & sandbox_id) . await ;
2080+ match self
2081+ . driver
2082+ . call ( "driver.stop_sandbox" , Some ( & sandbox_id) , |driver| {
2083+ let sandbox_id = sandbox_id. clone ( ) ;
2084+ let sandbox_name = sandbox_name. clone ( ) ;
2085+ async move {
2086+ driver
2087+ . stop_sandbox ( Request :: new ( StopSandboxRequest {
2088+ sandbox_id,
2089+ sandbox_name,
2090+ } ) )
2091+ . await
2092+ }
2093+ } )
2094+ . await
2095+ {
2096+ Ok ( _) => {
2097+ info ! (
2098+ sandbox_id = %sandbox. object_id( ) ,
2099+ sandbox_name = %sandbox. object_name( ) ,
2100+ ?phase,
2101+ "Stopped sandbox during gateway shutdown"
2102+ ) ;
2103+ stopped += 1 ;
2104+ }
2105+ Err ( err) => {
2106+ warn ! (
2107+ sandbox_id = %sandbox. object_id( ) ,
2108+ sandbox_name = %sandbox. object_name( ) ,
2109+ error = %err,
2110+ "Failed to stop sandbox during gateway shutdown"
2111+ ) ;
2112+ failed += 1 ;
2113+ }
2114+ }
2115+ }
2116+
2117+ if stopped > 0 || failed > 0 {
2118+ info ! ( stopped, failed, "Sandbox shutdown stop sweep complete" ) ;
2119+ }
2120+ if failed > 0 {
2121+ Err ( format ! (
2122+ "failed to stop {failed} sandbox(es) during gateway shutdown"
2123+ ) )
2124+ } else {
2125+ Ok ( ( ) )
20192126 }
2020- Ok ( ( ) )
20212127 }
20222128
20232129 /// Reconcile running intent for local compute after a gateway restart.
@@ -4275,6 +4381,7 @@ mod tests {
42754381 stop_release : Semaphore ,
42764382 stop_blocked : AtomicBool ,
42774383 stop_calls : AtomicUsize ,
4384+ stop_requests : TestMutex < Vec < ( String , String ) > > ,
42784385 stop_outcome : TestMutex < ControlledLifecycleOutcome > ,
42794386 start_started : Notify ,
42804387 start_finished : Notify ,
@@ -4306,6 +4413,7 @@ mod tests {
43064413 stop_release : Semaphore :: new ( 0 ) ,
43074414 stop_blocked : AtomicBool :: new ( false ) ,
43084415 stop_calls : AtomicUsize :: new ( 0 ) ,
4416+ stop_requests : TestMutex :: new ( Vec :: new ( ) ) ,
43094417 stop_outcome : TestMutex :: new ( ControlledLifecycleOutcome :: Ok ) ,
43104418 start_started : Notify :: new ( ) ,
43114419 start_finished : Notify :: new ( ) ,
@@ -4386,6 +4494,13 @@ mod tests {
43864494 self . stop_calls . load ( Ordering :: SeqCst )
43874495 }
43884496
4497+ fn stop_requests ( & self ) -> Vec < ( String , String ) > {
4498+ self . stop_requests
4499+ . lock ( )
4500+ . expect ( "stop requests lock poisoned" )
4501+ . clone ( )
4502+ }
4503+
43894504 fn start_calls ( & self ) -> usize {
43904505 self . start_calls . load ( Ordering :: SeqCst )
43914506 }
@@ -4486,8 +4601,13 @@ mod tests {
44864601
44874602 async fn stop_sandbox (
44884603 & self ,
4489- _request : Request < StopSandboxRequest > ,
4604+ request : Request < StopSandboxRequest > ,
44904605 ) -> Result < tonic:: Response < StopSandboxResponse > , Status > {
4606+ let request = request. into_inner ( ) ;
4607+ self . stop_requests
4608+ . lock ( )
4609+ . expect ( "stop requests lock poisoned" )
4610+ . push ( ( request. sandbox_id , request. sandbox_name ) ) ;
44914611 self . stop_calls . fetch_add ( 1 , Ordering :: SeqCst ) ;
44924612 self . stop_started . notify_one ( ) ;
44934613 if self . stop_blocked . load ( Ordering :: SeqCst ) {
@@ -8005,6 +8125,116 @@ mod tests {
80058125 ) ) ;
80068126 }
80078127
8128+ #[ tokio:: test]
8129+ async fn shutdown_stops_running_intent_without_changing_persisted_phase ( ) {
8130+ let driver = ControlledDriver :: new ( ) ;
8131+ let runtime = test_runtime_for_driver ( driver. clone ( ) , "docker" ) . await ;
8132+
8133+ for ( id, name, phase) in [
8134+ ( "sb-unspecified" , "unspecified" , SandboxPhase :: Unspecified ) ,
8135+ ( "sb-prov" , "prov" , SandboxPhase :: Provisioning ) ,
8136+ ( "sb-ready" , "ready" , SandboxPhase :: Ready ) ,
8137+ ( "sb-starting" , "starting" , SandboxPhase :: Starting ) ,
8138+ ( "sb-unknown" , "unknown" , SandboxPhase :: Unknown ) ,
8139+ ( "sb-stopping" , "stopping" , SandboxPhase :: Stopping ) ,
8140+ ( "sb-stopped" , "stopped" , SandboxPhase :: Stopped ) ,
8141+ ( "sb-deleting" , "deleting" , SandboxPhase :: Deleting ) ,
8142+ ( "sb-error" , "error" , SandboxPhase :: Error ) ,
8143+ ] {
8144+ runtime
8145+ . store
8146+ . put_message ( & sandbox_record ( id, name, phase) )
8147+ . await
8148+ . unwrap ( ) ;
8149+ }
8150+
8151+ runtime
8152+ . stop_persisted_sandboxes_on_shutdown ( )
8153+ . await
8154+ . unwrap ( ) ;
8155+
8156+ let mut called_ids = driver
8157+ . stop_requests ( )
8158+ . into_iter ( )
8159+ . map ( |( id, _) | id)
8160+ . collect :: < Vec < _ > > ( ) ;
8161+ called_ids. sort ( ) ;
8162+ assert_eq ! (
8163+ called_ids,
8164+ vec![
8165+ "sb-prov" . to_string( ) ,
8166+ "sb-ready" . to_string( ) ,
8167+ "sb-starting" . to_string( ) ,
8168+ "sb-unknown" . to_string( ) ,
8169+ "sb-unspecified" . to_string( ) ,
8170+ ]
8171+ ) ;
8172+
8173+ let stored = runtime
8174+ . store
8175+ . get_message :: < Sandbox > ( "sb-ready" )
8176+ . await
8177+ . unwrap ( )
8178+ . unwrap ( ) ;
8179+ assert_eq ! (
8180+ SandboxPhase :: try_from( stored. phase( ) ) . unwrap( ) ,
8181+ SandboxPhase :: Ready ,
8182+ "gateway shutdown must retain logical running intent"
8183+ ) ;
8184+ }
8185+
8186+ #[ tokio:: test]
8187+ async fn shutdown_stop_sweep_continues_after_driver_errors ( ) {
8188+ let driver = ControlledDriver :: new ( ) ;
8189+ driver. set_stop_outcome ( ControlledLifecycleOutcome :: Error ( "runtime angry" ) ) ;
8190+ let runtime = test_runtime_for_driver ( driver. clone ( ) , "podman" ) . await ;
8191+ for ( id, name) in [ ( "sb-1" , "one" ) , ( "sb-2" , "two" ) ] {
8192+ runtime
8193+ . store
8194+ . put_message ( & sandbox_record ( id, name, SandboxPhase :: Ready ) )
8195+ . await
8196+ . unwrap ( ) ;
8197+ }
8198+
8199+ let err = runtime
8200+ . stop_persisted_sandboxes_on_shutdown ( )
8201+ . await
8202+ . unwrap_err ( ) ;
8203+
8204+ assert ! ( err. contains( "failed to stop 2 sandbox(es)" ) ) ;
8205+ assert_eq ! ( driver. stop_calls( ) , 2 ) ;
8206+ }
8207+
8208+ #[ tokio:: test]
8209+ async fn shutdown_stop_sweep_runs_for_each_local_driver_only ( ) {
8210+ for ( driver_name, expected_calls) in [
8211+ ( "docker" , 1 ) ,
8212+ ( "podman" , 1 ) ,
8213+ ( "vm" , 1 ) ,
8214+ ( "kubernetes" , 0 ) ,
8215+ ( "extension" , 0 ) ,
8216+ ] {
8217+ let driver = ControlledDriver :: new ( ) ;
8218+ let runtime = test_runtime_for_driver ( driver. clone ( ) , driver_name) . await ;
8219+ runtime
8220+ . store
8221+ . put_message ( & sandbox_record ( "sb-1" , "sandbox" , SandboxPhase :: Ready ) )
8222+ . await
8223+ . unwrap ( ) ;
8224+
8225+ runtime
8226+ . stop_persisted_sandboxes_on_shutdown ( )
8227+ . await
8228+ . unwrap ( ) ;
8229+
8230+ assert_eq ! (
8231+ driver. stop_calls( ) ,
8232+ expected_calls,
8233+ "unexpected shutdown behavior for {driver_name}"
8234+ ) ;
8235+ }
8236+ }
8237+
80088238 #[ tokio:: test]
80098239 async fn start_persisted_sandboxes_starts_running_phases ( ) {
80108240 let driver = ControlledDriver :: new ( ) ;
0 commit comments