@@ -507,8 +507,9 @@ use tokio::time::timeout;
507507/// Time the workload gets to exit after TERM before it is killed.
508508const WORKLOAD_TERM_GRACE : Duration = Duration :: from_secs ( 5 ) ;
509509
510- /// Bound on stopping an instance, including its exit report and trace flush.
511- /// Below the 10 second grace the Docker driver gives the supervisor.
510+ /// Bound on stopping a dedicated supervisor's instance, including its exit
511+ /// report and trace flush. Below the 10 second grace the Docker driver gives
512+ /// the supervisor.
512513const SHUTDOWN_TIMEOUT : Duration = Duration :: from_secs ( 8 ) ;
513514
514515/// Exit code reported when the boundary is lost before it reports the
@@ -1817,41 +1818,24 @@ pub async fn run_sandbox_instance(
18171818 ) ) ;
18181819 }
18191820 ( ) = & mut shutdown_requested => {
1820- let deadline = tokio:: time:: Instant :: now( ) + SHUTDOWN_TIMEOUT ;
1821- shutdown_deadline = Some ( deadline) ;
18221821 if shutdown_ends_process {
1822+ shutdown_deadline = Some ( tokio:: time:: Instant :: now( ) + SHUTDOWN_TIMEOUT ) ;
18231823 running. begin_stop( ) ;
18241824 }
1825- let stop = async {
1825+ let wait = async {
18261826 let _ = agent
18271827 . signal( openshell_isolation_interface:: contract:: BoundarySignal :: Term )
18281828 . await ;
1829- let status = if let Ok ( result) = timeout( WORKLOAD_TERM_GRACE , agent. wait( ) ) . await {
1829+ if let Ok ( result) = timeout( WORKLOAD_TERM_GRACE , agent. wait( ) ) . await {
18301830 result
18311831 } else {
18321832 let _ = agent. terminate( ) . await ;
18331833 agent. wait( ) . await
1834- } ?;
1835- running. terminate( ) . await ?;
1836- Ok ( status)
1837- } ;
1838- let exit_code = match tokio:: time:: timeout_at( deadline, stop) . await {
1839- Ok ( Ok ( status) ) => boundary_exit_code( status) ,
1840- Ok ( Err (
1841- openshell_isolation_interface:: contract:: BackendError :: Unavailable ( error)
1842- | openshell_isolation_interface:: contract:: BackendError :: Terminated ( error) ,
1843- ) ) => {
1844- warn!( %error, "Isolation boundary lost while stopping; finishing shutdown" ) ;
1845- STOPPED_EXIT_CODE
1846- }
1847- Ok ( Err ( error) ) => {
1848- return Err ( miette:: miette!( "stop sandbox workload: {error}" ) ) ;
1849- }
1850- Err ( _) => {
1851- warn!( "Isolation boundary did not stop in time; finishing shutdown" ) ;
1852- STOPPED_EXIT_CODE
18531834 }
18541835 } ;
1836+ let exit_code =
1837+ Box :: pin( stop_workload( wait, || running. terminate( ) , shutdown_deadline) )
1838+ . await ?;
18551839 break Some ( ( exit_code, false ) ) ;
18561840 }
18571841 // The agent wait is cancelled while suspending so a fenced
@@ -1972,6 +1956,58 @@ pub async fn run_sandbox_instance(
19721956 Ok ( exit_code)
19731957}
19741958
1959+ /// Waits for a stopping workload, terminates its boundary, and returns the
1960+ /// workload's exit code.
1961+ ///
1962+ /// With a `deadline`, stopping also ends this process: the stop is abandoned
1963+ /// when the deadline passes, and a lost boundary does not fail it. The
1964+ /// workload's status is reported when the boundary returned one, otherwise
1965+ /// `STOPPED_EXIT_CODE`.
1966+ async fn stop_workload < W , T , F > (
1967+ wait : W ,
1968+ terminate : T ,
1969+ deadline : Option < tokio:: time:: Instant > ,
1970+ ) -> Result < i32 >
1971+ where
1972+ W : Future <
1973+ Output = std:: result:: Result <
1974+ openshell_isolation_interface:: contract:: BoundaryExitStatus ,
1975+ openshell_isolation_interface:: contract:: BackendError ,
1976+ > ,
1977+ > ,
1978+ T : FnOnce ( ) -> F ,
1979+ F : Future <
1980+ Output = std:: result:: Result < ( ) , openshell_isolation_interface:: contract:: BackendError > ,
1981+ > ,
1982+ {
1983+ let Some ( deadline) = deadline else {
1984+ let status = wait
1985+ . await
1986+ . map_err ( |error| miette:: miette!( error. to_string( ) ) ) ?;
1987+ terminate ( ) . await . map_err ( |error| {
1988+ miette:: miette!( "sandbox did not acknowledge terminal state: {error}" )
1989+ } ) ?;
1990+ return Ok ( boundary_exit_code ( status) ) ;
1991+ } ;
1992+ let mut observed = None ;
1993+ let stop = async {
1994+ observed = Some ( wait. await ?) ;
1995+ terminate ( ) . await
1996+ } ;
1997+ match tokio:: time:: timeout_at ( deadline, stop) . await {
1998+ Ok ( Ok ( ( ) ) ) => { }
1999+ Ok ( Err (
2000+ openshell_isolation_interface:: contract:: BackendError :: Unavailable ( error)
2001+ | openshell_isolation_interface:: contract:: BackendError :: Terminated ( error) ,
2002+ ) ) => {
2003+ warn ! ( %error, "Isolation boundary lost while stopping; finishing shutdown" ) ;
2004+ }
2005+ Ok ( Err ( error) ) => return Err ( miette:: miette!( "stop sandbox workload: {error}" ) ) ,
2006+ Err ( _) => warn ! ( "Isolation boundary did not stop in time; finishing shutdown" ) ,
2007+ }
2008+ Ok ( observed. map_or ( STOPPED_EXIT_CODE , boundary_exit_code) )
2009+ }
2010+
19752011fn boundary_exit_code ( status : openshell_isolation_interface:: contract:: BoundaryExitStatus ) -> i32 {
19762012 match status {
19772013 openshell_isolation_interface:: contract:: BoundaryExitStatus :: Exited ( code) => code,
@@ -5071,6 +5107,80 @@ mod tests {
50715107 ) ;
50725108 }
50735109
5110+ use openshell_isolation_interface:: contract:: { BackendError , BoundaryExitStatus } ;
5111+
5112+ fn stop_deadline ( ) -> tokio:: time:: Instant {
5113+ tokio:: time:: Instant :: now ( ) + SHUTDOWN_TIMEOUT
5114+ }
5115+
5116+ #[ tokio:: test]
5117+ async fn stopping_process_keeps_workload_status_when_boundary_is_lost ( ) {
5118+ let exit_code = stop_workload (
5119+ async { Ok ( BoundaryExitStatus :: Exited ( 0 ) ) } ,
5120+ || async { Err ( BackendError :: Unavailable ( "connection lost" . into ( ) ) ) } ,
5121+ Some ( stop_deadline ( ) ) ,
5122+ )
5123+ . await
5124+ . expect ( "a lost boundary does not fail the stop" ) ;
5125+ assert_eq ! ( exit_code, 0 ) ;
5126+ }
5127+
5128+ #[ tokio:: test]
5129+ async fn stopping_process_reports_terminated_without_workload_status ( ) {
5130+ let exit_code = stop_workload (
5131+ async { Err ( BackendError :: Terminated ( "boundary lost during wait" . into ( ) ) ) } ,
5132+ || async { Ok ( ( ) ) } ,
5133+ Some ( stop_deadline ( ) ) ,
5134+ )
5135+ . await
5136+ . expect ( "a lost boundary does not fail the stop" ) ;
5137+ assert_eq ! ( exit_code, STOPPED_EXIT_CODE ) ;
5138+ }
5139+
5140+ #[ tokio:: test( start_paused = true ) ]
5141+ async fn stopping_process_keeps_workload_status_past_its_deadline ( ) {
5142+ let exit_code = stop_workload (
5143+ async { Ok ( BoundaryExitStatus :: Exited ( 3 ) ) } ,
5144+ std:: future:: pending,
5145+ Some ( stop_deadline ( ) ) ,
5146+ )
5147+ . await
5148+ . expect ( "an expired deadline does not fail the stop" ) ;
5149+ assert_eq ! ( exit_code, 3 ) ;
5150+ }
5151+
5152+ #[ tokio:: test]
5153+ async fn hosted_instance_stop_fails_when_boundary_is_lost ( ) {
5154+ let error = stop_workload (
5155+ async { Ok ( BoundaryExitStatus :: Exited ( 0 ) ) } ,
5156+ || async { Err ( BackendError :: Unavailable ( "connection lost" . into ( ) ) ) } ,
5157+ None ,
5158+ )
5159+ . await
5160+ . expect_err ( "a hosted instance reports a stop it could not confirm" ) ;
5161+ assert ! (
5162+ error
5163+ . to_string( )
5164+ . contains( "sandbox did not acknowledge terminal state" ) ,
5165+ "{error}"
5166+ ) ;
5167+ }
5168+
5169+ #[ tokio:: test( start_paused = true ) ]
5170+ async fn hosted_instance_stop_is_not_bounded_by_shutdown_timeout ( ) {
5171+ let exit_code = stop_workload (
5172+ async {
5173+ tokio:: time:: sleep ( SHUTDOWN_TIMEOUT * 2 ) . await ;
5174+ Ok ( BoundaryExitStatus :: Exited ( 0 ) )
5175+ } ,
5176+ || async { Ok ( ( ) ) } ,
5177+ None ,
5178+ )
5179+ . await
5180+ . expect ( "hosted instance stops" ) ;
5181+ assert_eq ! ( exit_code, 0 ) ;
5182+ }
5183+
50745184 #[ test]
50755185 fn discover_policy_from_nonexistent_path_is_missing ( ) {
50765186 let path = std:: path:: Path :: new ( "/nonexistent/policy.yaml" ) ;
0 commit comments