@@ -93,6 +93,10 @@ mod linux {
9393 /// Bound the wait for a flush before acknowledging termination. A timed-out
9494 /// blocking task cannot be cancelled and may outlive the acknowledgement.
9595 const FILESYSTEM_FLUSH_TIMEOUT : Duration = Duration :: from_secs ( 3 ) ;
96+ /// Bound the process runtime's shutdown after the boundary stops serving.
97+ /// A blocking task such as a supervisor's agent `Wait` can outlive the
98+ /// termination that would end it, and the shutdown cannot cancel it.
99+ const PROCESS_RUNTIME_SHUTDOWN_TIMEOUT : Duration = Duration :: from_secs ( 2 ) ;
96100 const MAX_PENDING_HANDSHAKES : usize = 32 ;
97101 const MAX_CONTROL_CONNECTIONS : usize = 128 ;
98102 /// Bounds one runtime-initiated dial, including finding the supervisor
@@ -293,7 +297,17 @@ mod linux {
293297 launcher,
294298 qualification,
295299 ) ?) ;
296- serve ( & config. listener , runtime)
300+ serve_then_stop ( process_runtime, & config. listener , runtime)
301+ }
302+
303+ fn serve_then_stop (
304+ process_runtime : tokio:: runtime:: Runtime ,
305+ config : & BoundaryListenerConfig ,
306+ runtime : Arc < BoundaryRuntime > ,
307+ ) -> Result < ( ) , String > {
308+ let result = serve ( config, runtime) ;
309+ process_runtime. shutdown_timeout ( PROCESS_RUNTIME_SHUTDOWN_TIMEOUT ) ;
310+ result
297311 }
298312
299313 fn make_boundary_nondumpable ( ) -> Result < ( ) , String > {
@@ -6190,6 +6204,269 @@ mod linux {
61906204 ) ;
61916205 }
61926206
6207+ /// Send one logical request on its own exchange and return the reply.
6208+ async fn exchange_request (
6209+ channel : tonic:: transport:: Channel ,
6210+ handle : & str ,
6211+ request : Request ,
6212+ token : & str ,
6213+ ) -> Response {
6214+ let request = RequestEnvelope :: new ( request) . expect ( "encode request" ) ;
6215+ let frames = tokio_stream:: iter ( [
6216+ openshell_sandbox_backend:: isolation_protocol:: exchange_frame ( handle) ,
6217+ openshell_sandbox_backend:: isolation_protocol:: request_chunk ( & request)
6218+ . expect ( "typed logical request" ) ,
6219+ ] ) ;
6220+ let mut body = SandboxProtocolClient :: new ( channel)
6221+ . exchange ( bearer_request ( frames, token) )
6222+ . await
6223+ . expect ( "exchange logical request" )
6224+ . into_inner ( ) ;
6225+ let mut frame = Vec :: new ( ) ;
6226+ while let Some ( chunk) = body. message ( ) . await . expect ( "read gRPC response" ) {
6227+ frame. extend_from_slice ( & chunk_data ( chunk) . unwrap ( ) ) ;
6228+ }
6229+ openshell_sandbox_backend:: boundary_protocol:: decode_frame :: < ResponseEnvelope > ( & frame)
6230+ . expect ( "decode logical response" )
6231+ . response
6232+ }
6233+
6234+ /// A shared supervisor outlives the sandboxes it serves and keeps
6235+ /// their exchanges open, so SIGTERM must end the process without
6236+ /// waiting for the supervisor to close them.
6237+ #[ test]
6238+ fn sigterm_exits_while_the_supervisor_holds_an_exchange_open ( ) {
6239+ const CHILD_MARKER : & str = "OPENSHELL_TEST_BOUNDARY_SIGTERM_CHILD" ;
6240+ const READY : & str = "boundary-sigterm-test-ready" ;
6241+ const EXIT_BUDGET : Duration = Duration :: from_secs ( 10 ) ;
6242+ if std:: env:: var_os ( CHILD_MARKER ) . is_none ( ) {
6243+ use std:: io:: BufRead as _;
6244+ let mut child = std:: process:: Command :: new (
6245+ std:: env:: current_exe ( ) . expect ( "current test executable" ) ,
6246+ )
6247+ . args ( [
6248+ "--exact" ,
6249+ "boundary_server::linux::tests::sigterm_exits_while_the_supervisor_holds_an_exchange_open" ,
6250+ "--nocapture" ,
6251+ ] )
6252+ . env ( CHILD_MARKER , "1" )
6253+ . stdout ( std:: process:: Stdio :: piped ( ) )
6254+ . spawn ( )
6255+ . expect ( "run isolated termination test" ) ;
6256+ let mut lines =
6257+ io:: BufReader :: new ( child. stdout . take ( ) . expect ( "child stdout" ) ) . lines ( ) ;
6258+ assert ! (
6259+ lines. any( |line| line. is_ok_and( |line| line == READY ) ) ,
6260+ "the boundary never parked an exchange"
6261+ ) ;
6262+ std:: thread:: spawn ( move || lines. for_each ( drop) ) ;
6263+ nix:: sys:: signal:: kill (
6264+ nix:: unistd:: Pid :: from_raw ( i32:: try_from ( child. id ( ) ) . expect ( "child PID" ) ) ,
6265+ nix:: sys:: signal:: Signal :: SIGTERM ,
6266+ )
6267+ . expect ( "send SIGTERM" ) ;
6268+ let signalled = std:: time:: Instant :: now ( ) ;
6269+ let status = loop {
6270+ if let Some ( status) = child. try_wait ( ) . expect ( "poll boundary" ) {
6271+ break Some ( status) ;
6272+ }
6273+ if signalled. elapsed ( ) > EXIT_BUDGET {
6274+ break None ;
6275+ }
6276+ std:: thread:: sleep ( Duration :: from_millis ( 20 ) ) ;
6277+ } ;
6278+ let elapsed = signalled. elapsed ( ) ;
6279+ if status. is_none ( ) {
6280+ let _ = child. kill ( ) ;
6281+ let _ = child. wait ( ) ;
6282+ }
6283+ let status = status. unwrap_or_else ( || {
6284+ panic ! ( "the boundary was still running {elapsed:?} after SIGTERM" )
6285+ } ) ;
6286+ assert ! (
6287+ status. success( ) ,
6288+ "isolated termination test failed: {status}"
6289+ ) ;
6290+ return ;
6291+ }
6292+
6293+ let directory = tempfile:: tempdir ( ) . expect ( "temporary directory" ) ;
6294+ let ( server_tls, client_tls) = stage_test_tls ( directory. path ( ) , "sigterm" ) ;
6295+ let socket_path = directory. path ( ) . join ( "supervisor.sock" ) ;
6296+ let listener = BoundaryListenerConfig :: RuntimeInitiatedUnix {
6297+ socket_path : socket_path. clone ( ) ,
6298+ tls : server_tls,
6299+ } ;
6300+ let supervisor_runtime = tokio:: runtime:: Builder :: new_multi_thread ( )
6301+ . worker_threads ( 1 )
6302+ . enable_all ( )
6303+ . build ( )
6304+ . expect ( "supervisor runtime" ) ;
6305+ let supervisor_listener = {
6306+ let _entered = supervisor_runtime. enter ( ) ;
6307+ tokio:: net:: UnixListener :: bind ( & socket_path) . expect ( "bind supervisor socket" )
6308+ } ;
6309+ let process_runtime = tokio:: runtime:: Builder :: new_multi_thread ( )
6310+ . enable_all ( )
6311+ . build ( )
6312+ . expect ( "test process runtime" ) ;
6313+ let ( network_broker, workload_launcher) = test_network_broker ( ) ;
6314+ let ( verification_key, token) = test_auth_material ( "sigterm" ) ;
6315+ let boundary = {
6316+ let _entered = process_runtime. enter ( ) ;
6317+ Arc :: new (
6318+ BoundaryRuntime :: new (
6319+ BoundaryConfig {
6320+ boundary_id : "sigterm" . to_string ( ) ,
6321+ generation : "generation-1" . to_string ( ) ,
6322+ session_id : test_session_id ( ) ,
6323+ session_rotation : openshell_core:: jwt:: SessionRotation :: new ( 1 )
6324+ . expect ( "session rotation" ) ,
6325+ auth_epoch : CredentialEpoch :: new ( 1 ) . expect ( "auth epoch" ) ,
6326+ gateway_id : "test-gateway" . to_string ( ) ,
6327+ verification_keys : vec ! [ verification_key] ,
6328+ listener : listener. clone ( ) ,
6329+ resource_claims : std:: collections:: BTreeMap :: new ( ) ,
6330+ resource_claim_files : std:: collections:: BTreeMap :: new ( ) ,
6331+ workload_identity : test_workload_identity ( ) ,
6332+ provisioning_evidence : test_provisioning_evidence ( ) ,
6333+ child_env : std:: collections:: HashMap :: new ( ) ,
6334+ } ,
6335+ process_runtime. handle ( ) . clone ( ) ,
6336+ network_broker,
6337+ workload_launcher,
6338+ test_runtime_qualification ( ) ,
6339+ )
6340+ . expect ( "test boundary runtime" ) ,
6341+ )
6342+ } ;
6343+ let opening = test_open_request ( & boundary. config ) ;
6344+ std:: thread:: spawn ( move || {
6345+ let supervisor = std:: panic:: AssertUnwindSafe ( || {
6346+ supervisor_runtime. block_on ( async move {
6347+ let ( stream, _) = supervisor_listener. accept ( ) . await . expect ( "runtime dial" ) ;
6348+ let server_name =
6349+ rustls:: pki_types:: ServerName :: try_from ( client_tls. server_name . clone ( ) )
6350+ . expect ( "valid server name" ) ;
6351+ let stream = tokio_rustls:: TlsConnector :: from ( Arc :: new (
6352+ test_client_config ( & client_tls) ,
6353+ ) )
6354+ . connect ( server_name, stream)
6355+ . await
6356+ . expect ( "verify runtime TLS" ) ;
6357+ let mut stream = Some ( stream) ;
6358+ let channel = tonic:: transport:: Endpoint :: from_static (
6359+ "http://boundary.openshell.internal" ,
6360+ )
6361+ . connect_with_connector ( tower:: service_fn (
6362+ move |_: tonic:: transport:: Uri | {
6363+ let stream = stream. take ( ) ;
6364+ async move {
6365+ stream. map ( hyper_util:: rt:: TokioIo :: new) . ok_or_else ( || {
6366+ io:: Error :: other ( "the runtime connection is in use" )
6367+ } )
6368+ }
6369+ } ,
6370+ ) )
6371+ . await
6372+ . expect ( "gRPC over the runtime connection" ) ;
6373+ let handle = test_open_boundary ( channel. clone ( ) , opening, & token) . await ;
6374+ let attach = Request :: Attach {
6375+ supervisor_instance_id : test_supervisor_instance_id ( ) ,
6376+ policy : Box :: new ( takeover_test_policy ( ) ) ,
6377+ supervisor_takeover : None ,
6378+ supervisor_recovery_timeout_secs : None ,
6379+ } ;
6380+ assert ! ( matches!(
6381+ exchange_request( channel. clone( ) , & handle, attach, & token) . await ,
6382+ Response :: Attached { .. }
6383+ ) ) ;
6384+ assert ! ( matches!(
6385+ exchange_request( channel. clone( ) , & handle, Request :: Confirm , & token)
6386+ . await ,
6387+ Response :: Confirmed { .. }
6388+ ) ) ;
6389+ let start = Request :: StartAgent {
6390+ sandbox_id : "sigterm" . to_string ( ) ,
6391+ spec : AgentSpecWire {
6392+ program : "/bin/sleep" . to_string ( ) ,
6393+ args : vec ! [ "300" . to_string( ) ] ,
6394+ workdir : None ,
6395+ timeout_secs : 0 ,
6396+ interactive : false ,
6397+ otlp_relay_active : false ,
6398+ } ,
6399+ ca_cert : None ,
6400+ ca_bundle : None ,
6401+ provider_env_revision : 0 ,
6402+ provider_env : std:: collections:: HashMap :: new ( ) ,
6403+ provider_files : std:: collections:: HashMap :: new ( ) ,
6404+ } ;
6405+ let process_id =
6406+ match exchange_request ( channel. clone ( ) , & handle, start, & token) . await {
6407+ Response :: Started { process_id, .. } => process_id,
6408+ response => panic ! ( "the agent did not start: {response:?}" ) ,
6409+ } ;
6410+ // The supervisor waits for the agent and for workload
6411+ // connections for as long as the sandbox runs.
6412+ let mut parked = Vec :: new ( ) ;
6413+ for request in [ Request :: Wait { process_id } , Request :: AcceptNetwork ] {
6414+ let ( sender, receiver) = tokio:: sync:: mpsc:: channel ( 4 ) ;
6415+ sender
6416+ . send (
6417+ openshell_sandbox_backend:: isolation_protocol:: exchange_frame (
6418+ & handle,
6419+ ) ,
6420+ )
6421+ . await
6422+ . expect ( "bind exchange" ) ;
6423+ sender
6424+ . send (
6425+ openshell_sandbox_backend:: isolation_protocol:: request_chunk (
6426+ & RequestEnvelope :: new ( request) . expect ( "encode request" ) ,
6427+ )
6428+ . expect ( "typed request" ) ,
6429+ )
6430+ . await
6431+ . expect ( "send request" ) ;
6432+ let pending = SandboxProtocolClient :: new ( channel. clone ( ) )
6433+ . exchange ( bearer_request ( ReceiverStream :: new ( receiver) , & token) )
6434+ . await
6435+ . expect ( "open exchange" )
6436+ . into_inner ( ) ;
6437+ parked. push ( ( sender, pending) ) ;
6438+ }
6439+ for ( _, pending) in & mut parked {
6440+ assert ! (
6441+ tokio:: time:: timeout( Duration :: from_millis( 500 ) , pending. message( ) )
6442+ . await
6443+ . is_err( ) ,
6444+ "a parked exchange answered before termination"
6445+ ) ;
6446+ }
6447+ println ! ( "{READY}" ) ;
6448+ std:: future:: pending :: < ( ) > ( ) . await ;
6449+ drop ( parked) ;
6450+ } ) ;
6451+ } ) ;
6452+ if std:: panic:: catch_unwind ( supervisor) . is_err ( ) {
6453+ std:: process:: exit ( 1 ) ;
6454+ }
6455+ } ) ;
6456+
6457+ // The parked `Wait` blocks shutdown only when the shutdown cancels
6458+ // the agent's reaper before it records the exit, so a task that
6459+ // never ends stands in for that loss.
6460+ process_runtime. spawn_blocking ( || {
6461+ loop {
6462+ std:: thread:: park ( ) ;
6463+ }
6464+ } ) ;
6465+ install_boundary_signal_handlers ( ) . expect ( "install termination handlers" ) ;
6466+ serve_then_stop ( process_runtime, & listener, boundary)
6467+ . expect ( "terminate after supervisor confirmation" ) ;
6468+ }
6469+
61936470 #[ tokio:: test( flavor = "multi_thread" ) ]
61946471 async fn grpc_blackhole_expires_connection_and_releases_mediation_lease ( ) {
61956472 let ( runtime, token) = availability_test_runtime ( ) ;
0 commit comments