Skip to content

Commit e9ba8d9

Browse files
authored
fix(sandbox): bound runtime shutdown after SIGTERM (NVIDIA#5279)
* fix(sandbox): bound runtime shutdown after SIGTERM After a termination signal the boundary kills the workload and stops serving, then drops its process runtime. The drop waited without a limit for every blocking task. The supervisor keeps an agent `Wait` exchange open for the life of the sandbox, and that exchange blocks on a condition variable that only the agent's async reaper wakes. When the runtime shutdown cancelled the reaper before it recorded the exit, the `Wait` never returned and the process stayed alive until SIGKILL, so every delete or stop took the full termination grace period. Stop the process runtime with a two-second bound on every return path from `serve`. Workload kill and reaping are unchanged. Part of NVIDIA#5025 Signed-off-by: divesh <[email protected]> * test(sandbox): bound the whole SIGTERM exit test Run the child through run_isolated so a stall during setup fails the test instead of hanging it, and have the fake supervisor send SIGTERM once its exchanges are parked. Signed-off-by: divesh <[email protected]> --------- Signed-off-by: divesh <[email protected]>
1 parent 20357a9 commit e9ba8d9

1 file changed

Lines changed: 251 additions & 1 deletion

File tree

‎crates/openshell-sandbox/src/boundary_server.rs‎

Lines changed: 251 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,10 @@ mod linux {
9797
/// Bound the wait for a flush before acknowledging termination. A timed-out
9898
/// blocking task cannot be cancelled and may outlive the acknowledgement.
9999
const FILESYSTEM_FLUSH_TIMEOUT: Duration = Duration::from_secs(3);
100+
/// Bound the process runtime's shutdown after the boundary stops serving.
101+
/// A blocking task such as a supervisor's agent `Wait` can outlive the
102+
/// termination that would end it, and the shutdown cannot cancel it.
103+
const PROCESS_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
100104
const MAX_PENDING_HANDSHAKES: usize = 32;
101105
const MAX_CONTROL_CONNECTIONS: usize = 128;
102106
/// Bounds one runtime-initiated dial, including finding the supervisor
@@ -297,7 +301,17 @@ mod linux {
297301
launcher,
298302
qualification,
299303
)?);
300-
serve(&config.listener, runtime)
304+
serve_then_stop(process_runtime, &config.listener, runtime)
305+
}
306+
307+
fn serve_then_stop(
308+
process_runtime: tokio::runtime::Runtime,
309+
config: &BoundaryListenerConfig,
310+
runtime: Arc<BoundaryRuntime>,
311+
) -> Result<(), String> {
312+
let result = serve(config, runtime);
313+
process_runtime.shutdown_timeout(PROCESS_RUNTIME_SHUTDOWN_TIMEOUT);
314+
result
301315
}
302316

303317
fn make_boundary_nondumpable() -> Result<(), String> {
@@ -6232,6 +6246,242 @@ mod linux {
62326246
);
62336247
}
62346248

6249+
/// Send one logical request on its own exchange and return the reply.
6250+
async fn exchange_request(
6251+
channel: tonic::transport::Channel,
6252+
handle: &str,
6253+
request: Request,
6254+
token: &str,
6255+
) -> Response {
6256+
let request = RequestEnvelope::new(request).expect("encode request");
6257+
let frames = tokio_stream::iter([
6258+
openshell_sandbox_backend::isolation_protocol::exchange_frame(handle),
6259+
openshell_sandbox_backend::isolation_protocol::request_chunk(&request)
6260+
.expect("typed logical request"),
6261+
]);
6262+
let mut body = SandboxProtocolClient::new(channel)
6263+
.exchange(bearer_request(frames, token))
6264+
.await
6265+
.expect("exchange logical request")
6266+
.into_inner();
6267+
let mut frame = Vec::new();
6268+
while let Some(chunk) = body.message().await.expect("read gRPC response") {
6269+
frame.extend_from_slice(&chunk_data(chunk).unwrap());
6270+
}
6271+
openshell_sandbox_backend::boundary_protocol::decode_frame::<ResponseEnvelope>(&frame)
6272+
.expect("decode logical response")
6273+
.response
6274+
}
6275+
6276+
/// A shared supervisor outlives the sandboxes it serves and keeps
6277+
/// their exchanges open, so SIGTERM must end the process without
6278+
/// waiting for the supervisor to close them.
6279+
#[test]
6280+
fn sigterm_exits_while_the_supervisor_holds_an_exchange_open() {
6281+
const CHILD_MARKER: &str = "OPENSHELL_TEST_BOUNDARY_SIGTERM_CHILD";
6282+
const EXIT_BUDGET: Duration = Duration::from_secs(10);
6283+
if std::env::var_os(CHILD_MARKER).is_none() {
6284+
run_isolated(
6285+
"boundary_server::linux::tests::sigterm_exits_while_the_supervisor_holds_an_exchange_open",
6286+
CHILD_MARKER,
6287+
Duration::from_mins(1),
6288+
);
6289+
return;
6290+
}
6291+
6292+
let directory = tempfile::tempdir().expect("temporary directory");
6293+
let (server_tls, client_tls) = stage_test_tls(directory.path(), "sigterm");
6294+
let socket_path = directory.path().join("supervisor.sock");
6295+
let listener = BoundaryListenerConfig::RuntimeInitiatedUnix {
6296+
socket_path: socket_path.clone(),
6297+
tls: server_tls,
6298+
};
6299+
let supervisor_runtime = tokio::runtime::Builder::new_multi_thread()
6300+
.worker_threads(1)
6301+
.enable_all()
6302+
.build()
6303+
.expect("supervisor runtime");
6304+
let supervisor_listener = {
6305+
let _entered = supervisor_runtime.enter();
6306+
tokio::net::UnixListener::bind(&socket_path).expect("bind supervisor socket")
6307+
};
6308+
let process_runtime = tokio::runtime::Builder::new_multi_thread()
6309+
.enable_all()
6310+
.build()
6311+
.expect("test process runtime");
6312+
let (network_broker, workload_launcher) = test_network_broker();
6313+
let (verification_key, token) = test_auth_material("sigterm");
6314+
let boundary = {
6315+
let _entered = process_runtime.enter();
6316+
Arc::new(
6317+
BoundaryRuntime::new(
6318+
BoundaryConfig {
6319+
boundary_id: "sigterm".to_string(),
6320+
generation: "generation-1".to_string(),
6321+
session_id: test_session_id(),
6322+
session_rotation: openshell_core::jwt::SessionRotation::new(1)
6323+
.expect("session rotation"),
6324+
auth_epoch: CredentialEpoch::new(1).expect("auth epoch"),
6325+
gateway_id: "test-gateway".to_string(),
6326+
verification_keys: vec![verification_key],
6327+
listener: listener.clone(),
6328+
resource_claims: std::collections::BTreeMap::new(),
6329+
resource_claim_files: std::collections::BTreeMap::new(),
6330+
workload_identity: test_workload_identity(),
6331+
provisioning_evidence: test_provisioning_evidence(),
6332+
child_env: std::collections::HashMap::new(),
6333+
},
6334+
process_runtime.handle().clone(),
6335+
network_broker,
6336+
workload_launcher,
6337+
test_runtime_qualification(),
6338+
)
6339+
.expect("test boundary runtime"),
6340+
)
6341+
};
6342+
let opening = test_open_request(&boundary.config);
6343+
let (signalled_sender, signalled) = std::sync::mpsc::channel();
6344+
std::thread::spawn(move || {
6345+
let supervisor = std::panic::AssertUnwindSafe(|| {
6346+
supervisor_runtime.block_on(async move {
6347+
let (stream, _) = supervisor_listener.accept().await.expect("runtime dial");
6348+
let server_name =
6349+
rustls::pki_types::ServerName::try_from(client_tls.server_name.clone())
6350+
.expect("valid server name");
6351+
let stream = tokio_rustls::TlsConnector::from(Arc::new(
6352+
test_client_config(&client_tls),
6353+
))
6354+
.connect(server_name, stream)
6355+
.await
6356+
.expect("verify runtime TLS");
6357+
let mut stream = Some(stream);
6358+
let channel = tonic::transport::Endpoint::from_static(
6359+
"http://boundary.openshell.internal",
6360+
)
6361+
.connect_with_connector(tower::service_fn(
6362+
move |_: tonic::transport::Uri| {
6363+
let stream = stream.take();
6364+
async move {
6365+
stream.map(hyper_util::rt::TokioIo::new).ok_or_else(|| {
6366+
io::Error::other("the runtime connection is in use")
6367+
})
6368+
}
6369+
},
6370+
))
6371+
.await
6372+
.expect("gRPC over the runtime connection");
6373+
let handle = test_open_boundary(channel.clone(), opening, &token).await;
6374+
let attach = Request::Attach {
6375+
supervisor_instance_id: test_supervisor_instance_id(),
6376+
policy: Box::new(takeover_test_policy()),
6377+
supervisor_takeover: None,
6378+
supervisor_recovery_timeout_secs: None,
6379+
};
6380+
assert!(matches!(
6381+
exchange_request(channel.clone(), &handle, attach, &token).await,
6382+
Response::Attached { .. }
6383+
));
6384+
assert!(matches!(
6385+
exchange_request(channel.clone(), &handle, Request::Confirm, &token)
6386+
.await,
6387+
Response::Confirmed { .. }
6388+
));
6389+
let start = Request::StartAgent {
6390+
sandbox_id: "sigterm".to_string(),
6391+
spec: AgentSpecWire {
6392+
program: "/bin/sleep".to_string(),
6393+
args: vec!["300".to_string()],
6394+
workdir: None,
6395+
timeout_secs: 0,
6396+
interactive: false,
6397+
otlp_relay_active: false,
6398+
},
6399+
ca_cert: None,
6400+
ca_bundle: None,
6401+
provider_env_revision: 0,
6402+
provider_env: std::collections::HashMap::new(),
6403+
provider_files: std::collections::HashMap::new(),
6404+
};
6405+
let process_id =
6406+
match exchange_request(channel.clone(), &handle, start, &token).await {
6407+
Response::Started { process_id, .. } => process_id,
6408+
response => panic!("the agent did not start: {response:?}"),
6409+
};
6410+
// The supervisor waits for the agent and for workload
6411+
// connections for as long as the sandbox runs.
6412+
let mut parked = Vec::new();
6413+
for request in [Request::Wait { process_id }, Request::AcceptNetwork] {
6414+
let (sender, receiver) = tokio::sync::mpsc::channel(4);
6415+
sender
6416+
.send(
6417+
openshell_sandbox_backend::isolation_protocol::exchange_frame(
6418+
&handle,
6419+
),
6420+
)
6421+
.await
6422+
.expect("bind exchange");
6423+
sender
6424+
.send(
6425+
openshell_sandbox_backend::isolation_protocol::request_chunk(
6426+
&RequestEnvelope::new(request).expect("encode request"),
6427+
)
6428+
.expect("typed request"),
6429+
)
6430+
.await
6431+
.expect("send request");
6432+
let pending = SandboxProtocolClient::new(channel.clone())
6433+
.exchange(bearer_request(ReceiverStream::new(receiver), &token))
6434+
.await
6435+
.expect("open exchange")
6436+
.into_inner();
6437+
parked.push((sender, pending));
6438+
}
6439+
for (_, pending) in &mut parked {
6440+
assert!(
6441+
tokio::time::timeout(Duration::from_millis(500), pending.message())
6442+
.await
6443+
.is_err(),
6444+
"a parked exchange answered before termination"
6445+
);
6446+
}
6447+
signalled_sender
6448+
.send(std::time::Instant::now())
6449+
.expect("report SIGTERM time");
6450+
nix::sys::signal::kill(
6451+
nix::unistd::Pid::this(),
6452+
nix::sys::signal::Signal::SIGTERM,
6453+
)
6454+
.expect("send SIGTERM");
6455+
std::future::pending::<()>().await;
6456+
drop(parked);
6457+
});
6458+
});
6459+
if std::panic::catch_unwind(supervisor).is_err() {
6460+
std::process::exit(1);
6461+
}
6462+
});
6463+
6464+
// The parked `Wait` blocks shutdown only when the shutdown cancels
6465+
// the agent's reaper before it records the exit, so a task that
6466+
// never ends stands in for that loss.
6467+
process_runtime.spawn_blocking(|| {
6468+
loop {
6469+
std::thread::park();
6470+
}
6471+
});
6472+
install_boundary_signal_handlers().expect("install termination handlers");
6473+
serve_then_stop(process_runtime, &listener, boundary)
6474+
.expect("terminate after supervisor confirmation");
6475+
let elapsed = signalled
6476+
.recv()
6477+
.expect("the supervisor sent SIGTERM")
6478+
.elapsed();
6479+
assert!(
6480+
elapsed < EXIT_BUDGET,
6481+
"the boundary took {elapsed:?} to stop after SIGTERM"
6482+
);
6483+
}
6484+
62356485
#[tokio::test(flavor = "multi_thread")]
62366486
async fn grpc_blackhole_expires_connection_and_releases_mediation_lease() {
62376487
let (runtime, token) = availability_test_runtime();

0 commit comments

Comments
 (0)