Skip to content

Commit 0037efe

Browse files
committed
fix(sandbox): bound runtime shutdown after SIGTERM
After a termination signal the boundary kills the workload and stops serving, then drops its process runtime. The drop waited without a limit for every blocking task. The supervisor keeps an agent `Wait` exchange open for the life of the sandbox, and that exchange blocks on a condition variable that only the agent's async reaper wakes. When the runtime shutdown cancelled the reaper before it recorded the exit, the `Wait` never returned and the process stayed alive until SIGKILL, so every delete or stop took the full termination grace period. Stop the process runtime with a two-second bound on every return path from `serve`. Workload kill and reaping are unchanged. Part of #5025 Signed-off-by: divesh <[email protected]>
1 parent dbc0517 commit 0037efe

1 file changed

Lines changed: 278 additions & 1 deletion

File tree

‎crates/openshell-sandbox/src/boundary_server.rs‎

Lines changed: 278 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ mod linux {
9393
/// Bound the wait for a flush before acknowledging termination. A timed-out
9494
/// blocking task cannot be cancelled and may outlive the acknowledgement.
9595
const FILESYSTEM_FLUSH_TIMEOUT: Duration = Duration::from_secs(3);
96+
/// Bound the process runtime's shutdown after the boundary stops serving.
97+
/// A blocking task such as a supervisor's agent `Wait` can outlive the
98+
/// termination that would end it, and the shutdown cannot cancel it.
99+
const PROCESS_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
96100
const MAX_PENDING_HANDSHAKES: usize = 32;
97101
const MAX_CONTROL_CONNECTIONS: usize = 128;
98102
/// Bounds one runtime-initiated dial, including finding the supervisor
@@ -293,7 +297,17 @@ mod linux {
293297
launcher,
294298
qualification,
295299
)?);
296-
serve(&config.listener, runtime)
300+
serve_then_stop(process_runtime, &config.listener, runtime)
301+
}
302+
303+
fn serve_then_stop(
304+
process_runtime: tokio::runtime::Runtime,
305+
config: &BoundaryListenerConfig,
306+
runtime: Arc<BoundaryRuntime>,
307+
) -> Result<(), String> {
308+
let result = serve(config, runtime);
309+
process_runtime.shutdown_timeout(PROCESS_RUNTIME_SHUTDOWN_TIMEOUT);
310+
result
297311
}
298312

299313
fn make_boundary_nondumpable() -> Result<(), String> {
@@ -6190,6 +6204,269 @@ mod linux {
61906204
);
61916205
}
61926206

6207+
/// Send one logical request on its own exchange and return the reply.
6208+
async fn exchange_request(
6209+
channel: tonic::transport::Channel,
6210+
handle: &str,
6211+
request: Request,
6212+
token: &str,
6213+
) -> Response {
6214+
let request = RequestEnvelope::new(request).expect("encode request");
6215+
let frames = tokio_stream::iter([
6216+
openshell_sandbox_backend::isolation_protocol::exchange_frame(handle),
6217+
openshell_sandbox_backend::isolation_protocol::request_chunk(&request)
6218+
.expect("typed logical request"),
6219+
]);
6220+
let mut body = SandboxProtocolClient::new(channel)
6221+
.exchange(bearer_request(frames, token))
6222+
.await
6223+
.expect("exchange logical request")
6224+
.into_inner();
6225+
let mut frame = Vec::new();
6226+
while let Some(chunk) = body.message().await.expect("read gRPC response") {
6227+
frame.extend_from_slice(&chunk_data(chunk).unwrap());
6228+
}
6229+
openshell_sandbox_backend::boundary_protocol::decode_frame::<ResponseEnvelope>(&frame)
6230+
.expect("decode logical response")
6231+
.response
6232+
}
6233+
6234+
/// A shared supervisor outlives the sandboxes it serves and keeps
6235+
/// their exchanges open, so SIGTERM must end the process without
6236+
/// waiting for the supervisor to close them.
6237+
#[test]
6238+
fn sigterm_exits_while_the_supervisor_holds_an_exchange_open() {
6239+
const CHILD_MARKER: &str = "OPENSHELL_TEST_BOUNDARY_SIGTERM_CHILD";
6240+
const READY: &str = "boundary-sigterm-test-ready";
6241+
const EXIT_BUDGET: Duration = Duration::from_secs(10);
6242+
if std::env::var_os(CHILD_MARKER).is_none() {
6243+
use std::io::BufRead as _;
6244+
let mut child = std::process::Command::new(
6245+
std::env::current_exe().expect("current test executable"),
6246+
)
6247+
.args([
6248+
"--exact",
6249+
"boundary_server::linux::tests::sigterm_exits_while_the_supervisor_holds_an_exchange_open",
6250+
"--nocapture",
6251+
])
6252+
.env(CHILD_MARKER, "1")
6253+
.stdout(std::process::Stdio::piped())
6254+
.spawn()
6255+
.expect("run isolated termination test");
6256+
let mut lines =
6257+
io::BufReader::new(child.stdout.take().expect("child stdout")).lines();
6258+
assert!(
6259+
lines.any(|line| line.is_ok_and(|line| line == READY)),
6260+
"the boundary never parked an exchange"
6261+
);
6262+
std::thread::spawn(move || lines.for_each(drop));
6263+
nix::sys::signal::kill(
6264+
nix::unistd::Pid::from_raw(i32::try_from(child.id()).expect("child PID")),
6265+
nix::sys::signal::Signal::SIGTERM,
6266+
)
6267+
.expect("send SIGTERM");
6268+
let signalled = std::time::Instant::now();
6269+
let status = loop {
6270+
if let Some(status) = child.try_wait().expect("poll boundary") {
6271+
break Some(status);
6272+
}
6273+
if signalled.elapsed() > EXIT_BUDGET {
6274+
break None;
6275+
}
6276+
std::thread::sleep(Duration::from_millis(20));
6277+
};
6278+
let elapsed = signalled.elapsed();
6279+
if status.is_none() {
6280+
let _ = child.kill();
6281+
let _ = child.wait();
6282+
}
6283+
let status = status.unwrap_or_else(|| {
6284+
panic!("the boundary was still running {elapsed:?} after SIGTERM")
6285+
});
6286+
assert!(
6287+
status.success(),
6288+
"isolated termination test failed: {status}"
6289+
);
6290+
return;
6291+
}
6292+
6293+
let directory = tempfile::tempdir().expect("temporary directory");
6294+
let (server_tls, client_tls) = stage_test_tls(directory.path(), "sigterm");
6295+
let socket_path = directory.path().join("supervisor.sock");
6296+
let listener = BoundaryListenerConfig::RuntimeInitiatedUnix {
6297+
socket_path: socket_path.clone(),
6298+
tls: server_tls,
6299+
};
6300+
let supervisor_runtime = tokio::runtime::Builder::new_multi_thread()
6301+
.worker_threads(1)
6302+
.enable_all()
6303+
.build()
6304+
.expect("supervisor runtime");
6305+
let supervisor_listener = {
6306+
let _entered = supervisor_runtime.enter();
6307+
tokio::net::UnixListener::bind(&socket_path).expect("bind supervisor socket")
6308+
};
6309+
let process_runtime = tokio::runtime::Builder::new_multi_thread()
6310+
.enable_all()
6311+
.build()
6312+
.expect("test process runtime");
6313+
let (network_broker, workload_launcher) = test_network_broker();
6314+
let (verification_key, token) = test_auth_material("sigterm");
6315+
let boundary = {
6316+
let _entered = process_runtime.enter();
6317+
Arc::new(
6318+
BoundaryRuntime::new(
6319+
BoundaryConfig {
6320+
boundary_id: "sigterm".to_string(),
6321+
generation: "generation-1".to_string(),
6322+
session_id: test_session_id(),
6323+
session_rotation: openshell_core::jwt::SessionRotation::new(1)
6324+
.expect("session rotation"),
6325+
auth_epoch: CredentialEpoch::new(1).expect("auth epoch"),
6326+
gateway_id: "test-gateway".to_string(),
6327+
verification_keys: vec![verification_key],
6328+
listener: listener.clone(),
6329+
resource_claims: std::collections::BTreeMap::new(),
6330+
resource_claim_files: std::collections::BTreeMap::new(),
6331+
workload_identity: test_workload_identity(),
6332+
provisioning_evidence: test_provisioning_evidence(),
6333+
child_env: std::collections::HashMap::new(),
6334+
},
6335+
process_runtime.handle().clone(),
6336+
network_broker,
6337+
workload_launcher,
6338+
test_runtime_qualification(),
6339+
)
6340+
.expect("test boundary runtime"),
6341+
)
6342+
};
6343+
let opening = test_open_request(&boundary.config);
6344+
std::thread::spawn(move || {
6345+
let supervisor = std::panic::AssertUnwindSafe(|| {
6346+
supervisor_runtime.block_on(async move {
6347+
let (stream, _) = supervisor_listener.accept().await.expect("runtime dial");
6348+
let server_name =
6349+
rustls::pki_types::ServerName::try_from(client_tls.server_name.clone())
6350+
.expect("valid server name");
6351+
let stream = tokio_rustls::TlsConnector::from(Arc::new(
6352+
test_client_config(&client_tls),
6353+
))
6354+
.connect(server_name, stream)
6355+
.await
6356+
.expect("verify runtime TLS");
6357+
let mut stream = Some(stream);
6358+
let channel = tonic::transport::Endpoint::from_static(
6359+
"http://boundary.openshell.internal",
6360+
)
6361+
.connect_with_connector(tower::service_fn(
6362+
move |_: tonic::transport::Uri| {
6363+
let stream = stream.take();
6364+
async move {
6365+
stream.map(hyper_util::rt::TokioIo::new).ok_or_else(|| {
6366+
io::Error::other("the runtime connection is in use")
6367+
})
6368+
}
6369+
},
6370+
))
6371+
.await
6372+
.expect("gRPC over the runtime connection");
6373+
let handle = test_open_boundary(channel.clone(), opening, &token).await;
6374+
let attach = Request::Attach {
6375+
supervisor_instance_id: test_supervisor_instance_id(),
6376+
policy: Box::new(takeover_test_policy()),
6377+
supervisor_takeover: None,
6378+
supervisor_recovery_timeout_secs: None,
6379+
};
6380+
assert!(matches!(
6381+
exchange_request(channel.clone(), &handle, attach, &token).await,
6382+
Response::Attached { .. }
6383+
));
6384+
assert!(matches!(
6385+
exchange_request(channel.clone(), &handle, Request::Confirm, &token)
6386+
.await,
6387+
Response::Confirmed { .. }
6388+
));
6389+
let start = Request::StartAgent {
6390+
sandbox_id: "sigterm".to_string(),
6391+
spec: AgentSpecWire {
6392+
program: "/bin/sleep".to_string(),
6393+
args: vec!["300".to_string()],
6394+
workdir: None,
6395+
timeout_secs: 0,
6396+
interactive: false,
6397+
otlp_relay_active: false,
6398+
},
6399+
ca_cert: None,
6400+
ca_bundle: None,
6401+
provider_env_revision: 0,
6402+
provider_env: std::collections::HashMap::new(),
6403+
provider_files: std::collections::HashMap::new(),
6404+
};
6405+
let process_id =
6406+
match exchange_request(channel.clone(), &handle, start, &token).await {
6407+
Response::Started { process_id, .. } => process_id,
6408+
response => panic!("the agent did not start: {response:?}"),
6409+
};
6410+
// The supervisor waits for the agent and for workload
6411+
// connections for as long as the sandbox runs.
6412+
let mut parked = Vec::new();
6413+
for request in [Request::Wait { process_id }, Request::AcceptNetwork] {
6414+
let (sender, receiver) = tokio::sync::mpsc::channel(4);
6415+
sender
6416+
.send(
6417+
openshell_sandbox_backend::isolation_protocol::exchange_frame(
6418+
&handle,
6419+
),
6420+
)
6421+
.await
6422+
.expect("bind exchange");
6423+
sender
6424+
.send(
6425+
openshell_sandbox_backend::isolation_protocol::request_chunk(
6426+
&RequestEnvelope::new(request).expect("encode request"),
6427+
)
6428+
.expect("typed request"),
6429+
)
6430+
.await
6431+
.expect("send request");
6432+
let pending = SandboxProtocolClient::new(channel.clone())
6433+
.exchange(bearer_request(ReceiverStream::new(receiver), &token))
6434+
.await
6435+
.expect("open exchange")
6436+
.into_inner();
6437+
parked.push((sender, pending));
6438+
}
6439+
for (_, pending) in &mut parked {
6440+
assert!(
6441+
tokio::time::timeout(Duration::from_millis(500), pending.message())
6442+
.await
6443+
.is_err(),
6444+
"a parked exchange answered before termination"
6445+
);
6446+
}
6447+
println!("{READY}");
6448+
std::future::pending::<()>().await;
6449+
drop(parked);
6450+
});
6451+
});
6452+
if std::panic::catch_unwind(supervisor).is_err() {
6453+
std::process::exit(1);
6454+
}
6455+
});
6456+
6457+
// The parked `Wait` blocks shutdown only when the shutdown cancels
6458+
// the agent's reaper before it records the exit, so a task that
6459+
// never ends stands in for that loss.
6460+
process_runtime.spawn_blocking(|| {
6461+
loop {
6462+
std::thread::park();
6463+
}
6464+
});
6465+
install_boundary_signal_handlers().expect("install termination handlers");
6466+
serve_then_stop(process_runtime, &listener, boundary)
6467+
.expect("terminate after supervisor confirmation");
6468+
}
6469+
61936470
#[tokio::test(flavor = "multi_thread")]
61946471
async fn grpc_blackhole_expires_connection_and_releases_mediation_lease() {
61956472
let (runtime, token) = availability_test_runtime();

0 commit comments

Comments
 (0)