Skip to content

Commit 4894930

Browse files
committed
fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly
Add a oneshot notification channel to ProxyHandle that fires when the proxy task exits for any reason (panic, abort, or loop break). The sandbox main loop now races this signal alongside the entrypoint process and shutdown signals, terminating with a clear OCSF error if the proxy dies. This prevents the sandbox from continuing to report Ready with a dead proxy. Follows the existing sidecar control channel pattern. All five wait paths (process+sidecar, process-only, network+sidecar, network-only on Linux, and non-Linux) now monitor proxy liveness. Refs NVIDIA#2337 Signed-off-by: politerealism <[email protected]>
1 parent fb6610d commit 4894930

2 files changed

Lines changed: 113 additions & 7 deletions

File tree

  • crates

‎crates/openshell-sandbox/src/lib.rs‎

Lines changed: 98 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ mod sidecar_control;
1616

1717
use miette::{IntoDiagnostic, Result, WrapErr};
1818
use std::future::Future;
19+
use std::pin::Pin;
1920
use std::sync::Arc;
2021
#[cfg(target_os = "linux")]
2122
use std::sync::atomic::Ordering;
@@ -514,7 +515,7 @@ pub async fn run_sandbox(
514515
// API read the current value so proposals target the correct workspace.
515516
let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new());
516517

517-
let networking = if network_enabled {
518+
let mut networking = if network_enabled {
518519
#[cfg(target_os = "linux")]
519520
let proxy_bind_ip = netns
520521
.as_ref()
@@ -823,6 +824,18 @@ pub async fn run_sandbox(
823824
.zip(bootstrap.proxy_ca_bundle_path.clone())
824825
});
825826

827+
let proxy_exited: Pin<Box<dyn Future<Output = ()> + Send>> =
828+
if let Some(rx) = networking
829+
.as_mut()
830+
.and_then(|n| n.proxy.as_mut())
831+
.map(|p| p.take_exit_receiver())
832+
{
833+
Box::pin(async { let _ = rx.await; })
834+
} else {
835+
Box::pin(std::future::pending())
836+
};
837+
tokio::pin!(proxy_exited);
838+
826839
let exit_code = if process_enabled {
827840
let ca_file_paths = networking
828841
.as_ref()
@@ -935,9 +948,41 @@ pub async fn run_sandbox(
935948
"authoritative network-sidecar control channel closed"
936949
));
937950
}
951+
_ = &mut proxy_exited => {
952+
ocsf_emit!(
953+
AppLifecycleBuilder::new(ocsf_ctx())
954+
.activity(ActivityId::Fail)
955+
.severity(SeverityId::High)
956+
.status(StatusId::Failure)
957+
.message(
958+
"Proxy accept loop exited unexpectedly; terminating sandbox"
959+
)
960+
.build()
961+
);
962+
return Err(miette::miette!(
963+
"proxy accept loop exited unexpectedly"
964+
));
965+
}
938966
}
939967
} else {
940-
process.await?
968+
tokio::select! {
969+
result = process => result?,
970+
_ = &mut proxy_exited => {
971+
ocsf_emit!(
972+
AppLifecycleBuilder::new(ocsf_ctx())
973+
.activity(ActivityId::Fail)
974+
.severity(SeverityId::High)
975+
.status(StatusId::Failure)
976+
.message(
977+
"Proxy accept loop exited unexpectedly; terminating sandbox"
978+
)
979+
.build()
980+
);
981+
return Err(miette::miette!(
982+
"proxy accept loop exited unexpectedly"
983+
));
984+
}
985+
}
941986
}
942987
} else {
943988
// Network-only sidecar mode: keep the proxy and its background
@@ -953,15 +998,62 @@ pub async fn run_sandbox(
953998
warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar");
954999
1
9551000
}
1001+
_ = &mut proxy_exited => {
1002+
ocsf_emit!(
1003+
AppLifecycleBuilder::new(ocsf_ctx())
1004+
.activity(ActivityId::Fail)
1005+
.severity(SeverityId::High)
1006+
.status(StatusId::Failure)
1007+
.message(
1008+
"Proxy accept loop exited unexpectedly; terminating sandbox"
1009+
)
1010+
.build()
1011+
);
1012+
return Err(miette::miette!(
1013+
"proxy accept loop exited unexpectedly"
1014+
));
1015+
}
9561016
}
9571017
} else {
958-
wait_for_shutdown_signal().await;
959-
0
1018+
tokio::select! {
1019+
() = wait_for_shutdown_signal() => 0,
1020+
_ = &mut proxy_exited => {
1021+
ocsf_emit!(
1022+
AppLifecycleBuilder::new(ocsf_ctx())
1023+
.activity(ActivityId::Fail)
1024+
.severity(SeverityId::High)
1025+
.status(StatusId::Failure)
1026+
.message(
1027+
"Proxy accept loop exited unexpectedly; terminating sandbox"
1028+
)
1029+
.build()
1030+
);
1031+
return Err(miette::miette!(
1032+
"proxy accept loop exited unexpectedly"
1033+
));
1034+
}
1035+
}
9601036
}
9611037
#[cfg(not(target_os = "linux"))]
9621038
{
963-
wait_for_shutdown_signal().await;
964-
0
1039+
tokio::select! {
1040+
() = wait_for_shutdown_signal() => 0,
1041+
_ = &mut proxy_exited => {
1042+
ocsf_emit!(
1043+
AppLifecycleBuilder::new(ocsf_ctx())
1044+
.activity(ActivityId::Fail)
1045+
.severity(SeverityId::High)
1046+
.status(StatusId::Failure)
1047+
.message(
1048+
"Proxy accept loop exited unexpectedly; terminating sandbox"
1049+
)
1050+
.build()
1051+
);
1052+
return Err(miette::miette!(
1053+
"proxy accept loop exited unexpectedly"
1054+
));
1055+
}
1056+
}
9651057
}
9661058
};
9671059

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,11 +228,11 @@ impl InferenceContext {
228228
}
229229
}
230230

231-
#[derive(Debug)]
232231
pub struct ProxyHandle {
233232
#[allow(dead_code)]
234233
http_addr: Option<SocketAddr>,
235234
join: JoinHandle<()>,
235+
exited_rx: tokio::sync::oneshot::Receiver<()>,
236236
}
237237

238238
impl ProxyHandle {
@@ -337,7 +337,13 @@ impl ProxyHandle {
337337
ocsf_emit!(event);
338338
}
339339

340+
let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>();
340341
let join = tokio::spawn(async move {
342+
// Hold the sender for the lifetime of this task — when the task
343+
// exits (panic, abort, or loop break), the sender drops and the
344+
// receiver fires, notifying the sandbox that the proxy is gone.
345+
let _proxy_exit_guard = exited_tx;
346+
341347
// Wait for the OPA engine's symlink resolution reload to complete
342348
// before accepting connections. This prevents requests from
343349
// observing a generation transition mid-flight, which would cause
@@ -448,13 +454,21 @@ impl ProxyHandle {
448454
Ok(Self {
449455
http_addr: Some(local_addr),
450456
join,
457+
exited_rx,
451458
})
452459
}
453460

454461
#[allow(dead_code)]
455462
pub const fn http_addr(&self) -> Option<SocketAddr> {
456463
self.http_addr
457464
}
465+
466+
/// Take the exit notification receiver for health monitoring.
467+
/// Resolves when the proxy accept loop task exits for any reason.
468+
pub fn take_exit_receiver(&mut self) -> tokio::sync::oneshot::Receiver<()> {
469+
let (_tx, dummy_rx) = tokio::sync::oneshot::channel();
470+
std::mem::replace(&mut self.exited_rx, dummy_rx)
471+
}
458472
}
459473

460474
impl Drop for ProxyHandle {

0 commit comments

Comments
 (0)