Skip to content

Commit 86048c1

Browse files
committed
fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly
Add a oneshot notification channel to ProxyHandle that fires when the proxy task exits for any reason (panic, abort, or loop break). The sandbox main loop now races this signal alongside the entrypoint process and shutdown signals, terminating with a clear OCSF error if the proxy dies. This prevents the sandbox from continuing to report Ready with a dead proxy. Follows the existing sidecar control channel pattern. All five wait paths (process+sidecar, process-only, network+sidecar, network-only on Linux, and non-Linux) now monitor proxy liveness. Refs NVIDIA#2337 Signed-off-by: Sean Burdine <[email protected]> Signed-off-by: politerealism <[email protected]>
1 parent 88cf35e commit 86048c1

2 files changed

Lines changed: 113 additions & 7 deletions

File tree

  • crates

‎crates/openshell-sandbox/src/lib.rs‎

Lines changed: 98 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ mod sidecar_control;
1616

1717
use miette::{IntoDiagnostic, Result, WrapErr};
1818
use std::future::Future;
19+
use std::pin::Pin;
1920
use std::sync::Arc;
2021
#[cfg(target_os = "linux")]
2122
use std::sync::atomic::Ordering;
@@ -405,7 +406,7 @@ pub async fn run_sandbox(
405406
// API read the current value so proposals target the correct workspace.
406407
let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new());
407408

408-
let networking = if network_enabled {
409+
let mut networking = if network_enabled {
409410
#[cfg(target_os = "linux")]
410411
let proxy_bind_ip = netns
411412
.as_ref()
@@ -703,6 +704,18 @@ pub async fn run_sandbox(
703704
.zip(bootstrap.proxy_ca_bundle_path.clone())
704705
});
705706

707+
let proxy_exited: Pin<Box<dyn Future<Output = ()> + Send>> =
708+
if let Some(rx) = networking
709+
.as_mut()
710+
.and_then(|n| n.proxy.as_mut())
711+
.map(|p| p.take_exit_receiver())
712+
{
713+
Box::pin(async { let _ = rx.await; })
714+
} else {
715+
Box::pin(std::future::pending())
716+
};
717+
tokio::pin!(proxy_exited);
718+
706719
let exit_code = if process_enabled {
707720
let ca_file_paths = networking
708721
.as_ref()
@@ -784,9 +797,41 @@ pub async fn run_sandbox(
784797
"authoritative network-sidecar control channel closed"
785798
));
786799
}
800+
_ = &mut proxy_exited => {
801+
ocsf_emit!(
802+
AppLifecycleBuilder::new(ocsf_ctx())
803+
.activity(ActivityId::Fail)
804+
.severity(SeverityId::High)
805+
.status(StatusId::Failure)
806+
.message(
807+
"Proxy accept loop exited unexpectedly; terminating sandbox"
808+
)
809+
.build()
810+
);
811+
return Err(miette::miette!(
812+
"proxy accept loop exited unexpectedly"
813+
));
814+
}
787815
}
788816
} else {
789-
process.await?
817+
tokio::select! {
818+
result = process => result?,
819+
_ = &mut proxy_exited => {
820+
ocsf_emit!(
821+
AppLifecycleBuilder::new(ocsf_ctx())
822+
.activity(ActivityId::Fail)
823+
.severity(SeverityId::High)
824+
.status(StatusId::Failure)
825+
.message(
826+
"Proxy accept loop exited unexpectedly; terminating sandbox"
827+
)
828+
.build()
829+
);
830+
return Err(miette::miette!(
831+
"proxy accept loop exited unexpectedly"
832+
));
833+
}
834+
}
790835
}
791836
} else {
792837
// Network-only sidecar mode: keep the proxy and its background
@@ -802,15 +847,62 @@ pub async fn run_sandbox(
802847
warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar");
803848
1
804849
}
850+
_ = &mut proxy_exited => {
851+
ocsf_emit!(
852+
AppLifecycleBuilder::new(ocsf_ctx())
853+
.activity(ActivityId::Fail)
854+
.severity(SeverityId::High)
855+
.status(StatusId::Failure)
856+
.message(
857+
"Proxy accept loop exited unexpectedly; terminating sandbox"
858+
)
859+
.build()
860+
);
861+
return Err(miette::miette!(
862+
"proxy accept loop exited unexpectedly"
863+
));
864+
}
805865
}
806866
} else {
807-
wait_for_shutdown_signal().await;
808-
0
867+
tokio::select! {
868+
() = wait_for_shutdown_signal() => 0,
869+
_ = &mut proxy_exited => {
870+
ocsf_emit!(
871+
AppLifecycleBuilder::new(ocsf_ctx())
872+
.activity(ActivityId::Fail)
873+
.severity(SeverityId::High)
874+
.status(StatusId::Failure)
875+
.message(
876+
"Proxy accept loop exited unexpectedly; terminating sandbox"
877+
)
878+
.build()
879+
);
880+
return Err(miette::miette!(
881+
"proxy accept loop exited unexpectedly"
882+
));
883+
}
884+
}
809885
}
810886
#[cfg(not(target_os = "linux"))]
811887
{
812-
wait_for_shutdown_signal().await;
813-
0
888+
tokio::select! {
889+
() = wait_for_shutdown_signal() => 0,
890+
_ = &mut proxy_exited => {
891+
ocsf_emit!(
892+
AppLifecycleBuilder::new(ocsf_ctx())
893+
.activity(ActivityId::Fail)
894+
.severity(SeverityId::High)
895+
.status(StatusId::Failure)
896+
.message(
897+
"Proxy accept loop exited unexpectedly; terminating sandbox"
898+
)
899+
.build()
900+
);
901+
return Err(miette::miette!(
902+
"proxy accept loop exited unexpectedly"
903+
));
904+
}
905+
}
814906
}
815907
};
816908

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,11 +200,11 @@ impl InferenceContext {
200200
}
201201
}
202202

203-
#[derive(Debug)]
204203
pub struct ProxyHandle {
205204
#[allow(dead_code)]
206205
http_addr: Option<SocketAddr>,
207206
join: JoinHandle<()>,
207+
exited_rx: tokio::sync::oneshot::Receiver<()>,
208208
}
209209

210210
impl ProxyHandle {
@@ -309,7 +309,13 @@ impl ProxyHandle {
309309
ocsf_emit!(event);
310310
}
311311

312+
let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>();
312313
let join = tokio::spawn(async move {
314+
// Hold the sender for the lifetime of this task — when the task
315+
// exits (panic, abort, or loop break), the sender drops and the
316+
// receiver fires, notifying the sandbox that the proxy is gone.
317+
let _proxy_exit_guard = exited_tx;
318+
313319
// Wait for the OPA engine's symlink resolution reload to complete
314320
// before accepting connections. This prevents requests from
315321
// observing a generation transition mid-flight, which would cause
@@ -420,13 +426,21 @@ impl ProxyHandle {
420426
Ok(Self {
421427
http_addr: Some(local_addr),
422428
join,
429+
exited_rx,
423430
})
424431
}
425432

426433
#[allow(dead_code)]
427434
pub const fn http_addr(&self) -> Option<SocketAddr> {
428435
self.http_addr
429436
}
437+
438+
/// Take the exit notification receiver for health monitoring.
439+
/// Resolves when the proxy accept loop task exits for any reason.
440+
pub fn take_exit_receiver(&mut self) -> tokio::sync::oneshot::Receiver<()> {
441+
let (_tx, dummy_rx) = tokio::sync::oneshot::channel();
442+
std::mem::replace(&mut self.exited_rx, dummy_rx)
443+
}
430444
}
431445

432446
impl Drop for ProxyHandle {

0 commit comments

Comments
 (0)