Skip to content

Commit 1f2e030

Browse files
committed
fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly
Add a oneshot notification channel to ProxyHandle that fires when the proxy task exits for any reason (panic, abort, or loop break). The sandbox main loop now races this signal alongside the entrypoint process and shutdown signals, terminating with a clear OCSF error if the proxy dies. This prevents the sandbox from continuing to report Ready with a dead proxy. Follows the existing sidecar control channel pattern. All five wait paths (process+sidecar, process-only, network+sidecar, network-only on Linux, and non-Linux) now monitor proxy liveness. Refs NVIDIA#2337 Signed-off-by: Sean Burdine <[email protected]> Signed-off-by: politerealism <[email protected]>
1 parent 0a3ec7a commit 1f2e030

2 files changed

Lines changed: 113 additions & 7 deletions

File tree

  • crates

‎crates/openshell-sandbox/src/lib.rs‎

Lines changed: 98 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ mod sidecar_control;
1616

1717
use miette::{IntoDiagnostic, Result, WrapErr};
1818
use std::future::Future;
19+
use std::pin::Pin;
1920
use std::sync::Arc;
2021
#[cfg(target_os = "linux")]
2122
use std::sync::atomic::Ordering;
@@ -352,7 +353,7 @@ pub async fn run_sandbox(
352353
// API read the current value so proposals target the correct workspace.
353354
let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new());
354355

355-
let networking = if network_enabled {
356+
let mut networking = if network_enabled {
356357
#[cfg(target_os = "linux")]
357358
let proxy_bind_ip = netns
358359
.as_ref()
@@ -647,6 +648,18 @@ pub async fn run_sandbox(
647648
.zip(bootstrap.proxy_ca_bundle_path.clone())
648649
});
649650

651+
let proxy_exited: Pin<Box<dyn Future<Output = ()> + Send>> =
652+
if let Some(rx) = networking
653+
.as_mut()
654+
.and_then(|n| n.proxy.as_mut())
655+
.map(|p| p.take_exit_receiver())
656+
{
657+
Box::pin(async { let _ = rx.await; })
658+
} else {
659+
Box::pin(std::future::pending())
660+
};
661+
tokio::pin!(proxy_exited);
662+
650663
let exit_code = if process_enabled {
651664
let ca_file_paths = networking
652665
.as_ref()
@@ -728,9 +741,41 @@ pub async fn run_sandbox(
728741
"authoritative network-sidecar control channel closed"
729742
));
730743
}
744+
_ = &mut proxy_exited => {
745+
ocsf_emit!(
746+
AppLifecycleBuilder::new(ocsf_ctx())
747+
.activity(ActivityId::Fail)
748+
.severity(SeverityId::High)
749+
.status(StatusId::Failure)
750+
.message(
751+
"Proxy accept loop exited unexpectedly; terminating sandbox"
752+
)
753+
.build()
754+
);
755+
return Err(miette::miette!(
756+
"proxy accept loop exited unexpectedly"
757+
));
758+
}
731759
}
732760
} else {
733-
process.await?
761+
tokio::select! {
762+
result = process => result?,
763+
_ = &mut proxy_exited => {
764+
ocsf_emit!(
765+
AppLifecycleBuilder::new(ocsf_ctx())
766+
.activity(ActivityId::Fail)
767+
.severity(SeverityId::High)
768+
.status(StatusId::Failure)
769+
.message(
770+
"Proxy accept loop exited unexpectedly; terminating sandbox"
771+
)
772+
.build()
773+
);
774+
return Err(miette::miette!(
775+
"proxy accept loop exited unexpectedly"
776+
));
777+
}
778+
}
734779
}
735780
} else {
736781
// Network-only sidecar mode: keep the proxy and its background
@@ -746,15 +791,62 @@ pub async fn run_sandbox(
746791
warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar");
747792
1
748793
}
794+
_ = &mut proxy_exited => {
795+
ocsf_emit!(
796+
AppLifecycleBuilder::new(ocsf_ctx())
797+
.activity(ActivityId::Fail)
798+
.severity(SeverityId::High)
799+
.status(StatusId::Failure)
800+
.message(
801+
"Proxy accept loop exited unexpectedly; terminating sandbox"
802+
)
803+
.build()
804+
);
805+
return Err(miette::miette!(
806+
"proxy accept loop exited unexpectedly"
807+
));
808+
}
749809
}
750810
} else {
751-
wait_for_shutdown_signal().await;
752-
0
811+
tokio::select! {
812+
() = wait_for_shutdown_signal() => 0,
813+
_ = &mut proxy_exited => {
814+
ocsf_emit!(
815+
AppLifecycleBuilder::new(ocsf_ctx())
816+
.activity(ActivityId::Fail)
817+
.severity(SeverityId::High)
818+
.status(StatusId::Failure)
819+
.message(
820+
"Proxy accept loop exited unexpectedly; terminating sandbox"
821+
)
822+
.build()
823+
);
824+
return Err(miette::miette!(
825+
"proxy accept loop exited unexpectedly"
826+
));
827+
}
828+
}
753829
}
754830
#[cfg(not(target_os = "linux"))]
755831
{
756-
wait_for_shutdown_signal().await;
757-
0
832+
tokio::select! {
833+
() = wait_for_shutdown_signal() => 0,
834+
_ = &mut proxy_exited => {
835+
ocsf_emit!(
836+
AppLifecycleBuilder::new(ocsf_ctx())
837+
.activity(ActivityId::Fail)
838+
.severity(SeverityId::High)
839+
.status(StatusId::Failure)
840+
.message(
841+
"Proxy accept loop exited unexpectedly; terminating sandbox"
842+
)
843+
.build()
844+
);
845+
return Err(miette::miette!(
846+
"proxy accept loop exited unexpectedly"
847+
));
848+
}
849+
}
758850
}
759851
};
760852

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,11 +173,11 @@ impl InferenceContext {
173173
}
174174
}
175175

176-
#[derive(Debug)]
177176
pub struct ProxyHandle {
178177
#[allow(dead_code)]
179178
http_addr: Option<SocketAddr>,
180179
join: JoinHandle<()>,
180+
exited_rx: tokio::sync::oneshot::Receiver<()>,
181181
}
182182

183183
impl ProxyHandle {
@@ -282,7 +282,13 @@ impl ProxyHandle {
282282
ocsf_emit!(event);
283283
}
284284

285+
let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>();
285286
let join = tokio::spawn(async move {
287+
// Hold the sender for the lifetime of this task — when the task
288+
// exits (panic, abort, or loop break), the sender drops and the
289+
// receiver fires, notifying the sandbox that the proxy is gone.
290+
let _proxy_exit_guard = exited_tx;
291+
286292
// Wait for the OPA engine's symlink resolution reload to complete
287293
// before accepting connections. This prevents requests from
288294
// observing a generation transition mid-flight, which would cause
@@ -390,13 +396,21 @@ impl ProxyHandle {
390396
Ok(Self {
391397
http_addr: Some(local_addr),
392398
join,
399+
exited_rx,
393400
})
394401
}
395402

396403
#[allow(dead_code)]
397404
pub const fn http_addr(&self) -> Option<SocketAddr> {
398405
self.http_addr
399406
}
407+
408+
/// Take the exit notification receiver for health monitoring.
409+
/// Resolves when the proxy accept loop task exits for any reason.
410+
pub fn take_exit_receiver(&mut self) -> tokio::sync::oneshot::Receiver<()> {
411+
let (_tx, dummy_rx) = tokio::sync::oneshot::channel();
412+
std::mem::replace(&mut self.exited_rx, dummy_rx)
413+
}
400414
}
401415

402416
impl Drop for ProxyHandle {

0 commit comments

Comments
 (0)