Skip to content

Commit 74015b9

Browse files
fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly (NVIDIA#2370)
* fix(sandbox): terminate sandbox when proxy accept loop exits unexpectedly Add a oneshot notification channel to ProxyHandle that fires when the proxy task exits for any reason (panic, abort, or loop break). The sandbox main loop now races this signal alongside the entrypoint process and shutdown signals, terminating with a clear OCSF error if the proxy dies. This prevents the sandbox from continuing to report Ready with a dead proxy. Follows the existing sidecar control channel pattern. All five wait paths (process+sidecar, process-only, network+sidecar, network-only on Linux, and non-Linux) now monitor proxy liveness. Refs NVIDIA#2337 Signed-off-by: politerealism <[email protected]> * test(proxy): add unit tests for exit detection drop-guard and take_exit_receiver Verify the oneshot drop-guard pattern fires on normal task exit and abort, and document the take_exit_receiver contract including the double-call hazard. Signed-off-by: politerealism <[email protected]> * fix(proxy): classify terminal vs transient accept errors and exit on unrecoverable failures Extract accept-loop error handling into classify_accept_error() with AcceptAction enum so the decision logic is independently testable. Terminal errors (EBADF/EINVAL/ENOTSOCK) now exit the loop immediately, allowing the supervisor to detect proxy death. Unknown errors exit after 10 consecutive failures. FD exhaustion retries with exponential backoff capped at 5s (fix unreachable cap from min(6) to min(7)). Adds 9 unit tests covering classification, counter interactions, backoff progression, and bounded exit behavior. Signed-off-by: politerealism <[email protected]> * refactor(proxy): use libc constants and Option-based exit receiver Replace raw errno numbers with libc:: constants behind #[cfg(unix)] gates in classify_accept_error and its tests. Refactor ProxyHandle::exited_rx to Option<Receiver<()>> so take_exit_receiver uses .take() instead of allocating a dummy sender. Add a ProxyHandle-level test that verifies the full drop → abort → receiver fires path. Signed-off-by: politerealism <[email protected]> * fix(proxy): restore full transient errno classification from main The refactored classifier only recognized EMFILE/ENFILE as retryable, routing all other documented transient accept errors (ENOBUFS, ENOMEM, ECONNABORTED, ECONNRESET, EINTR, ENETDOWN, etc.) through the unknown- error budget which terminates the proxy after 10 consecutive failures. Restore the complete transient errno table from main so these errors retry indefinitely with appropriate backoff. Signed-off-by: politerealism <[email protected]> * fix(sandbox): resolve Clippy warnings in proxy-exit detection code Replace redundant closure |p| p.take_exit_receiver() with the method reference ProxyHandle::take_exit_receiver, and change all five select! arms from _ = &mut proxy_exited to () = &mut proxy_exited to satisfy clippy::ignored_unit_patterns. Also run cargo fmt on the new proxy test in proxy.rs. Signed-off-by: politerealism <[email protected]> * ci: trigger required check re-run Signed-off-by: politerealism <[email protected]> * fix(sandbox): remove unnecessary std::pin qualification Signed-off-by: politerealism <[email protected]> --------- Signed-off-by: politerealism <[email protected]>
1 parent 18ce13b commit 74015b9

2 files changed

Lines changed: 447 additions & 382 deletions

File tree

  • crates

‎crates/openshell-sandbox/src/lib.rs‎

Lines changed: 101 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ mod sidecar_control;
1616

1717
use miette::{IntoDiagnostic, Result, WrapErr};
1818
use std::future::Future;
19+
use std::pin::Pin;
1920
use std::sync::Arc;
2021
#[cfg(target_os = "linux")]
2122
use std::sync::atomic::Ordering;
@@ -62,6 +63,7 @@ use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPol
6263
use openshell_core::proposals::AgentProposals;
6364
use openshell_core::provider_credentials::ProviderCredentialState;
6465
use openshell_supervisor_network::opa::OpaEngine;
66+
use openshell_supervisor_network::proxy::ProxyHandle;
6567
use openshell_supervisor_process::process::ProcessEnforcementMode;
6668
pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus};
6769
use openshell_supervisor_process::skills;
@@ -514,7 +516,7 @@ pub async fn run_sandbox(
514516
// API read the current value so proposals target the correct workspace.
515517
let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new());
516518

517-
let networking = if network_enabled {
519+
let mut networking = if network_enabled {
518520
#[cfg(target_os = "linux")]
519521
let proxy_bind_ip = netns
520522
.as_ref()
@@ -823,6 +825,19 @@ pub async fn run_sandbox(
823825
.zip(bootstrap.proxy_ca_bundle_path.clone())
824826
});
825827

828+
let proxy_exited: Pin<Box<dyn Future<Output = ()> + Send>> = if let Some(rx) = networking
829+
.as_mut()
830+
.and_then(|n| n.proxy.as_mut())
831+
.and_then(ProxyHandle::take_exit_receiver)
832+
{
833+
Box::pin(async {
834+
let _ = rx.await;
835+
})
836+
} else {
837+
Box::pin(std::future::pending())
838+
};
839+
tokio::pin!(proxy_exited);
840+
826841
let exit_code = if process_enabled {
827842
let ca_file_paths = networking
828843
.as_ref()
@@ -935,9 +950,41 @@ pub async fn run_sandbox(
935950
"authoritative network-sidecar control channel closed"
936951
));
937952
}
953+
() = &mut proxy_exited => {
954+
ocsf_emit!(
955+
AppLifecycleBuilder::new(ocsf_ctx())
956+
.activity(ActivityId::Fail)
957+
.severity(SeverityId::High)
958+
.status(StatusId::Failure)
959+
.message(
960+
"Proxy accept loop exited unexpectedly; terminating sandbox"
961+
)
962+
.build()
963+
);
964+
return Err(miette::miette!(
965+
"proxy accept loop exited unexpectedly"
966+
));
967+
}
938968
}
939969
} else {
940-
process.await?
970+
tokio::select! {
971+
result = process => result?,
972+
() = &mut proxy_exited => {
973+
ocsf_emit!(
974+
AppLifecycleBuilder::new(ocsf_ctx())
975+
.activity(ActivityId::Fail)
976+
.severity(SeverityId::High)
977+
.status(StatusId::Failure)
978+
.message(
979+
"Proxy accept loop exited unexpectedly; terminating sandbox"
980+
)
981+
.build()
982+
);
983+
return Err(miette::miette!(
984+
"proxy accept loop exited unexpectedly"
985+
));
986+
}
987+
}
941988
}
942989
} else {
943990
// Network-only sidecar mode: keep the proxy and its background
@@ -953,15 +1000,62 @@ pub async fn run_sandbox(
9531000
warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar");
9541001
1
9551002
}
1003+
() = &mut proxy_exited => {
1004+
ocsf_emit!(
1005+
AppLifecycleBuilder::new(ocsf_ctx())
1006+
.activity(ActivityId::Fail)
1007+
.severity(SeverityId::High)
1008+
.status(StatusId::Failure)
1009+
.message(
1010+
"Proxy accept loop exited unexpectedly; terminating sandbox"
1011+
)
1012+
.build()
1013+
);
1014+
return Err(miette::miette!(
1015+
"proxy accept loop exited unexpectedly"
1016+
));
1017+
}
9561018
}
9571019
} else {
958-
wait_for_shutdown_signal().await;
959-
0
1020+
tokio::select! {
1021+
() = wait_for_shutdown_signal() => 0,
1022+
() = &mut proxy_exited => {
1023+
ocsf_emit!(
1024+
AppLifecycleBuilder::new(ocsf_ctx())
1025+
.activity(ActivityId::Fail)
1026+
.severity(SeverityId::High)
1027+
.status(StatusId::Failure)
1028+
.message(
1029+
"Proxy accept loop exited unexpectedly; terminating sandbox"
1030+
)
1031+
.build()
1032+
);
1033+
return Err(miette::miette!(
1034+
"proxy accept loop exited unexpectedly"
1035+
));
1036+
}
1037+
}
9601038
}
9611039
#[cfg(not(target_os = "linux"))]
9621040
{
963-
wait_for_shutdown_signal().await;
964-
0
1041+
tokio::select! {
1042+
() = wait_for_shutdown_signal() => 0,
1043+
() = &mut proxy_exited => {
1044+
ocsf_emit!(
1045+
AppLifecycleBuilder::new(ocsf_ctx())
1046+
.activity(ActivityId::Fail)
1047+
.severity(SeverityId::High)
1048+
.status(StatusId::Failure)
1049+
.message(
1050+
"Proxy accept loop exited unexpectedly; terminating sandbox"
1051+
)
1052+
.build()
1053+
);
1054+
return Err(miette::miette!(
1055+
"proxy accept loop exited unexpectedly"
1056+
));
1057+
}
1058+
}
9651059
}
9661060
};
9671061

@@ -3157,7 +3251,7 @@ type MiddlewareConnector = Arc<
31573251
dyn Fn(
31583252
Vec<openshell_core::proto::SupervisorMiddlewareService>,
31593253
MiddlewareAuthentication,
3160-
) -> std::pin::Pin<
3254+
) -> Pin<
31613255
Box<
31623256
dyn std::future::Future<
31633257
Output = Result<openshell_supervisor_middleware::MiddlewareRegistry>,

0 commit comments

Comments
 (0)