Skip to content

Commit a991b8e

Browse files
authored
fix(sandbox): preserve exec capacity under failure and load (#4324)
* fix(sandbox): reclaim failed exec waits Signed-off-by: Charan Jagwani <[email protected]> * test(e2e): cover exec recovery after policy denial Signed-off-by: Charan Jagwani <[email protected]> * fix(sandbox): preserve live descriptor headroom Signed-off-by: Charan Jagwani <[email protected]> * fix(sandbox): reclaim descriptors before exec Signed-off-by: Charan Jagwani <[email protected]> * fix(sandbox): separate exec and socket capacity Signed-off-by: Charan Jagwani <[email protected]> * fix(sandbox): stream Landlock baseline entries Signed-off-by: Charan Jagwani <[email protected]> --------- Signed-off-by: Charan Jagwani <[email protected]>
1 parent b959bb6 commit a991b8e

6 files changed

Lines changed: 360 additions & 70 deletions

File tree

‎crates/openshell-sandbox/src/boundary_server.rs‎

Lines changed: 83 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ mod linux {
8282
const MAX_CONTROL_CONNECTIONS: usize = 128;
8383
const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096;
8484
const MAX_RETAINED_EXEC_PROCESSES: usize = 64;
85+
// Three stdio pipes require six descriptors before fork, and Rust's
86+
// process launcher may briefly need additional bookkeeping descriptors.
87+
// Reclaim closed mediated sockets before each serialized spawn so stale
88+
// socket sources cannot consume that bounded launch budget.
89+
const EXEC_SPAWN_DESCRIPTOR_HEADROOM: usize = 16;
8590

8691
// NVML may traverse the persistenced socket directory during initialization;
8792
// WSL2 supplies GPU libraries under /usr/lib/wsl and the /dev/dxg device.
@@ -1398,6 +1403,29 @@ mod linux {
13981403
status: Arc<Mutex<Option<ExitStatusWire>>>,
13991404
}
14001405

1406+
async fn retain_exec_completion(
1407+
process: Arc<dyn BoundaryProcess>,
1408+
session: Arc<MainSession>,
1409+
status: Arc<Mutex<Option<ExitStatusWire>>>,
1410+
) {
1411+
let exit_status = match process.wait().await {
1412+
Ok(exit_status) => ExitStatusWire::from(exit_status),
1413+
Err(error) => {
1414+
// A process wait error is terminal per the BoundaryProcess
1415+
// contract. Retaining `None` here makes this handle look live
1416+
// forever and can permanently exhaust exec admission.
1417+
tracing::warn!(%error, "Exec process wait failed; recording terminal status");
1418+
ExitStatusWire::Exited(74)
1419+
}
1420+
};
1421+
*lock(&status) = Some(exit_status);
1422+
let exit_code = match exit_status {
1423+
ExitStatusWire::Exited(code) => code,
1424+
ExitStatusWire::Signaled(signal) => 128 + signal,
1425+
};
1426+
let _ = session.finish_remote(exit_code, false).await;
1427+
}
1428+
14011429
#[derive(Default)]
14021430
struct ExecRequestLedger {
14031431
requests: std::collections::HashSet<String>,
@@ -2162,6 +2190,14 @@ mod linux {
21622190
}
21632191
requests.reserve(request_id, expires_at)?;
21642192
drop(requests);
2193+
self.network_broker
2194+
.ensure_descriptor_headroom(EXEC_SPAWN_DESCRIPTOR_HEADROOM)
2195+
.map_err(|error| {
2196+
guest_error(
2197+
BoundaryErrorKind::Unavailable,
2198+
format!("exec descriptor headroom unavailable: {error}"),
2199+
)
2200+
})?;
21652201
let session = self
21662202
.process_runtime
21672203
.block_on(executor.exec(spec.into()))
@@ -2202,18 +2238,7 @@ mod linux {
22022238
let wait_session = retained.clone();
22032239
let wait_status = status.clone();
22042240
self.process_runtime.spawn(async move {
2205-
if let Ok(exit_status) = wait_process.wait().await {
2206-
*lock(&wait_status) = Some(ExitStatusWire::from(exit_status));
2207-
let exit_code = match exit_status {
2208-
openshell_isolation_interface::contract::BoundaryExitStatus::Exited(
2209-
code,
2210-
) => code,
2211-
openshell_isolation_interface::contract::BoundaryExitStatus::Signaled(
2212-
signal,
2213-
) => 128 + signal,
2214-
};
2215-
let _ = wait_session.finish_remote(exit_code, false).await;
2216-
}
2241+
retain_exec_completion(wait_process, wait_session, wait_status).await;
22172242
});
22182243
let handle = ExecHandle {
22192244
request_id: request_id.to_string(),
@@ -3820,6 +3845,52 @@ mod linux {
38203845
};
38213846
use rcgen::{KeyPair, PKCS_ED25519};
38223847

3848+
struct FailedWaitProcess;
3849+
3850+
#[async_trait::async_trait]
3851+
impl BoundaryProcess for FailedWaitProcess {
3852+
async fn wait(
3853+
&self,
3854+
) -> Result<
3855+
openshell_isolation_interface::contract::BoundaryExitStatus,
3856+
openshell_isolation_interface::contract::BackendError,
3857+
> {
3858+
Err(
3859+
openshell_isolation_interface::contract::BackendError::Terminated(
3860+
"test boundary loss".to_string(),
3861+
),
3862+
)
3863+
}
3864+
3865+
async fn signal(
3866+
&self,
3867+
_signal: openshell_isolation_interface::contract::BoundarySignal,
3868+
) -> Result<(), openshell_isolation_interface::contract::BackendError> {
3869+
Ok(())
3870+
}
3871+
3872+
async fn terminate(
3873+
&self,
3874+
) -> Result<(), openshell_isolation_interface::contract::BackendError> {
3875+
Ok(())
3876+
}
3877+
}
3878+
3879+
#[tokio::test]
3880+
async fn exec_wait_failure_records_reclaimable_terminal_status() {
3881+
let session = MainSession::inert();
3882+
let mut output = session.subscribe();
3883+
let status = Arc::new(Mutex::new(None));
3884+
3885+
retain_exec_completion(Arc::new(FailedWaitProcess), session, status.clone()).await;
3886+
3887+
assert_eq!(*lock(&status), Some(ExitStatusWire::Exited(74)));
3888+
assert!(matches!(
3889+
output.recv().await.expect("terminal exec output"),
3890+
MainOutput::Exit(74)
3891+
));
3892+
}
3893+
38233894
#[test]
38243895
fn exec_tombstones_expire_without_a_lifetime_limit() {
38253896
let mut ledger = ExecRequestLedger::default();

0 commit comments

Comments
 (0)