Skip to content

Commit 6482d53

Browse files
committed
fix(supervisor): keep shared stops unbounded and report the workload's exit
Only a dedicated supervisor's own shutdown is bounded by the 8 second deadline and treats a lost boundary as the end of the workload. A shared supervisor stopping one hosted instance keeps the host's removal grace and reports a stop it could not confirm as a failure, as before. When the boundary reports the workload's exit status before it is lost, that status is reported rather than the terminated fallback. Signed-off-by: divesh <[email protected]>
1 parent 91b6ccf commit 6482d53

2 files changed

Lines changed: 141 additions & 28 deletions

File tree

‎crates/openshell-supervisor/README.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,12 @@ push and the OCSF JSON settings with its logging layers, so renewal by either
5858
gateway client keeps both current. `run_sandbox_with_backend` sets the
5959
process-wide OCSF context, binds the readiness socket and port to instance
6060
readiness, and stops the instance on SIGTERM or SIGINT. Stopping gives the
61-
workload 5 seconds to exit after TERM and finishes, including the exit report
62-
and trace flush, within 8 seconds. A boundary connection lost while the
63-
process stops is not recovered; the workload is reported as terminated.
61+
workload 5 seconds to exit after TERM, and the instance stops, including its
62+
exit report and trace flush, within 8 seconds. Telemetry exporters shut down
63+
after that and are not covered by this bound. A boundary connection lost while
64+
the process stops is not recovered; the workload's exit status is reported
65+
when the boundary returned it, otherwise the workload is reported as
66+
terminated.
6467

6568
`sandbox_instance_host::SandboxInstanceHost` runs several sandbox instances in
6669
one process. Each instance gets its own Tokio runtime; every thread of that

‎crates/openshell-supervisor/src/lib.rs‎

Lines changed: 135 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -507,8 +507,9 @@ use tokio::time::timeout;
507507
/// Time the workload gets to exit after TERM before it is killed.
508508
const WORKLOAD_TERM_GRACE: Duration = Duration::from_secs(5);
509509

510-
/// Bound on stopping an instance, including its exit report and trace flush.
511-
/// Below the 10 second grace the Docker driver gives the supervisor.
510+
/// Bound on stopping a dedicated supervisor's instance, including its exit
511+
/// report and trace flush. Below the 10 second grace the Docker driver gives
512+
/// the supervisor.
512513
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(8);
513514

514515
/// Exit code reported when the boundary is lost before it reports the
@@ -1817,41 +1818,24 @@ pub async fn run_sandbox_instance(
18171818
));
18181819
}
18191820
() = &mut shutdown_requested => {
1820-
let deadline = tokio::time::Instant::now() + SHUTDOWN_TIMEOUT;
1821-
shutdown_deadline = Some(deadline);
18221821
if shutdown_ends_process {
1822+
shutdown_deadline = Some(tokio::time::Instant::now() + SHUTDOWN_TIMEOUT);
18231823
running.begin_stop();
18241824
}
1825-
let stop = async {
1825+
let wait = async {
18261826
let _ = agent
18271827
.signal(openshell_isolation_interface::contract::BoundarySignal::Term)
18281828
.await;
1829-
let status = if let Ok(result) = timeout(WORKLOAD_TERM_GRACE, agent.wait()).await {
1829+
if let Ok(result) = timeout(WORKLOAD_TERM_GRACE, agent.wait()).await {
18301830
result
18311831
} else {
18321832
let _ = agent.terminate().await;
18331833
agent.wait().await
1834-
}?;
1835-
running.terminate().await?;
1836-
Ok(status)
1837-
};
1838-
let exit_code = match tokio::time::timeout_at(deadline, stop).await {
1839-
Ok(Ok(status)) => boundary_exit_code(status),
1840-
Ok(Err(
1841-
openshell_isolation_interface::contract::BackendError::Unavailable(error)
1842-
| openshell_isolation_interface::contract::BackendError::Terminated(error),
1843-
)) => {
1844-
warn!(%error, "Isolation boundary lost while stopping; finishing shutdown");
1845-
STOPPED_EXIT_CODE
1846-
}
1847-
Ok(Err(error)) => {
1848-
return Err(miette::miette!("stop sandbox workload: {error}"));
1849-
}
1850-
Err(_) => {
1851-
warn!("Isolation boundary did not stop in time; finishing shutdown");
1852-
STOPPED_EXIT_CODE
18531834
}
18541835
};
1836+
let exit_code =
1837+
Box::pin(stop_workload(wait, || running.terminate(), shutdown_deadline))
1838+
.await?;
18551839
break Some((exit_code, false));
18561840
}
18571841
// The agent wait is cancelled while suspending so a fenced
@@ -1972,6 +1956,58 @@ pub async fn run_sandbox_instance(
19721956
Ok(exit_code)
19731957
}
19741958

1959+
/// Waits for a stopping workload, terminates its boundary, and returns the
1960+
/// workload's exit code.
1961+
///
1962+
/// With a `deadline`, stopping also ends this process: the stop is abandoned
1963+
/// when the deadline passes, and a lost boundary does not fail it. The
1964+
/// workload's status is reported when the boundary returned one, otherwise
1965+
/// `STOPPED_EXIT_CODE`.
1966+
async fn stop_workload<W, T, F>(
1967+
wait: W,
1968+
terminate: T,
1969+
deadline: Option<tokio::time::Instant>,
1970+
) -> Result<i32>
1971+
where
1972+
W: Future<
1973+
Output = std::result::Result<
1974+
openshell_isolation_interface::contract::BoundaryExitStatus,
1975+
openshell_isolation_interface::contract::BackendError,
1976+
>,
1977+
>,
1978+
T: FnOnce() -> F,
1979+
F: Future<
1980+
Output = std::result::Result<(), openshell_isolation_interface::contract::BackendError>,
1981+
>,
1982+
{
1983+
let Some(deadline) = deadline else {
1984+
let status = wait
1985+
.await
1986+
.map_err(|error| miette::miette!(error.to_string()))?;
1987+
terminate().await.map_err(|error| {
1988+
miette::miette!("sandbox did not acknowledge terminal state: {error}")
1989+
})?;
1990+
return Ok(boundary_exit_code(status));
1991+
};
1992+
let mut observed = None;
1993+
let stop = async {
1994+
observed = Some(wait.await?);
1995+
terminate().await
1996+
};
1997+
match tokio::time::timeout_at(deadline, stop).await {
1998+
Ok(Ok(())) => {}
1999+
Ok(Err(
2000+
openshell_isolation_interface::contract::BackendError::Unavailable(error)
2001+
| openshell_isolation_interface::contract::BackendError::Terminated(error),
2002+
)) => {
2003+
warn!(%error, "Isolation boundary lost while stopping; finishing shutdown");
2004+
}
2005+
Ok(Err(error)) => return Err(miette::miette!("stop sandbox workload: {error}")),
2006+
Err(_) => warn!("Isolation boundary did not stop in time; finishing shutdown"),
2007+
}
2008+
Ok(observed.map_or(STOPPED_EXIT_CODE, boundary_exit_code))
2009+
}
2010+
19752011
fn boundary_exit_code(status: openshell_isolation_interface::contract::BoundaryExitStatus) -> i32 {
19762012
match status {
19772013
openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => code,
@@ -5071,6 +5107,80 @@ mod tests {
50715107
);
50725108
}
50735109

5110+
use openshell_isolation_interface::contract::{BackendError, BoundaryExitStatus};
5111+
5112+
fn stop_deadline() -> tokio::time::Instant {
5113+
tokio::time::Instant::now() + SHUTDOWN_TIMEOUT
5114+
}
5115+
5116+
#[tokio::test]
5117+
async fn stopping_process_keeps_workload_status_when_boundary_is_lost() {
5118+
let exit_code = stop_workload(
5119+
async { Ok(BoundaryExitStatus::Exited(0)) },
5120+
|| async { Err(BackendError::Unavailable("connection lost".into())) },
5121+
Some(stop_deadline()),
5122+
)
5123+
.await
5124+
.expect("a lost boundary does not fail the stop");
5125+
assert_eq!(exit_code, 0);
5126+
}
5127+
5128+
#[tokio::test]
5129+
async fn stopping_process_reports_terminated_without_workload_status() {
5130+
let exit_code = stop_workload(
5131+
async { Err(BackendError::Terminated("boundary lost during wait".into())) },
5132+
|| async { Ok(()) },
5133+
Some(stop_deadline()),
5134+
)
5135+
.await
5136+
.expect("a lost boundary does not fail the stop");
5137+
assert_eq!(exit_code, STOPPED_EXIT_CODE);
5138+
}
5139+
5140+
#[tokio::test(start_paused = true)]
5141+
async fn stopping_process_keeps_workload_status_past_its_deadline() {
5142+
let exit_code = stop_workload(
5143+
async { Ok(BoundaryExitStatus::Exited(3)) },
5144+
std::future::pending,
5145+
Some(stop_deadline()),
5146+
)
5147+
.await
5148+
.expect("an expired deadline does not fail the stop");
5149+
assert_eq!(exit_code, 3);
5150+
}
5151+
5152+
#[tokio::test]
5153+
async fn hosted_instance_stop_fails_when_boundary_is_lost() {
5154+
let error = stop_workload(
5155+
async { Ok(BoundaryExitStatus::Exited(0)) },
5156+
|| async { Err(BackendError::Unavailable("connection lost".into())) },
5157+
None,
5158+
)
5159+
.await
5160+
.expect_err("a hosted instance reports a stop it could not confirm");
5161+
assert!(
5162+
error
5163+
.to_string()
5164+
.contains("sandbox did not acknowledge terminal state"),
5165+
"{error}"
5166+
);
5167+
}
5168+
5169+
#[tokio::test(start_paused = true)]
5170+
async fn hosted_instance_stop_is_not_bounded_by_shutdown_timeout() {
5171+
let exit_code = stop_workload(
5172+
async {
5173+
tokio::time::sleep(SHUTDOWN_TIMEOUT * 2).await;
5174+
Ok(BoundaryExitStatus::Exited(0))
5175+
},
5176+
|| async { Ok(()) },
5177+
None,
5178+
)
5179+
.await
5180+
.expect("hosted instance stops");
5181+
assert_eq!(exit_code, 0);
5182+
}
5183+
50745184
#[test]
50755185
fn discover_policy_from_nonexistent_path_is_missing() {
50765186
let path = std::path::Path::new("/nonexistent/policy.yaml");

0 commit comments

Comments
 (0)