@@ -45,6 +45,7 @@ use oci_client::secrets::RegistryAuth;
4545use oci_client::{Reference, RegistryOperation};
4646use openshell_core::UpstreamProxyConfig;
4747use openshell_core::driver_mounts;
48+ use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
4849use openshell_core::gpu::{
4950 driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request,
5051};
@@ -7462,7 +7463,51 @@ fn overlay_staging_dir(overlay_disk: &Path) -> PathBuf {
74627463 ))
74637464}
74647465
7466+ /// How long a host process gets to exit after SIGTERM before it is killed.
7467+ const PROCESS_TERMINATION_GRACE: Duration = Duration::from_secs(5);
7468+
7469+ /// Seconds the supervisor waits for the workload to exit after SIGTERM before
7470+ /// it kills the workload.
7471+ const SUPERVISOR_WORKLOAD_TERM_SECS: u64 = 5;
7472+ /// Seconds the guest waits for the sandbox's remaining processes to exit, then
7473+ /// for the forced kill to be reaped, when it terminates the boundary.
7474+ const GUEST_PROCESS_TREE_SECS: u64 = DEFAULT_STOP_TIMEOUT_SECS as u64 + 2;
7475+ /// Seconds the guest allows for flushing its filesystem.
7476+ const GUEST_FLUSH_SECS: u64 = 3;
7477+
7478+ /// How long the host supervisor gets to finish its orderly shutdown after
7479+ /// SIGTERM: the workload's termination wait, then the guest's process-tree
7480+ /// termination and filesystem flush. The values mirror those bounds, so a
7481+ /// healthy stop, even a slow one, finishes inside it. A supervisor still
7482+ /// running after this is stuck on an unresponsive guest and is killed.
7483+ ///
7484+ /// The workload wait, the flush bound and the caller deadline below are copies
7485+ /// of values defined in other crates; only the stop timeout is shared. Nothing
7486+ /// fails to compile if one of those changes, so the assertion below checks the
7487+ /// copies, not the originals. The grace is also the exact sum of the stage
7488+ /// maxima with no slack: a stop in which every stage runs to its limit can
7489+ /// kill the supervisor just before it acknowledges.
7490+ const SUPERVISOR_SHUTDOWN_GRACE: Duration =
7491+ Duration::from_secs(SUPERVISOR_WORKLOAD_TERM_SECS + GUEST_PROCESS_TREE_SECS + GUEST_FLUSH_SECS);
7492+
7493+ /// The gateway cancels a stop that runs longer than this when it reclaims an
7494+ /// expired provisioning attempt, which would skip the stop's cleanup steps.
7495+ const STOP_CALLER_DEADLINE: Duration = Duration::from_secs(30);
7496+
7497+ // The longest stop is the supervisor's grace plus the VM's.
7498+ const _: () = assert!(
7499+ SUPERVISOR_SHUTDOWN_GRACE.as_secs() + PROCESS_TERMINATION_GRACE.as_secs()
7500+ < STOP_CALLER_DEADLINE.as_secs()
7501+ );
7502+
74657503async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
7504+ terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE).await
7505+ }
7506+
7507+ async fn terminate_vm_process_within(
7508+ child: &mut Child,
7509+ grace: Duration,
7510+ ) -> Result<(), std::io::Error> {
74667511 if let Some(pid) = child.id()
74677512 && let Err(err) = kill(Pid::from_raw(pid.cast_signed()), Signal::SIGTERM)
74687513 && err != Errno::ESRCH
@@ -7472,7 +7517,7 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
74727517 )));
74737518 }
74747519
7475- match tokio::time::timeout(Duration::from_secs(5) , child.wait()).await {
7520+ match tokio::time::timeout(grace , child.wait()).await {
74767521 Ok(Ok(_)) => Ok(()),
74777522 Ok(Err(err)) => Err(err),
74787523 Err(_) => {
@@ -7482,13 +7527,24 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
74827527 }
74837528}
74847529
7530+ /// Stop the host supervisor first, then the VM it was supervising.
7531+ ///
7532+ /// The supervisor exits immediately when the parent-liveness pipe reaches EOF,
7533+ /// treating it as driver death. The pipe therefore stays open until the
7534+ /// supervisor has exited, so SIGTERM runs its orderly shutdown (terminate the
7535+ /// workload, then have the guest terminate and flush its filesystem) instead
7536+ /// of losing a race with the pipe closing.
74857537async fn terminate_sandbox_processes(process: &mut VmProcess) -> Result<(), std::io::Error> {
7486- // Liveness EOF makes the supervisor exit at once, skipping workload
7487- // teardown and its OCSF export flush, so close it only after SIGTERM has
7488- // had its chance. Each process keeps its five-second budget, so a stop
7489- // still completes within ten seconds; a slow workload stop can leave the
7490- // supervisor too little time to flush.
7491- let supervisor_error = terminate_vm_process(&mut process.supervisor).await.err();
7538+ terminate_sandbox_processes_within(process, SUPERVISOR_SHUTDOWN_GRACE).await
7539+ }
7540+
7541+ async fn terminate_sandbox_processes_within(
7542+ process: &mut VmProcess,
7543+ supervisor_grace: Duration,
7544+ ) -> Result<(), std::io::Error> {
7545+ let supervisor_error = terminate_vm_process_within(&mut process.supervisor, supervisor_grace)
7546+ .await
7547+ .err();
74927548 process.supervisor_liveness.take();
74937549 let vm_error = terminate_vm_process(&mut process.child).await.err();
74947550
@@ -12061,6 +12117,163 @@ exit 1"#,
1206112117 let _ = fs::remove_dir_all(dir);
1206212118 }
1206312119
12120+ /// The supervisor exits at once when its parent-liveness pipe reaches EOF,
12121+ /// so a stop must signal it while the pipe is still open. The stand-in
12122+ /// supervisor takes 200 ms to handle SIGTERM, long enough for a prematurely
12123+ /// closed pipe to be seen first.
12124+ #[tokio::test]
12125+ async fn stop_signals_supervisor_before_closing_its_liveness_pipe() {
12126+ let dir = unique_temp_dir();
12127+ fs::create_dir_all(&dir).unwrap();
12128+ let events = dir.join("events");
12129+ let (read_end, write_end) = nix::unistd::pipe().unwrap();
12130+ for fd in [&read_end, &write_end] {
12131+ nix::fcntl::fcntl(
12132+ fd.as_raw_fd(),
12133+ nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC),
12134+ )
12135+ .unwrap();
12136+ }
12137+ let read_fd = read_end.as_raw_fd();
12138+ let mut command = Command::new("sh");
12139+ command
12140+ .arg("-c")
12141+ .arg(
12142+ r#"events=$1
12143+ (cat <&3 >/dev/null; echo eof >>"$events") &
12144+ trap 'sleep 0.2; echo term >>"$events"; exit 0' TERM
12145+ echo ready >>"$events"
12146+ while :; do sleep 0.05; done"#,
12147+ )
12148+ .arg("supervisor")
12149+ .arg(&events)
12150+ .stdin(Stdio::null())
12151+ .stdout(Stdio::null())
12152+ .stderr(Stdio::null());
12153+ // SAFETY: only async-signal-safe `dup2` runs between fork and exec.
12154+ unsafe {
12155+ command.pre_exec(move || {
12156+ if libc::dup2(read_fd, 3) < 0 {
12157+ return Err(std::io::Error::last_os_error());
12158+ }
12159+ Ok(())
12160+ });
12161+ }
12162+ let supervisor = command.spawn().unwrap();
12163+ drop(read_end);
12164+
12165+ let ready = async {
12166+ while !fs::read_to_string(&events).is_ok_and(|log| log.contains("ready")) {
12167+ tokio::time::sleep(Duration::from_millis(10)).await;
12168+ }
12169+ };
12170+ tokio::time::timeout(Duration::from_secs(5), ready)
12171+ .await
12172+ .expect("stand-in supervisor installs its SIGTERM handler");
12173+
12174+ let mut process = VmProcess {
12175+ child: spawn_exited_child(),
12176+ supervisor,
12177+ supervisor_liveness: Some(fs::File::from(write_end)),
12178+ deleting: false,
12179+ };
12180+ terminate_sandbox_processes(&mut process).await.unwrap();
12181+
12182+ let log = fs::read_to_string(&events).unwrap();
12183+ let first_event = log.lines().find(|line| *line != "ready");
12184+ assert_eq!(
12185+ first_event,
12186+ Some("term"),
12187+ "SIGTERM must reach the supervisor before its liveness pipe closes; events: {log:?}"
12188+ );
12189+ fs::remove_dir_all(&dir).ok();
12190+ }
12191+
12192+ /// A supervisor that ignores SIGTERM is killed once its grace expires, and
12193+ /// only then is its liveness pipe closed.
12194+ #[tokio::test]
12195+ async fn stop_kills_a_supervisor_that_ignores_sigterm_after_the_grace() {
12196+ let dir = unique_temp_dir();
12197+ fs::create_dir_all(&dir).unwrap();
12198+ let events = dir.join("events");
12199+ let (read_end, write_end) = nix::unistd::pipe().unwrap();
12200+ for fd in [&read_end, &write_end] {
12201+ nix::fcntl::fcntl(
12202+ fd.as_raw_fd(),
12203+ nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC),
12204+ )
12205+ .unwrap();
12206+ }
12207+ let read_fd = read_end.as_raw_fd();
12208+ let mut command = Command::new("sh");
12209+ command
12210+ .arg("-c")
12211+ .arg(
12212+ r#"events=$1
12213+ (cat <&3 >/dev/null; echo eof >>"$events") &
12214+ trap '' TERM
12215+ echo ready >>"$events"
12216+ while :; do sleep 0.05; done"#,
12217+ )
12218+ .arg("supervisor")
12219+ .arg(&events)
12220+ .stdin(Stdio::null())
12221+ .stdout(Stdio::null())
12222+ .stderr(Stdio::null());
12223+ // SAFETY: only async-signal-safe `dup2` runs between fork and exec.
12224+ unsafe {
12225+ command.pre_exec(move || {
12226+ if libc::dup2(read_fd, 3) < 0 {
12227+ return Err(std::io::Error::last_os_error());
12228+ }
12229+ Ok(())
12230+ });
12231+ }
12232+ let supervisor = command.spawn().unwrap();
12233+ drop(read_end);
12234+ let ready = async {
12235+ while !fs::read_to_string(&events).is_ok_and(|log| log.contains("ready")) {
12236+ tokio::time::sleep(Duration::from_millis(10)).await;
12237+ }
12238+ };
12239+ tokio::time::timeout(Duration::from_secs(5), ready)
12240+ .await
12241+ .expect("stand-in supervisor starts");
12242+
12243+ let mut process = VmProcess {
12244+ child: spawn_exited_child(),
12245+ supervisor,
12246+ supervisor_liveness: Some(fs::File::from(write_end)),
12247+ deleting: false,
12248+ };
12249+ let grace = Duration::from_millis(300);
12250+ let started = tokio::time::Instant::now();
12251+ terminate_sandbox_processes_within(&mut process, grace)
12252+ .await
12253+ .unwrap();
12254+ let elapsed = started.elapsed();
12255+
12256+ assert!(
12257+ elapsed >= grace,
12258+ "the supervisor must get its full grace before the kill: {elapsed:?}"
12259+ );
12260+ assert!(
12261+ process.supervisor.try_wait().unwrap().is_some(),
12262+ "the supervisor is gone once the grace has expired"
12263+ );
12264+ assert!(process.supervisor_liveness.is_none());
12265+ // The liveness pipe closes only after the supervisor has been dealt with.
12266+ let closed = async {
12267+ while !fs::read_to_string(&events).is_ok_and(|log| log.contains("eof")) {
12268+ tokio::time::sleep(Duration::from_millis(10)).await;
12269+ }
12270+ };
12271+ tokio::time::timeout(Duration::from_secs(5), closed)
12272+ .await
12273+ .expect("the liveness pipe is closed after the stop");
12274+ fs::remove_dir_all(&dir).ok();
12275+ }
12276+
1206412277 async fn insert_test_record(
1206512278 driver: &VmDriver,
1206612279 sandbox_id: &str,
0 commit comments