Skip to content

Commit 6d9e223

Browse files
shiju-nvjohntmyers
authored andcommitted
fix(vm): flush guest filesystem before a planned sandbox stop
A planned stop of a MicroVM sandbox ended the VM before the guest wrote its dirty pages to disk, so files written shortly before the stop were missing after the next start. The driver closed the host supervisor's parent-liveness pipe before it sent SIGTERM. The supervisor exits at once on that end-of-file, so it never ran its orderly shutdown: terminate the workload, then ask the guest to terminate. Signal the supervisor first, keep the pipe open until it has exited, and give it a grace period derived from the supervisor's own bounds and kept below the gateway's reclaim deadline. In the guest runtime, syncfs the root filesystem after the process tree is terminated and before the termination is acknowledged, bounded to 3 seconds and best effort. Only the VM backend, which uses a vsock boundary listener, flushes; the container drivers share this runtime and are unchanged. A sandbox whose main process has already exited is not covered. Closes NVIDIA#4185 Signed-off-by: Shiju <[email protected]>
1 parent a2982bf commit 6d9e223

2 files changed

Lines changed: 530 additions & 38 deletions

File tree

‎crates/openshell-driver-vm/src/driver.rs‎

Lines changed: 220 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ use oci_client::secrets::RegistryAuth;
4545
use oci_client::{Reference, RegistryOperation};
4646
use openshell_core::UpstreamProxyConfig;
4747
use openshell_core::driver_mounts;
48+
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
4849
use openshell_core::gpu::{
4950
driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request,
5051
};
@@ -7462,7 +7463,51 @@ fn overlay_staging_dir(overlay_disk: &Path) -> PathBuf {
74627463
))
74637464
}
74647465

7466+
/// How long a host process gets to exit after SIGTERM before it is killed.
7467+
const PROCESS_TERMINATION_GRACE: Duration = Duration::from_secs(5);
7468+
7469+
/// Seconds the supervisor waits for the workload to exit after SIGTERM before
7470+
/// it kills the workload.
7471+
const SUPERVISOR_WORKLOAD_TERM_SECS: u64 = 5;
7472+
/// Seconds the guest waits for the sandbox's remaining processes to exit, then
7473+
/// for the forced kill to be reaped, when it terminates the boundary.
7474+
const GUEST_PROCESS_TREE_SECS: u64 = DEFAULT_STOP_TIMEOUT_SECS as u64 + 2;
7475+
/// Seconds the guest allows for flushing its filesystem.
7476+
const GUEST_FLUSH_SECS: u64 = 3;
7477+
7478+
/// How long the host supervisor gets to finish its orderly shutdown after
7479+
/// SIGTERM: the workload's termination wait, then the guest's process-tree
7480+
/// termination and filesystem flush. The values mirror those bounds, so a
7481+
/// healthy stop, even a slow one, finishes inside it. A supervisor still
7482+
/// running after this is stuck on an unresponsive guest and is killed.
7483+
///
7484+
/// The workload wait, the flush bound and the caller deadline below are copies
7485+
/// of values defined in other crates; only the stop timeout is shared. Nothing
7486+
/// fails to compile if one of those changes, so the assertion below checks the
7487+
/// copies, not the originals. The grace is also the exact sum of the stage
7488+
/// maxima with no slack: a stop in which every stage runs to its limit can
7489+
/// kill the supervisor just before it acknowledges.
7490+
const SUPERVISOR_SHUTDOWN_GRACE: Duration =
7491+
Duration::from_secs(SUPERVISOR_WORKLOAD_TERM_SECS + GUEST_PROCESS_TREE_SECS + GUEST_FLUSH_SECS);
7492+
7493+
/// The gateway cancels a stop that runs longer than this when it reclaims an
7494+
/// expired provisioning attempt, which would skip the stop's cleanup steps.
7495+
const STOP_CALLER_DEADLINE: Duration = Duration::from_secs(30);
7496+
7497+
// The longest stop is the supervisor's grace plus the VM's.
7498+
const _: () = assert!(
7499+
SUPERVISOR_SHUTDOWN_GRACE.as_secs() + PROCESS_TERMINATION_GRACE.as_secs()
7500+
< STOP_CALLER_DEADLINE.as_secs()
7501+
);
7502+
74657503
async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
7504+
terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE).await
7505+
}
7506+
7507+
async fn terminate_vm_process_within(
7508+
child: &mut Child,
7509+
grace: Duration,
7510+
) -> Result<(), std::io::Error> {
74667511
if let Some(pid) = child.id()
74677512
&& let Err(err) = kill(Pid::from_raw(pid.cast_signed()), Signal::SIGTERM)
74687513
&& err != Errno::ESRCH
@@ -7472,7 +7517,7 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
74727517
)));
74737518
}
74747519

7475-
match tokio::time::timeout(Duration::from_secs(5), child.wait()).await {
7520+
match tokio::time::timeout(grace, child.wait()).await {
74767521
Ok(Ok(_)) => Ok(()),
74777522
Ok(Err(err)) => Err(err),
74787523
Err(_) => {
@@ -7482,13 +7527,24 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
74827527
}
74837528
}
74847529

7530+
/// Stop the host supervisor first, then the VM it was supervising.
7531+
///
7532+
/// The supervisor exits immediately when the parent-liveness pipe reaches EOF,
7533+
/// treating it as driver death. The pipe therefore stays open until the
7534+
/// supervisor has exited, so SIGTERM runs its orderly shutdown (terminate the
7535+
/// workload, then have the guest terminate and flush its filesystem) instead
7536+
/// of losing a race with the pipe closing.
74857537
async fn terminate_sandbox_processes(process: &mut VmProcess) -> Result<(), std::io::Error> {
7486-
// Liveness EOF makes the supervisor exit at once, skipping workload
7487-
// teardown and its OCSF export flush, so close it only after SIGTERM has
7488-
// had its chance. Each process keeps its five-second budget, so a stop
7489-
// still completes within ten seconds; a slow workload stop can leave the
7490-
// supervisor too little time to flush.
7491-
let supervisor_error = terminate_vm_process(&mut process.supervisor).await.err();
7538+
terminate_sandbox_processes_within(process, SUPERVISOR_SHUTDOWN_GRACE).await
7539+
}
7540+
7541+
async fn terminate_sandbox_processes_within(
7542+
process: &mut VmProcess,
7543+
supervisor_grace: Duration,
7544+
) -> Result<(), std::io::Error> {
7545+
let supervisor_error = terminate_vm_process_within(&mut process.supervisor, supervisor_grace)
7546+
.await
7547+
.err();
74927548
process.supervisor_liveness.take();
74937549
let vm_error = terminate_vm_process(&mut process.child).await.err();
74947550

@@ -12061,6 +12117,163 @@ exit 1"#,
1206112117
let _ = fs::remove_dir_all(dir);
1206212118
}
1206312119

12120+
/// The supervisor exits at once when its parent-liveness pipe reaches EOF,
12121+
/// so a stop must signal it while the pipe is still open. The stand-in
12122+
/// supervisor takes 200 ms to handle SIGTERM, long enough for a prematurely
12123+
/// closed pipe to be seen first.
12124+
#[tokio::test]
12125+
async fn stop_signals_supervisor_before_closing_its_liveness_pipe() {
12126+
let dir = unique_temp_dir();
12127+
fs::create_dir_all(&dir).unwrap();
12128+
let events = dir.join("events");
12129+
let (read_end, write_end) = nix::unistd::pipe().unwrap();
12130+
for fd in [&read_end, &write_end] {
12131+
nix::fcntl::fcntl(
12132+
fd.as_raw_fd(),
12133+
nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC),
12134+
)
12135+
.unwrap();
12136+
}
12137+
let read_fd = read_end.as_raw_fd();
12138+
let mut command = Command::new("sh");
12139+
command
12140+
.arg("-c")
12141+
.arg(
12142+
r#"events=$1
12143+
(cat <&3 >/dev/null; echo eof >>"$events") &
12144+
trap 'sleep 0.2; echo term >>"$events"; exit 0' TERM
12145+
echo ready >>"$events"
12146+
while :; do sleep 0.05; done"#,
12147+
)
12148+
.arg("supervisor")
12149+
.arg(&events)
12150+
.stdin(Stdio::null())
12151+
.stdout(Stdio::null())
12152+
.stderr(Stdio::null());
12153+
// SAFETY: only async-signal-safe `dup2` runs between fork and exec.
12154+
unsafe {
12155+
command.pre_exec(move || {
12156+
if libc::dup2(read_fd, 3) < 0 {
12157+
return Err(std::io::Error::last_os_error());
12158+
}
12159+
Ok(())
12160+
});
12161+
}
12162+
let supervisor = command.spawn().unwrap();
12163+
drop(read_end);
12164+
12165+
let ready = async {
12166+
while !fs::read_to_string(&events).is_ok_and(|log| log.contains("ready")) {
12167+
tokio::time::sleep(Duration::from_millis(10)).await;
12168+
}
12169+
};
12170+
tokio::time::timeout(Duration::from_secs(5), ready)
12171+
.await
12172+
.expect("stand-in supervisor installs its SIGTERM handler");
12173+
12174+
let mut process = VmProcess {
12175+
child: spawn_exited_child(),
12176+
supervisor,
12177+
supervisor_liveness: Some(fs::File::from(write_end)),
12178+
deleting: false,
12179+
};
12180+
terminate_sandbox_processes(&mut process).await.unwrap();
12181+
12182+
let log = fs::read_to_string(&events).unwrap();
12183+
let first_event = log.lines().find(|line| *line != "ready");
12184+
assert_eq!(
12185+
first_event,
12186+
Some("term"),
12187+
"SIGTERM must reach the supervisor before its liveness pipe closes; events: {log:?}"
12188+
);
12189+
fs::remove_dir_all(&dir).ok();
12190+
}
12191+
12192+
/// A supervisor that ignores SIGTERM is killed once its grace expires, and
12193+
/// only then is its liveness pipe closed.
12194+
#[tokio::test]
12195+
async fn stop_kills_a_supervisor_that_ignores_sigterm_after_the_grace() {
12196+
let dir = unique_temp_dir();
12197+
fs::create_dir_all(&dir).unwrap();
12198+
let events = dir.join("events");
12199+
let (read_end, write_end) = nix::unistd::pipe().unwrap();
12200+
for fd in [&read_end, &write_end] {
12201+
nix::fcntl::fcntl(
12202+
fd.as_raw_fd(),
12203+
nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC),
12204+
)
12205+
.unwrap();
12206+
}
12207+
let read_fd = read_end.as_raw_fd();
12208+
let mut command = Command::new("sh");
12209+
command
12210+
.arg("-c")
12211+
.arg(
12212+
r#"events=$1
12213+
(cat <&3 >/dev/null; echo eof >>"$events") &
12214+
trap '' TERM
12215+
echo ready >>"$events"
12216+
while :; do sleep 0.05; done"#,
12217+
)
12218+
.arg("supervisor")
12219+
.arg(&events)
12220+
.stdin(Stdio::null())
12221+
.stdout(Stdio::null())
12222+
.stderr(Stdio::null());
12223+
// SAFETY: only async-signal-safe `dup2` runs between fork and exec.
12224+
unsafe {
12225+
command.pre_exec(move || {
12226+
if libc::dup2(read_fd, 3) < 0 {
12227+
return Err(std::io::Error::last_os_error());
12228+
}
12229+
Ok(())
12230+
});
12231+
}
12232+
let supervisor = command.spawn().unwrap();
12233+
drop(read_end);
12234+
let ready = async {
12235+
while !fs::read_to_string(&events).is_ok_and(|log| log.contains("ready")) {
12236+
tokio::time::sleep(Duration::from_millis(10)).await;
12237+
}
12238+
};
12239+
tokio::time::timeout(Duration::from_secs(5), ready)
12240+
.await
12241+
.expect("stand-in supervisor starts");
12242+
12243+
let mut process = VmProcess {
12244+
child: spawn_exited_child(),
12245+
supervisor,
12246+
supervisor_liveness: Some(fs::File::from(write_end)),
12247+
deleting: false,
12248+
};
12249+
let grace = Duration::from_millis(300);
12250+
let started = tokio::time::Instant::now();
12251+
terminate_sandbox_processes_within(&mut process, grace)
12252+
.await
12253+
.unwrap();
12254+
let elapsed = started.elapsed();
12255+
12256+
assert!(
12257+
elapsed >= grace,
12258+
"the supervisor must get its full grace before the kill: {elapsed:?}"
12259+
);
12260+
assert!(
12261+
process.supervisor.try_wait().unwrap().is_some(),
12262+
"the supervisor is gone once the grace has expired"
12263+
);
12264+
assert!(process.supervisor_liveness.is_none());
12265+
// The liveness pipe closes only after the supervisor has been dealt with.
12266+
let closed = async {
12267+
while !fs::read_to_string(&events).is_ok_and(|log| log.contains("eof")) {
12268+
tokio::time::sleep(Duration::from_millis(10)).await;
12269+
}
12270+
};
12271+
tokio::time::timeout(Duration::from_secs(5), closed)
12272+
.await
12273+
.expect("the liveness pipe is closed after the stop");
12274+
fs::remove_dir_all(&dir).ok();
12275+
}
12276+
1206412277
async fn insert_test_record(
1206512278
driver: &VmDriver,
1206612279
sandbox_id: &str,

0 commit comments

Comments
 (0)