Skip to content

Commit d1949f4

Browse files
shiju-nvjohntmyers
authored andcommitted
fix(vm): warn about unconfirmed flush after supervisor kill
Preserve the supervisor exit status so a planned stop warns when a signal killed the supervisor and guest writeback was not confirmed. Keep normal workload exit codes quiet, including 143 after a handled SIGTERM. Extend the existing process tests to check the warning after a forced kill and its absence after an orderly shutdown. Refs NVIDIA#4185 Signed-off-by: Shiju <[email protected]>
1 parent f328bdf commit d1949f4

1 file changed

Lines changed: 48 additions & 7 deletions

File tree

‎crates/openshell-driver-vm/src/driver.rs‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,11 @@ use std::os::fd::AsRawFd as _;
9191
use std::os::fd::OwnedFd;
9292
#[cfg(unix)]
9393
use std::os::unix::fs::PermissionsExt;
94+
#[cfg(unix)]
95+
use std::os::unix::process::ExitStatusExt as _;
9496
use std::path::{Component, Path, PathBuf};
9597
use std::pin::Pin;
96-
use std::process::Stdio;
98+
use std::process::{ExitStatus, Stdio};
9799
use std::sync::Arc;
98100
use std::sync::atomic::{AtomicU64, Ordering};
99101
use std::time::Duration;
@@ -7501,13 +7503,15 @@ const _: () = assert!(
75017503
);
75027504

75037505
async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
7504-
terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE).await
7506+
terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE)
7507+
.await
7508+
.map(|_| ())
75057509
}
75067510

75077511
async fn terminate_vm_process_within(
75087512
child: &mut Child,
75097513
grace: Duration,
7510-
) -> Result<(), std::io::Error> {
7514+
) -> Result<ExitStatus, std::io::Error> {
75117515
if let Some(pid) = child.id()
75127516
&& let Err(err) = kill(Pid::from_raw(pid.cast_signed()), Signal::SIGTERM)
75137517
&& err != Errno::ESRCH
@@ -7518,11 +7522,10 @@ async fn terminate_vm_process_within(
75187522
}
75197523

75207524
match tokio::time::timeout(grace, child.wait()).await {
7521-
Ok(Ok(_)) => Ok(()),
7522-
Ok(Err(err)) => Err(err),
7525+
Ok(status) => status,
75237526
Err(_) => {
75247527
child.kill().await?;
7525-
child.wait().await.map(|_| ())
7528+
child.wait().await
75267529
}
75277530
}
75287531
}
@@ -7542,8 +7545,20 @@ async fn terminate_sandbox_processes_within(
75427545
process: &mut VmProcess,
75437546
supervisor_grace: Duration,
75447547
) -> Result<(), std::io::Error> {
7548+
let supervisor_pid = process.supervisor.id();
75457549
let supervisor_error = terminate_vm_process_within(&mut process.supervisor, supervisor_grace)
75467550
.await
7551+
.inspect(|status| {
7552+
// An orderly shutdown returns the workload's exit code, which can
7553+
// be nonzero even after the guest acknowledges termination.
7554+
if status.signal().is_some() {
7555+
warn!(
7556+
?supervisor_pid,
7557+
%status,
7558+
"VM host supervisor was terminated by a signal; guest filesystem flush was not confirmed and recent writes may be lost"
7559+
);
7560+
}
7561+
})
75477562
.err();
75487563
process.supervisor_liveness.take();
75497564
let vm_error = terminate_vm_process(&mut process.child).await.err();
@@ -12125,6 +12140,14 @@ exit 1"#,
1212512140
async fn stop_signals_supervisor_before_closing_its_liveness_pipe() {
1212612141
let dir = unique_temp_dir();
1212712142
fs::create_dir_all(&dir).unwrap();
12143+
let driver_log = dir.join("driver.log");
12144+
let subscriber = tracing_subscriber::fmt()
12145+
.with_max_level(tracing::Level::WARN)
12146+
.without_time()
12147+
.with_ansi(false)
12148+
.with_writer(Arc::new(fs::File::create(&driver_log).unwrap()))
12149+
.finish();
12150+
let _tracing = tracing::subscriber::set_default(subscriber);
1212812151
let events = dir.join("events");
1212912152
let (read_end, write_end) = nix::unistd::pipe().unwrap();
1213012153
for fd in [&read_end, &write_end] {
@@ -12141,7 +12164,7 @@ exit 1"#,
1214112164
.arg(
1214212165
r#"events=$1
1214312166
(cat <&3 >/dev/null; echo eof >>"$events") &
12144-
trap 'sleep 0.2; echo term >>"$events"; exit 0' TERM
12167+
trap 'sleep 0.2; echo term >>"$events"; exit 143' TERM
1214512168
echo ready >>"$events"
1214612169
while :; do sleep 0.05; done"#,
1214712170
)
@@ -12186,6 +12209,10 @@ while :; do sleep 0.05; done"#,
1218612209
Some("term"),
1218712210
"SIGTERM must reach the supervisor before its liveness pipe closes; events: {log:?}"
1218812211
);
12212+
assert!(
12213+
fs::read_to_string(&driver_log).unwrap().is_empty(),
12214+
"an orderly supervisor shutdown must not warn about lost writes"
12215+
);
1218912216
fs::remove_dir_all(&dir).ok();
1219012217
}
1219112218

@@ -12195,6 +12222,14 @@ while :; do sleep 0.05; done"#,
1219512222
async fn stop_kills_a_supervisor_that_ignores_sigterm_after_the_grace() {
1219612223
let dir = unique_temp_dir();
1219712224
fs::create_dir_all(&dir).unwrap();
12225+
let driver_log = dir.join("driver.log");
12226+
let subscriber = tracing_subscriber::fmt()
12227+
.with_max_level(tracing::Level::WARN)
12228+
.without_time()
12229+
.with_ansi(false)
12230+
.with_writer(Arc::new(fs::File::create(&driver_log).unwrap()))
12231+
.finish();
12232+
let _tracing = tracing::subscriber::set_default(subscriber);
1219812233
let events = dir.join("events");
1219912234
let (read_end, write_end) = nix::unistd::pipe().unwrap();
1220012235
for fd in [&read_end, &write_end] {
@@ -12271,6 +12306,12 @@ while :; do sleep 0.05; done"#,
1227112306
tokio::time::timeout(Duration::from_secs(5), closed)
1227212307
.await
1227312308
.expect("the liveness pipe is closed after the stop");
12309+
let driver_log = fs::read_to_string(&driver_log).unwrap();
12310+
assert!(
12311+
driver_log.contains("WARN")
12312+
&& driver_log.contains("guest filesystem flush was not confirmed"),
12313+
"a forced supervisor shutdown must warn that recent writes may be lost: {driver_log:?}"
12314+
);
1227412315
fs::remove_dir_all(&dir).ok();
1227512316
}
1227612317

0 commit comments

Comments
 (0)