Skip to content

Commit a0604d1

Browse files
committed
fix(vm): warn about unconfirmed flush after supervisor kill
Preserve the supervisor exit status so a planned stop warns when a signal killed the supervisor and guest writeback was not confirmed. Keep normal workload exit codes quiet, including 143 after a handled SIGTERM. Extend the existing process tests to check the warning after a forced kill and its absence after an orderly shutdown. Refs NVIDIA#4185 Signed-off-by: Shiju <[email protected]>
1 parent d42d056 commit a0604d1

1 file changed

Lines changed: 48 additions & 7 deletions

File tree

‎crates/openshell-driver-vm/src/driver.rs‎

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,11 @@ use std::os::fd::AsRawFd as _;
8686
use std::os::fd::OwnedFd;
8787
#[cfg(unix)]
8888
use std::os::unix::fs::PermissionsExt;
89+
#[cfg(unix)]
90+
use std::os::unix::process::ExitStatusExt as _;
8991
use std::path::{Component, Path, PathBuf};
9092
use std::pin::Pin;
91-
use std::process::Stdio;
93+
use std::process::{ExitStatus, Stdio};
9294
use std::sync::Arc;
9395
use std::sync::atomic::{AtomicU64, Ordering};
9496
use std::time::Duration;
@@ -7166,13 +7168,15 @@ const _: () = assert!(
71667168
);
71677169

71687170
async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
7169-
terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE).await
7171+
terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE)
7172+
.await
7173+
.map(|_| ())
71707174
}
71717175

71727176
async fn terminate_vm_process_within(
71737177
child: &mut Child,
71747178
grace: Duration,
7175-
) -> Result<(), std::io::Error> {
7179+
) -> Result<ExitStatus, std::io::Error> {
71767180
if let Some(pid) = child.id()
71777181
&& let Err(err) = kill(Pid::from_raw(pid.cast_signed()), Signal::SIGTERM)
71787182
&& err != Errno::ESRCH
@@ -7183,11 +7187,10 @@ async fn terminate_vm_process_within(
71837187
}
71847188

71857189
match tokio::time::timeout(grace, child.wait()).await {
7186-
Ok(Ok(_)) => Ok(()),
7187-
Ok(Err(err)) => Err(err),
7190+
Ok(status) => status,
71887191
Err(_) => {
71897192
child.kill().await?;
7190-
child.wait().await.map(|_| ())
7193+
child.wait().await
71917194
}
71927195
}
71937196
}
@@ -7207,8 +7210,20 @@ async fn terminate_sandbox_processes_within(
72077210
process: &mut VmProcess,
72087211
supervisor_grace: Duration,
72097212
) -> Result<(), std::io::Error> {
7213+
let supervisor_pid = process.supervisor.id();
72107214
let supervisor_error = terminate_vm_process_within(&mut process.supervisor, supervisor_grace)
72117215
.await
7216+
.inspect(|status| {
7217+
// An orderly shutdown returns the workload's exit code, which can
7218+
// be nonzero even after the guest acknowledges termination.
7219+
if status.signal().is_some() {
7220+
warn!(
7221+
?supervisor_pid,
7222+
%status,
7223+
"VM host supervisor was terminated by a signal; guest filesystem flush was not confirmed and recent writes may be lost"
7224+
);
7225+
}
7226+
})
72127227
.err();
72137228
process.supervisor_liveness.take();
72147229
let vm_error = terminate_vm_process(&mut process.child).await.err();
@@ -11734,6 +11749,14 @@ mod tests {
1173411749
async fn stop_signals_supervisor_before_closing_its_liveness_pipe() {
1173511750
let dir = unique_temp_dir();
1173611751
fs::create_dir_all(&dir).unwrap();
11752+
let driver_log = dir.join("driver.log");
11753+
let subscriber = tracing_subscriber::fmt()
11754+
.with_max_level(tracing::Level::WARN)
11755+
.without_time()
11756+
.with_ansi(false)
11757+
.with_writer(Arc::new(fs::File::create(&driver_log).unwrap()))
11758+
.finish();
11759+
let _tracing = tracing::subscriber::set_default(subscriber);
1173711760
let events = dir.join("events");
1173811761
let (read_end, write_end) = nix::unistd::pipe().unwrap();
1173911762
for fd in [&read_end, &write_end] {
@@ -11750,7 +11773,7 @@ mod tests {
1175011773
.arg(
1175111774
r#"events=$1
1175211775
(cat <&3 >/dev/null; echo eof >>"$events") &
11753-
trap 'sleep 0.2; echo term >>"$events"; exit 0' TERM
11776+
trap 'sleep 0.2; echo term >>"$events"; exit 143' TERM
1175411777
echo ready >>"$events"
1175511778
while :; do sleep 0.05; done"#,
1175611779
)
@@ -11795,6 +11818,10 @@ while :; do sleep 0.05; done"#,
1179511818
Some("term"),
1179611819
"SIGTERM must reach the supervisor before its liveness pipe closes; events: {log:?}"
1179711820
);
11821+
assert!(
11822+
fs::read_to_string(&driver_log).unwrap().is_empty(),
11823+
"an orderly supervisor shutdown must not warn about lost writes"
11824+
);
1179811825
fs::remove_dir_all(&dir).ok();
1179911826
}
1180011827

@@ -11804,6 +11831,14 @@ while :; do sleep 0.05; done"#,
1180411831
async fn stop_kills_a_supervisor_that_ignores_sigterm_after_the_grace() {
1180511832
let dir = unique_temp_dir();
1180611833
fs::create_dir_all(&dir).unwrap();
11834+
let driver_log = dir.join("driver.log");
11835+
let subscriber = tracing_subscriber::fmt()
11836+
.with_max_level(tracing::Level::WARN)
11837+
.without_time()
11838+
.with_ansi(false)
11839+
.with_writer(Arc::new(fs::File::create(&driver_log).unwrap()))
11840+
.finish();
11841+
let _tracing = tracing::subscriber::set_default(subscriber);
1180711842
let events = dir.join("events");
1180811843
let (read_end, write_end) = nix::unistd::pipe().unwrap();
1180911844
for fd in [&read_end, &write_end] {
@@ -11880,6 +11915,12 @@ while :; do sleep 0.05; done"#,
1188011915
tokio::time::timeout(Duration::from_secs(5), closed)
1188111916
.await
1188211917
.expect("the liveness pipe is closed after the stop");
11918+
let driver_log = fs::read_to_string(&driver_log).unwrap();
11919+
assert!(
11920+
driver_log.contains("WARN")
11921+
&& driver_log.contains("guest filesystem flush was not confirmed"),
11922+
"a forced supervisor shutdown must warn that recent writes may be lost: {driver_log:?}"
11923+
);
1188311924
fs::remove_dir_all(&dir).ok();
1188411925
}
1188511926

0 commit comments

Comments
 (0)