Skip to content

Commit 978c175

Browse files
committed
fix(vm-driver): unpack registry images correctly and validate prepared disks
The image-prep guest expected `umoci raw unpack` to produce a bundle-style rootfs/ subdirectory, but raw unpack extracts the filesystem directly into the target. Every registry-path prep therefore FATALed after a successful unpack, and two masking defects turned that into a silent permanent failure: guest init exit codes do not survive the libkrun boundary (the VM powers off cleanly regardless), and the prepared disk was cached without validation, poisoning the content-addressed cache for the image digest. - accept the raw-unpack layout in openshell-vm-sandbox-init.sh, keeping a defensive branch for bundle-style output - validate that a freshly built prep disk contains /image-rootfs before renaming it into the cache; on failure, do not cache and surface the guest console tail instead of reporting success - size the prep disk for the payload and the unpacked rootfs coexisting (payload*4 + 1 GiB; the image file is sparse so headroom is free) Fixes NVIDIA#2358 Signed-off-by: s2cube <[email protected]>
1 parent 8cf2673 commit 978c175

3 files changed

Lines changed: 105 additions & 8 deletions

File tree

‎crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,20 @@ prepare_guest_image_rootfs() {
172172
/opt/openshell/bin/umoci raw unpack \
173173
--image "$payload_dir/oci:openshell" \
174174
"$partial_root"
175-
if [ ! -d "$partial_root/rootfs" ]; then
176-
ts "FATAL: umoci unpack did not produce rootfs directory"
175+
# `umoci raw unpack` extracts the image filesystem directly into
176+
# the target directory — there is no bundle-style rootfs/
177+
# subdirectory. Accept both layouts defensively.
178+
if [ -d "$partial_root/rootfs" ] \
179+
&& [ ! -e "$partial_root/bin" ] \
180+
&& [ ! -e "$partial_root/usr" ]; then
181+
mv "$partial_root/rootfs" "$image_root"
182+
rm -rf "$partial_root"
183+
elif [ -d "$partial_root" ]; then
184+
mv "$partial_root" "$image_root"
185+
else
186+
ts "FATAL: umoci unpack did not produce a rootfs"
177187
exit 1
178188
fi
179-
mv "$partial_root/rootfs" "$image_root"
180-
rm -rf "$partial_root"
181189
;;
182190
*)
183191
ts "FATAL: unknown guest image payload source: ${source:-missing}"

‎crates/openshell-driver-vm/src/driver.rs‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ use crate::lifecycle::{
1010
};
1111
use crate::rootfs::{
1212
clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir,
13-
extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path,
14-
set_rootfs_image_file_mode, write_rootfs_image_file,
13+
ext4_image_has_directory, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root,
14+
sandbox_guest_init_path, set_rootfs_image_file_mode, write_rootfs_image_file,
1515
};
1616
use crate::runtime::VmBackend;
1717
use bollard::Docker;
@@ -2437,6 +2437,37 @@ impl VmDriver {
24372437
return Err(err);
24382438
}
24392439

2440+
// Validate before caching: guest init exit codes do not survive the
2441+
// libkrun boundary (the VM powers off "cleanly" whatever status PID 1
2442+
// exited with), so a prep failure would otherwise cache a broken disk
2443+
// that poisons every later sandbox using this image.
2444+
let disk_to_validate = prepared_image.clone();
2445+
let has_rootfs = tokio::task::spawn_blocking(move || {
2446+
ext4_image_has_directory(&disk_to_validate, "/image-rootfs")
2447+
})
2448+
.await
2449+
.map_err(|err| Status::internal(format!("prepared image validation panicked: {err}")))?
2450+
.map_err(Status::internal)?;
2451+
if !has_rootfs {
2452+
let console = tokio::fs::read_to_string(staging_dir.join("image-prep-console.log"))
2453+
.await
2454+
.unwrap_or_default();
2455+
let tail = console
2456+
.lines()
2457+
.rev()
2458+
.take(12)
2459+
.collect::<Vec<_>>()
2460+
.into_iter()
2461+
.rev()
2462+
.collect::<Vec<_>>()
2463+
.join("\n");
2464+
let _ = tokio::fs::remove_dir_all(staging_dir).await;
2465+
return Err(Status::failed_precondition(format!(
2466+
"image-prep for '{image_ref}' produced a disk without /image-rootfs \
2467+
(not cached). guest console tail:\n{tail}"
2468+
)));
2469+
}
2470+
24402471
if tokio::fs::metadata(&image_path).await.is_ok() {
24412472
let _ = tokio::fs::remove_dir_all(staging_dir).await;
24422473
return Ok(());
@@ -4884,9 +4915,13 @@ fn prepared_image_disk_size_bytes(
48844915
.map_err(|err| format!("stat {}: {err}", rootfs_archive.display()))?
48854916
.len(),
48864917
};
4918+
// The prep disk must hold the payload AND the unpacked rootfs at the
4919+
// same time (the payload is only removed after a successful unpack), and
4920+
// compressed layers commonly expand ~2.5-3x. The image file is created
4921+
// sparse, so generous headroom costs no real disk space.
48874922
let requested = payload_size
4888-
.saturating_mul(3)
4889-
.saturating_add(512 * 1024 * 1024);
4923+
.saturating_mul(4)
4924+
.saturating_add(1024 * 1024 * 1024);
48904925
Ok(minimum_size_bytes.max(requested))
48914926
}
48924927

‎crates/openshell-driver-vm/src/rootfs.rs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,60 @@ fn sandbox_guest_user_ids(rootfs: &Path) -> Result<Option<(u32, u32)>, String> {
655655
Ok(None)
656656
}
657657

658+
/// Check (read-only, without mounting) whether an ext4 image contains a
659+
/// directory at `guest_path`.
660+
///
661+
/// Used to validate image-prep output before it is cached: guest init exit
662+
/// codes do not survive the libkrun boundary — the VM powers off "cleanly"
663+
/// whatever status PID 1 exited with — so without an explicit content check
664+
/// a failed prep would poison the prepared-image cache forever.
665+
pub fn ext4_image_has_directory(image_path: &Path, guest_path: &str) -> Result<bool, String> {
666+
let mut last_error = None;
667+
for candidate in e2fs_tool_candidates("debugfs") {
668+
let label = candidate.display().to_string();
669+
let output = Command::new(&candidate)
670+
.arg("-R")
671+
.arg(format!("stat {guest_path}"))
672+
.arg(image_path)
673+
.output();
674+
match output {
675+
Ok(output) if output.status.success() => {
676+
let stdout = String::from_utf8_lossy(&output.stdout);
677+
let stderr = String::from_utf8_lossy(&output.stderr);
678+
// debugfs exits 0 even when the target is missing; the
679+
// result is only in its output text.
680+
if stdout.contains("Type: directory") {
681+
return Ok(true);
682+
}
683+
if stdout.contains("File not found") || stderr.contains("File not found") {
684+
return Ok(false);
685+
}
686+
last_error = Some(format!(
687+
"{label} produced unrecognized output for {guest_path}\nstdout: {stdout}\nstderr: {stderr}"
688+
));
689+
}
690+
Ok(output) => {
691+
last_error = Some(format!(
692+
"{label} failed with status {}\nstderr: {}",
693+
output.status,
694+
String::from_utf8_lossy(&output.stderr)
695+
));
696+
}
697+
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
698+
last_error = Some(format!("{label} not found"));
699+
}
700+
Err(err) => {
701+
last_error = Some(format!("run {label}: {err}"));
702+
}
703+
}
704+
}
705+
Err(format!(
706+
"inspect {} for {guest_path} failed: {}. Install e2fsprogs (debugfs) and retry",
707+
image_path.display(),
708+
last_error.unwrap_or_else(|| "debugfs not found".to_string())
709+
))
710+
}
711+
658712
fn run_debugfs_batch(image_path: &Path, commands: &[String]) -> Result<(), String> {
659713
let command_path = temporary_injection_path(image_path);
660714
let mut contents = commands.join("\n");

0 commit comments

Comments
 (0)