Skip to content

Commit 9ec8a59

Browse files
committed
fix(vm-driver): match the gateway generation after a gateway restart
On first launch the driver recorded a random token as its boundary generation marker. After a gateway restart the driver relaunches the VM and the gateway's start sweep sends StartSandbox with the sandbox's runtime generation. The marker could never equal it, so the start was rejected and a sandbox that had never been stopped ended in Error with reason StartFailed. Record the launch authentication's generation on first launch. For sandboxes created before this change, also accept the generation recorded in the persisted launch authentication. Signed-off-by: Naum Khart <[email protected]>
1 parent be7af99 commit 9ec8a59

1 file changed

Lines changed: 141 additions & 19 deletions

File tree

‎crates/openshell-driver-vm/src/driver.rs‎

Lines changed: 141 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1378,19 +1378,11 @@ impl VmDriver {
13781378
let image_disk = image_plan.image_disk;
13791379
let owner_source_disk = image_disk.as_ref().unwrap_or(&root_disk).clone();
13801380
let overlay_disk = disk_paths.overlay_disk;
1381-
let boundary_generation =
1382-
match tokio::fs::read_to_string(state_dir.join(HOST_BOUNDARY_GENERATION_FILE)).await {
1383-
Ok(generation) if !generation.trim().is_empty() => generation.trim().to_string(),
1384-
Ok(_) => random_boundary_token(),
1385-
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1386-
random_boundary_token()
1387-
}
1388-
Err(error) => {
1389-
return Err(Status::internal(format!(
1390-
"read VM boundary generation: {error}"
1391-
)));
1392-
}
1393-
};
1381+
let boundary_generation = launch_boundary_generation(
1382+
&state_dir,
1383+
&launch_authentication.supervisor.runtime_generation,
1384+
)
1385+
.await?;
13941386
self.publish_platform_event(
13951387
sandbox.id.clone(),
13961388
platform_event(
@@ -1918,7 +1910,9 @@ impl VmDriver {
19181910
"read active VM sandbox generation: {error}"
19191911
))
19201912
})?;
1921-
if active_generation.trim() != generation.as_str() {
1913+
if active_generation.trim() != generation.as_str()
1914+
&& !persisted_launch_generation_matches(&state_dir, &generation).await
1915+
{
19221916
return Err(Status::failed_precondition(format!(
19231917
"VM sandbox is already running generation {}",
19241918
active_generation.trim()
@@ -5904,12 +5898,43 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap<String, String> {
59045898
environment
59055899
}
59065900

5907-
fn random_boundary_token() -> String {
5908-
let mut token = String::with_capacity(64);
5909-
for byte in rand::random::<[u8; 32]>() {
5910-
write!(&mut token, "{byte:02x}").expect("writing to String cannot fail");
5901+
/// Generation that names a launch's boundary material. An explicit start
5902+
/// persists the gateway's generation before provisioning. A first launch has
5903+
/// no marker yet and takes the generation from its launch authentication, so
5904+
/// the marker matches what the gateway sends on a later start.
5905+
async fn launch_boundary_generation(
5906+
state_dir: &Path,
5907+
launch_generation: &openshell_core::sandbox_generation::SandboxGenerationId,
5908+
) -> Result<String, Status> {
5909+
match tokio::fs::read_to_string(state_dir.join(HOST_BOUNDARY_GENERATION_FILE)).await {
5910+
Ok(generation) if !generation.trim().is_empty() => Ok(generation.trim().to_string()),
5911+
Ok(_) => Ok(launch_generation.to_string()),
5912+
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
5913+
Ok(launch_generation.to_string())
5914+
}
5915+
Err(error) => Err(Status::internal(format!(
5916+
"read VM boundary generation: {error}"
5917+
))),
59115918
}
5912-
token
5919+
}
5920+
5921+
/// Sandboxes first launched before the marker carried the gateway generation
5922+
/// hold a driver-local token there. Their gateway generation is still recorded
5923+
/// in the persisted launch authentication.
5924+
async fn persisted_launch_generation_matches(
5925+
state_dir: &Path,
5926+
generation: &openshell_core::sandbox_generation::SandboxGenerationId,
5927+
) -> bool {
5928+
let Ok(sandbox) = read_sandbox_request(&state_dir.join(SANDBOX_REQUEST_FILE)).await else {
5929+
return false;
5930+
};
5931+
decode_launch_authentication(
5932+
sandbox
5933+
.spec
5934+
.as_ref()
5935+
.map_or(&[], |spec| spec.launch_authentication.as_slice()),
5936+
)
5937+
.is_ok_and(|authentication| authentication.supervisor.runtime_generation == *generation)
59135938
}
59145939

59155940
fn build_guest_environment(sandbox: &Sandbox, config: &VmDriverConfig) -> Vec<String> {
@@ -9676,6 +9701,103 @@ mod tests {
96769701
task.abort();
96779702
}
96789703

9704+
#[tokio::test]
9705+
async fn first_launch_records_gateway_generation_as_boundary_marker() {
9706+
let temp = tempfile::tempdir().unwrap();
9707+
let launch_generation =
9708+
openshell_core::sandbox_generation::SandboxGenerationId::parse("generation-1")
9709+
.expect("runtime generation");
9710+
9711+
assert_eq!(
9712+
launch_boundary_generation(temp.path(), &launch_generation)
9713+
.await
9714+
.unwrap(),
9715+
"generation-1"
9716+
);
9717+
9718+
tokio::fs::write(
9719+
temp.path().join(HOST_BOUNDARY_GENERATION_FILE),
9720+
b"g0000000000000002\n",
9721+
)
9722+
.await
9723+
.unwrap();
9724+
assert_eq!(
9725+
launch_boundary_generation(temp.path(), &launch_generation)
9726+
.await
9727+
.unwrap(),
9728+
"g0000000000000002",
9729+
"an explicit start's persisted generation must win"
9730+
);
9731+
}
9732+
9733+
#[tokio::test]
9734+
async fn already_running_start_matches_gateway_generation_of_first_launch() {
9735+
let temp = tempfile::tempdir().unwrap();
9736+
let mut driver = test_driver_with_extensions(LifecycleExtensionRegistry::new());
9737+
driver.config.state_dir = temp.path().to_path_buf();
9738+
let sandbox = Sandbox {
9739+
id: "sandbox-first-launch".to_string(),
9740+
name: "first-launch".to_string(),
9741+
spec: Some(SandboxSpec {
9742+
launch_authentication: test_launch_authentication("existing").0,
9743+
..Default::default()
9744+
}),
9745+
..Default::default()
9746+
};
9747+
let state_dir = temp.path().join("sandboxes").join(&sandbox.id);
9748+
create_private_dir_all(&state_dir).await.unwrap();
9749+
write_sandbox_request(&state_dir, &sandbox).await.unwrap();
9750+
// Sandboxes first launched before the marker carried the gateway
9751+
// generation hold a random token here.
9752+
tokio::fs::write(
9753+
state_dir.join(HOST_BOUNDARY_GENERATION_FILE),
9754+
"5f".repeat(32),
9755+
)
9756+
.await
9757+
.unwrap();
9758+
let provisioning_task = tokio::spawn(std::future::pending());
9759+
let snapshot = sandbox_snapshot(&sandbox, provisioning_condition(), false);
9760+
driver.registry.lock().await.insert(
9761+
sandbox.id.clone(),
9762+
SandboxRecord {
9763+
snapshot,
9764+
state_dir,
9765+
process: None,
9766+
provisioning_task: Some(provisioning_task),
9767+
preparation: None,
9768+
gpu_bdf: None,
9769+
deleting: false,
9770+
},
9771+
);
9772+
9773+
driver
9774+
.start_sandbox(&sandbox.id, &sandbox.name, "generation-1", Vec::new())
9775+
.await
9776+
.expect("start for the generation the sandbox was launched with must match");
9777+
9778+
let error = driver
9779+
.start_sandbox(&sandbox.id, &sandbox.name, "g0000000000000002", Vec::new())
9780+
.await
9781+
.expect_err("start for another generation must still be rejected");
9782+
assert_eq!(error.code(), Code::FailedPrecondition);
9783+
assert!(
9784+
error
9785+
.message()
9786+
.starts_with("VM sandbox is already running generation"),
9787+
"unexpected message: {}",
9788+
error.message()
9789+
);
9790+
9791+
let task = driver
9792+
.registry
9793+
.lock()
9794+
.await
9795+
.remove(&sandbox.id)
9796+
.and_then(|record| record.provisioning_task)
9797+
.unwrap();
9798+
task.abort();
9799+
}
9800+
96799801
#[tokio::test]
96809802
async fn blocked_worker_does_not_delay_independent_image_and_overlay_requests() {
96819803
let root = tempfile::tempdir().unwrap();

0 commit comments

Comments
 (0)