Skip to content

Commit b013446

Browse files
committed
test(e2e): keep the on-failure restart and global settings tests in e2e/rust
Restore `on_failure_policy_replaces_runtime_and_preserves_workspace` in `sandbox_lifecycle.rs` and `settings_management.rs` as they were, and drop their conformance counterparts. The on-failure restart test ran on the Docker and Kubernetes lanes but never on VM or Podman. Ported to conformance it fails on every VM row (#5507), and an `#[ignore]` there removes the coverage it gave Docker and Kubernetes. The global settings test changes a setting for the whole gateway, so it can't share a conformance run with other tests until the runner has a test group for it (#5295). `sandbox_setting_round_trip` stays in conformance. Signed-off-by: Evan Lezar <[email protected]>
1 parent 139dc5d commit b013446

5 files changed

Lines changed: 442 additions & 155 deletions

File tree

‎e2e/rust/tests/sandbox_lifecycle.rs‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,16 @@ use std::process::Stdio;
99
use std::time::Duration;
1010

1111
use openshell_e2e::harness::binary::{openshell_cmd, openshell_tty_cmd};
12+
use openshell_e2e::harness::cli::{run_cli, wait_for_sandbox_exec_contains};
1213
use openshell_e2e::harness::output::{extract_field, strip_ansi};
13-
use openshell_e2e::harness::sandbox::{SandboxGuard, unique_sandbox_name};
14+
use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard, unique_sandbox_name};
1415
use serial_test::serial;
1516
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
1617
use tokio::time::{Instant, sleep};
1718

1819
const SANDBOX_PRESENCE_TIMEOUT: Duration = Duration::from_secs(30);
1920
const SANDBOX_LIST_POLL_INTERVAL: Duration = Duration::from_millis(500);
21+
const SANDBOX_RESTART_TIMEOUT: Duration = Duration::from_secs(60);
2022

2123
fn normalize_output(output: &str) -> String {
2224
let stripped = strip_ansi(output).replace('\r', "");
@@ -1086,3 +1088,105 @@ async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() {
10861088
);
10871089
}
10881090
}
1091+
1092+
async fn sandbox_details(name: &str) -> String {
1093+
let (details, exit_code) = run_cli(&["sandbox", "get", name]).await;
1094+
let details = normalize_output(&details);
1095+
assert_eq!(
1096+
exit_code, 0,
1097+
"sandbox get should succeed for {name}:\n{details}"
1098+
);
1099+
details
1100+
}
1101+
1102+
// The conformance suite has no counterpart for this test: on the VM driver the
1103+
// replacement never starts (https://github.com/NVIDIA/OpenShell/issues/5507),
1104+
// and the legacy lanes that run this file are Docker and Kubernetes.
1105+
#[tokio::test]
1106+
#[serial(sandbox_lifecycle)]
1107+
async fn on_failure_policy_replaces_runtime_and_preserves_workspace() {
1108+
const SCRIPT: &str = r#"
1109+
marker=/sandbox/.openshell-restart-e2e
1110+
if [ -e "$marker" ]; then
1111+
printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run
1112+
sleep 300
1113+
else
1114+
touch "$marker"
1115+
printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run
1116+
while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done
1117+
exit 17
1118+
fi
1119+
"#;
1120+
1121+
let mut sandbox = SandboxGuard::manage_existing(unique_sandbox_name());
1122+
let (output, exit_code) = run_cli(&[
1123+
"sandbox",
1124+
"create",
1125+
"--name",
1126+
&sandbox.name,
1127+
"--from",
1128+
E2E_WORKLOAD_IMAGE,
1129+
"--detach",
1130+
"--restart-policy",
1131+
"on-failure",
1132+
"--",
1133+
"sh",
1134+
"-lc",
1135+
SCRIPT,
1136+
])
1137+
.await;
1138+
assert_eq!(
1139+
exit_code, 0,
1140+
"create sandbox with OnFailure policy: {output}"
1141+
);
1142+
wait_for_sandbox_exec_contains(
1143+
&sandbox.name,
1144+
&["cat", "/sandbox/initial-run"],
1145+
"initial-",
1146+
SANDBOX_RESTART_TIMEOUT,
1147+
)
1148+
.await
1149+
.expect("initial main process should write its workspace marker");
1150+
1151+
let initial_details = sandbox_details(&sandbox.name).await;
1152+
let initial_instance = extract_field(&initial_details, "Main process instance")
1153+
.expect("initial main process instance");
1154+
sandbox
1155+
.exec(&["touch", "/sandbox/.openshell-restart-release"])
1156+
.await
1157+
.expect("release initial main process");
1158+
1159+
let deadline = Instant::now() + SANDBOX_RESTART_TIMEOUT;
1160+
let mut last_runs = Err("replacement is not ready".to_string());
1161+
let final_details = loop {
1162+
let details = sandbox_details(&sandbox.name).await;
1163+
let replacement_instance = extract_field(&details, "Main process instance")
1164+
.is_some_and(|instance| instance != "-" && instance != initial_instance);
1165+
if details.contains("Phase: Ready")
1166+
&& extract_field(&details, "Restart count").as_deref() == Some("1")
1167+
&& replacement_instance
1168+
{
1169+
last_runs = sandbox
1170+
.exec(&["cat", "/sandbox/initial-run", "/sandbox/replacement-run"])
1171+
.await;
1172+
if let Ok(runs) = &last_runs
1173+
&& runs.contains("initial-")
1174+
&& runs.contains("replacement-")
1175+
{
1176+
break details;
1177+
}
1178+
}
1179+
assert!(
1180+
Instant::now() < deadline,
1181+
"sandbox did not complete its policy-driven restart within \
1182+
{SANDBOX_RESTART_TIMEOUT:?}; last details:\n{details}\nworkspace: {last_runs:?}"
1183+
);
1184+
sleep(Duration::from_millis(250)).await;
1185+
};
1186+
1187+
assert!(
1188+
final_details.contains("Restart policy: on-failure"),
1189+
"restart policy should remain visible after replacement:\n{final_details}"
1190+
);
1191+
sandbox.cleanup().await;
1192+
}

0 commit comments

Comments
 (0)