@@ -9,14 +9,16 @@ use std::process::Stdio;
99use std:: time:: Duration ;
1010
1111use openshell_e2e:: harness:: binary:: { openshell_cmd, openshell_tty_cmd} ;
12+ use openshell_e2e:: harness:: cli:: { run_cli, wait_for_sandbox_exec_contains} ;
1213use openshell_e2e:: harness:: output:: { extract_field, strip_ansi} ;
13- use openshell_e2e:: harness:: sandbox:: { SandboxGuard , unique_sandbox_name} ;
14+ use openshell_e2e:: harness:: sandbox:: { E2E_WORKLOAD_IMAGE , SandboxGuard , unique_sandbox_name} ;
1415use serial_test:: serial;
1516use tokio:: io:: { AsyncBufReadExt , AsyncWriteExt , BufReader } ;
1617use tokio:: time:: { Instant , sleep} ;
1718
1819const SANDBOX_PRESENCE_TIMEOUT : Duration = Duration :: from_secs ( 30 ) ;
1920const SANDBOX_LIST_POLL_INTERVAL : Duration = Duration :: from_millis ( 500 ) ;
21+ const SANDBOX_RESTART_TIMEOUT : Duration = Duration :: from_secs ( 60 ) ;
2022
2123fn normalize_output ( output : & str ) -> String {
2224 let stripped = strip_ansi ( output) . replace ( '\r' , "" ) ;
@@ -1086,3 +1088,105 @@ async fn sandbox_create_with_no_keep_preserves_failure_then_cleans_up() {
10861088 ) ;
10871089 }
10881090}
1091+
1092+ async fn sandbox_details ( name : & str ) -> String {
1093+ let ( details, exit_code) = run_cli ( & [ "sandbox" , "get" , name] ) . await ;
1094+ let details = normalize_output ( & details) ;
1095+ assert_eq ! (
1096+ exit_code, 0 ,
1097+ "sandbox get should succeed for {name}:\n {details}"
1098+ ) ;
1099+ details
1100+ }
1101+
1102+ // The conformance suite has no counterpart for this test: on the VM driver the
1103+ // replacement never starts (https://github.com/NVIDIA/OpenShell/issues/5507),
1104+ // and the legacy lanes that run this file are Docker and Kubernetes.
1105+ #[ tokio:: test]
1106+ #[ serial( sandbox_lifecycle) ]
1107+ async fn on_failure_policy_replaces_runtime_and_preserves_workspace ( ) {
1108+ const SCRIPT : & str = r#"
1109+ marker=/sandbox/.openshell-restart-e2e
1110+ if [ -e "$marker" ]; then
1111+ printf 'replacement-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/replacement-run
1112+ sleep 300
1113+ else
1114+ touch "$marker"
1115+ printf 'initial-%s\n' "$(cat /proc/sys/kernel/random/uuid)" > /sandbox/initial-run
1116+ while [ ! -e /sandbox/.openshell-restart-release ]; do sleep 0.05; done
1117+ exit 17
1118+ fi
1119+ "# ;
1120+
1121+ let mut sandbox = SandboxGuard :: manage_existing ( unique_sandbox_name ( ) ) ;
1122+ let ( output, exit_code) = run_cli ( & [
1123+ "sandbox" ,
1124+ "create" ,
1125+ "--name" ,
1126+ & sandbox. name ,
1127+ "--from" ,
1128+ E2E_WORKLOAD_IMAGE ,
1129+ "--detach" ,
1130+ "--restart-policy" ,
1131+ "on-failure" ,
1132+ "--" ,
1133+ "sh" ,
1134+ "-lc" ,
1135+ SCRIPT ,
1136+ ] )
1137+ . await ;
1138+ assert_eq ! (
1139+ exit_code, 0 ,
1140+ "create sandbox with OnFailure policy: {output}"
1141+ ) ;
1142+ wait_for_sandbox_exec_contains (
1143+ & sandbox. name ,
1144+ & [ "cat" , "/sandbox/initial-run" ] ,
1145+ "initial-" ,
1146+ SANDBOX_RESTART_TIMEOUT ,
1147+ )
1148+ . await
1149+ . expect ( "initial main process should write its workspace marker" ) ;
1150+
1151+ let initial_details = sandbox_details ( & sandbox. name ) . await ;
1152+ let initial_instance = extract_field ( & initial_details, "Main process instance" )
1153+ . expect ( "initial main process instance" ) ;
1154+ sandbox
1155+ . exec ( & [ "touch" , "/sandbox/.openshell-restart-release" ] )
1156+ . await
1157+ . expect ( "release initial main process" ) ;
1158+
1159+ let deadline = Instant :: now ( ) + SANDBOX_RESTART_TIMEOUT ;
1160+ let mut last_runs = Err ( "replacement is not ready" . to_string ( ) ) ;
1161+ let final_details = loop {
1162+ let details = sandbox_details ( & sandbox. name ) . await ;
1163+ let replacement_instance = extract_field ( & details, "Main process instance" )
1164+ . is_some_and ( |instance| instance != "-" && instance != initial_instance) ;
1165+ if details. contains ( "Phase: Ready" )
1166+ && extract_field ( & details, "Restart count" ) . as_deref ( ) == Some ( "1" )
1167+ && replacement_instance
1168+ {
1169+ last_runs = sandbox
1170+ . exec ( & [ "cat" , "/sandbox/initial-run" , "/sandbox/replacement-run" ] )
1171+ . await ;
1172+ if let Ok ( runs) = & last_runs
1173+ && runs. contains ( "initial-" )
1174+ && runs. contains ( "replacement-" )
1175+ {
1176+ break details;
1177+ }
1178+ }
1179+ assert ! (
1180+ Instant :: now( ) < deadline,
1181+ "sandbox did not complete its policy-driven restart within \
1182+ {SANDBOX_RESTART_TIMEOUT:?}; last details:\n {details}\n workspace: {last_runs:?}"
1183+ ) ;
1184+ sleep ( Duration :: from_millis ( 250 ) ) . await ;
1185+ } ;
1186+
1187+ assert ! (
1188+ final_details. contains( "Restart policy: on-failure" ) ,
1189+ "restart policy should remain visible after replacement:\n {final_details}"
1190+ ) ;
1191+ sandbox. cleanup ( ) . await ;
1192+ }
0 commit comments