@@ -26,8 +26,10 @@ use tracing::{debug, info, warn};
2626// Condition reason constants shared across event-building paths.
2727const CONDITION_RUNNING : & str = "ContainerRunning" ;
2828const CONDITION_STARTING : & str = "ContainerStarting" ;
29- const CONDITION_EXITED : & str = "ContainerExited" ;
30- const CONDITION_STOPPED : & str = "ContainerStopped" ;
29+ use openshell_core:: driver_utils:: {
30+ CONDITION_EXITED , CONDITION_FAILED , CONDITION_STOPPED , EXIT_CODE_SIGTERM ,
31+ is_external_signal_exit,
32+ } ;
3133
3234pub type WatchStream =
3335 Pin < Box < dyn Stream < Item = Result < WatchSandboxesEvent , ComputeDriverError > > + Send > > ;
@@ -143,10 +145,13 @@ pub async fn start_watch(
143145 let existing = client. list_containers ( & [ LABEL_MANAGED_FILTER ] ) . await ?;
144146
145147 for entry in & existing {
146- // For running containers, use inspect to get full state including
147- // health check status — matching the same condition derivation used
148- // for live events.
149- if entry. state == "running" {
148+ // Inspect running containers for health status, and exited/stopped
149+ // containers for the exit code and OOM flag — matching the condition
150+ // derivation used for live events. The list entry carries no OOM flag,
151+ // so an already-exited container observed only through the list path
152+ // cannot distinguish an OOM kill (137 + oom flag) from a SIGKILL
153+ // restart; inspecting here lets `condition_from_state` classify it.
154+ if matches ! ( entry. state. as_str( ) , "running" | "exited" | "stopped" ) {
150155 match client. inspect_container ( & entry. id ) . await {
151156 Ok ( inspect) => {
152157 if let Some ( sandbox) = driver_sandbox_from_inspect ( & inspect) {
@@ -161,8 +166,9 @@ pub async fn start_watch(
161166 Err ( e) => {
162167 warn ! (
163168 container_id = %entry. id,
169+ state = %entry. state,
164170 error = %e,
165- "Failed to inspect running container during initial sync, falling back to list entry"
171+ "Failed to inspect container during initial sync, falling back to list entry"
166172 ) ;
167173 }
168174 }
@@ -405,7 +411,19 @@ pub fn driver_sandbox_from_list_entry(entry: &ContainerListEntry) -> Option<Driv
405411 "Container is running" . to_string ( ) ,
406412 ) ,
407413 "created" => ( "ContainerCreated" , "False" , String :: new ( ) ) ,
408- "exited" => ( CONDITION_EXITED , "False" , String :: new ( ) ) ,
414+ // The list entry carries no OOM flag, so a 137 (SIGKILL) exit cannot be
415+ // told apart from an OOM kill here. Fail closed: only a SIGTERM (143)
416+ // exit — which is never an OOM kill — is treated as a recoverable
417+ // external-signal exit from the list alone; every other exit becomes a
418+ // non-recoverable `CONDITION_FAILED`. Already-exited containers are
419+ // inspected during initial sync, so the inspect-based
420+ // `condition_from_state` path (which sees the OOM flag) classifies 137
421+ // exits and OOM kills correctly; this list path is the degraded
422+ // fallback used only when that inspect fails.
423+ "exited" if entry. exit_code == EXIT_CODE_SIGTERM => {
424+ ( CONDITION_EXITED , "False" , String :: new ( ) )
425+ }
426+ "exited" => ( CONDITION_FAILED , "False" , String :: new ( ) ) ,
409427 "stopped" => ( CONDITION_STOPPED , "False" , String :: new ( ) ) ,
410428 "removing" => ( "ContainerRemoving" , "False" , String :: new ( ) ) ,
411429 _ => ( "Unknown" , "Unknown" , String :: new ( ) ) ,
@@ -444,16 +462,30 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition {
444462 _ => ( "False" , CONDITION_STARTING , String :: new ( ) ) ,
445463 } ,
446464 "created" => ( "False" , "ContainerCreated" , String :: new ( ) ) ,
447- "exited" | "stopped" => {
465+ // An explicitly stopped container is recoverable regardless of its exit
466+ // code, matching the list-path classification and keeping the two paths
467+ // consistent.
468+ "stopped" => ( "False" , CONDITION_STOPPED , "Container stopped" . to_string ( ) ) ,
469+ "exited" => {
448470 let msg = if state. oom_killed {
449471 "Container was killed by the OOM killer" . to_string ( )
450472 } else {
451473 format ! ( "Container exited with code {}" , state. exit_code)
452474 } ;
475+ // This is the only path that sees Podman's OOM flag, so it — not the
476+ // list path — decides how a 137 exit is classified. Trust the flag:
477+ // a real OOM kill is non-recoverable `OOMKilled`, while an
478+ // external-signal exit (SIGTERM 143 / SIGKILL 137, e.g. a machine
479+ // restart) is a recoverable `CONDITION_EXITED`. If Podman ever
480+ // under-reports oom_killed, an OOM container would be restarted once
481+ // at the next gateway startup sweep — recovery is one-shot per
482+ // startup, not a restart loop.
453483 let reason = if state. oom_killed {
454484 "OOMKilled"
455- } else {
485+ } else if is_external_signal_exit ( state . exit_code ) {
456486 CONDITION_EXITED
487+ } else {
488+ CONDITION_FAILED
457489 } ;
458490 ( "False" , reason, msg)
459491 }
@@ -584,7 +616,9 @@ mod tests {
584616 }
585617
586618 #[ test]
587- fn condition_normal_exit ( ) {
619+ fn condition_self_exit_is_failed ( ) {
620+ // A small non-signal exit code is a self-exit (crash or clean exit),
621+ // not an external signal. Not eligible for startup recovery.
588622 let state = ContainerState {
589623 status : "exited" . to_string ( ) ,
590624 running : false ,
@@ -596,10 +630,31 @@ mod tests {
596630 } ;
597631 let cond = condition_from_state ( & state) ;
598632 assert_eq ! ( cond. status, "False" ) ;
599- assert_eq ! ( cond. reason, "ContainerExited " ) ;
633+ assert_eq ! ( cond. reason, "ContainerFailed " ) ;
600634 assert ! ( cond. message. contains( "code 1" ) ) ;
601635 }
602636
637+ #[ test]
638+ fn condition_external_signal_exit_is_recoverable ( ) {
639+ // SIGTERM (143) / SIGKILL (137) without OOM indicate an external signal
640+ // such as a machine or daemon restart. Eligible for startup recovery.
641+ for exit_code in [ 143 , 137 ] {
642+ let state = ContainerState {
643+ status : "exited" . to_string ( ) ,
644+ running : false ,
645+ exit_code,
646+ oom_killed : false ,
647+ health : None ,
648+ started_at : None ,
649+ finished_at : Some ( "2026-04-14T12:00:00Z" . to_string ( ) ) ,
650+ } ;
651+ let cond = condition_from_state ( & state) ;
652+ assert_eq ! ( cond. status, "False" ) ;
653+ assert_eq ! ( cond. reason, "ContainerExited" ) ;
654+ assert ! ( cond. message. contains( & format!( "code {exit_code}" ) ) ) ;
655+ }
656+ }
657+
603658 #[ test]
604659 fn short_id_truncates ( ) {
605660 assert_eq ! ( short_id( "abc123def456789" ) , "abc123def456" ) ;
@@ -632,6 +687,75 @@ mod tests {
632687 assert ! ( !status. deleting) ;
633688 }
634689
690+ fn list_entry ( state : & str , exit_code : i64 ) -> ContainerListEntry {
691+ let mut labels = HashMap :: new ( ) ;
692+ labels. insert ( LABEL_SANDBOX_ID . to_string ( ) , "test-id" . to_string ( ) ) ;
693+ labels. insert ( LABEL_SANDBOX_NAME . to_string ( ) , "test-name" . to_string ( ) ) ;
694+ labels. insert ( LABEL_SANDBOX_WORKSPACE . to_string ( ) , "default" . to_string ( ) ) ;
695+ ContainerListEntry {
696+ id : "abc123def456789" . to_string ( ) ,
697+ names : vec ! [ "openshell-sandbox-test-name" . to_string( ) ] ,
698+ state : state. to_string ( ) ,
699+ labels,
700+ ports : None ,
701+ networks : None ,
702+ exit_code,
703+ }
704+ }
705+
706+ fn list_entry_reason ( state : & str , exit_code : i64 ) -> String {
707+ let sandbox = driver_sandbox_from_list_entry ( & list_entry ( state, exit_code) )
708+ . expect ( "should produce a sandbox" ) ;
709+ sandbox. status . expect ( "should have status" ) . conditions [ 0 ]
710+ . reason
711+ . clone ( )
712+ }
713+
714+ #[ test]
715+ fn list_entry_sigterm_exit_is_recoverable ( ) {
716+ // SIGTERM (143) is never an OOM kill, so it is safe to classify as a
717+ // recoverable external-signal exit even without the OOM flag.
718+ assert_eq ! ( list_entry_reason( "exited" , 143 ) , CONDITION_EXITED ) ;
719+ }
720+
721+ #[ test]
722+ fn list_entry_sigkill_exit_fails_closed ( ) {
723+ // A 137 exit is indistinguishable from an OOM kill without the OOM flag,
724+ // which the list entry does not carry. Fail closed to a non-recoverable
725+ // failure; the inspect path reclassifies genuine SIGKILL restarts.
726+ assert_eq ! ( list_entry_reason( "exited" , 137 ) , CONDITION_FAILED ) ;
727+ }
728+
729+ #[ test]
730+ fn list_entry_self_exit_is_failed ( ) {
731+ assert_eq ! ( list_entry_reason( "exited" , 1 ) , CONDITION_FAILED ) ;
732+ assert_eq ! ( list_entry_reason( "exited" , 0 ) , CONDITION_FAILED ) ;
733+ }
734+
735+ #[ test]
736+ fn list_entry_stopped_is_recoverable ( ) {
737+ assert_eq ! ( list_entry_reason( "stopped" , 0 ) , CONDITION_STOPPED ) ;
738+ }
739+
740+ #[ test]
741+ fn condition_stopped_is_recoverable ( ) {
742+ // The inspect path must yield `CONDITION_STOPPED` for a stopped
743+ // container, matching the list path rather than folding it into the
744+ // exited/failed classification.
745+ let state = ContainerState {
746+ status : "stopped" . to_string ( ) ,
747+ running : false ,
748+ exit_code : 0 ,
749+ oom_killed : false ,
750+ health : None ,
751+ started_at : None ,
752+ finished_at : Some ( "2026-04-14T12:00:00Z" . to_string ( ) ) ,
753+ } ;
754+ let cond = condition_from_state ( & state) ;
755+ assert_eq ! ( cond. status, "False" ) ;
756+ assert_eq ! ( cond. reason, CONDITION_STOPPED ) ;
757+ }
758+
635759 #[ test]
636760 fn synthetic_inspect_failed_event_structure ( ) {
637761 // Verify the structure of an inspect-failure event by constructing one
0 commit comments