Skip to content

Commit b778f01

Browse files
committed
fix(docker): reclassify signal-killed containers as runtime restart
Mirror the Podman classification in the Docker driver so that externally signal-killed containers (exit 137/143, non-OOM) are persisted with the ContainerRuntimeRestart ready reason and become eligible for startup recovery, while ordinary exits and OOM kills stay terminal. current_snapshots now inspects EXITED containers to obtain the exit code and OOM flag, then applies the shared exit classification. Without the inspect step the list-summary path lacks an exit code, so the recovery gate could never fire for Docker. Adds unit tests covering signal-kill reclassification (137/143), ordinary exit staying terminal (exit 1), and OOM staying terminal despite exit 137. Signed-off-by: Ian Miller <[email protected]>
1 parent b58ecee commit b778f01

2 files changed

Lines changed: 149 additions & 14 deletions

File tree

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 64 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ use openshell_core::config::{
2424
};
2525
use openshell_core::driver_mounts;
2626
use openshell_core::driver_utils::{
27-
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
28-
LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH,
29-
extract_first_tar_entry, supervisor_image_should_refresh, temp_extract_container_name,
30-
validate_linux_elf_binary, write_cache_binary_atomic,
27+
CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE,
28+
LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE,
29+
SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh,
30+
temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic,
3131
};
3232
use openshell_core::gpu::{
3333
CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements,
@@ -688,10 +688,37 @@ impl DockerComputeDriver {
688688

689689
async fn current_snapshots(&self) -> Result<Vec<DriverSandbox>, Status> {
690690
let containers = self.list_managed_container_summaries().await?;
691-
let container_sandboxes = containers
692-
.iter()
693-
.filter_map(sandbox_from_container_summary)
694-
.collect::<Vec<_>>();
691+
let mut container_sandboxes = Vec::with_capacity(containers.len());
692+
for summary in &containers {
693+
let Some(mut sandbox) = sandbox_from_container_summary(summary) else {
694+
continue;
695+
};
696+
// Docker's list summary carries no exit code, so an exited
697+
// container is reported as the generic terminal `ContainerExited`.
698+
// Inspect it to tell a machine/daemon-restart signal kill apart
699+
// from an ordinary application exit, mirroring the Podman driver,
700+
// so startup recovery can revive restart victims while leaving
701+
// crashes terminal.
702+
if summary.state == Some(ContainerSummaryStateEnum::EXITED)
703+
&& let Some(container_id) = summary.id.as_deref()
704+
{
705+
match self.docker.inspect_container(container_id, None).await {
706+
Ok(inspected) => {
707+
if let Some(state) = inspected.state.as_ref() {
708+
apply_docker_exit_classification(&mut sandbox, state);
709+
}
710+
}
711+
Err(err) => {
712+
debug!(
713+
container_id,
714+
error = %err,
715+
"Could not inspect exited Docker container to classify its exit"
716+
);
717+
}
718+
}
719+
}
720+
container_sandboxes.push(sandbox);
721+
}
695722
let mut by_id = self.pending_snapshot_map().await;
696723
for sandbox in container_sandboxes {
697724
by_id.insert(sandbox.id.clone(), sandbox);
@@ -3092,6 +3119,34 @@ fn driver_status_from_summary(
30923119
}
30933120
}
30943121

3122+
/// Refine an exited Docker sandbox's `Ready` condition from inspected state.
3123+
///
3124+
/// A signal kill (exit 137/143 = SIGKILL/SIGTERM, not OOM) is the signature of
3125+
/// a machine/daemon restart terminating a running container. Reclassify it from
3126+
/// the generic terminal `ContainerExited` to the recoverable
3127+
/// `ContainerRuntimeRestart` so gateway startup can revive it. OOM kills and
3128+
/// ordinary application exits stay `ContainerExited` and terminal.
3129+
fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &ContainerState) {
3130+
if state.oom_killed == Some(true) {
3131+
return;
3132+
}
3133+
let Some(code) = state.exit_code.filter(|&code| matches!(code, 137 | 143)) else {
3134+
return;
3135+
};
3136+
let Some(condition) = sandbox
3137+
.status
3138+
.as_mut()
3139+
.and_then(|status| status.conditions.iter_mut().find(|c| c.r#type == "Ready"))
3140+
else {
3141+
return;
3142+
};
3143+
if condition.reason != CONDITION_EXITED {
3144+
return;
3145+
}
3146+
condition.reason = CONDITION_RUNTIME_RESTART.to_string();
3147+
condition.message = format!("Container terminated by signal (exit code {code})");
3148+
}
3149+
30953150
fn container_ready_condition(
30963151
state: ContainerSummaryStateEnum,
30973152
) -> (&'static str, &'static str, &'static str, bool) {
@@ -3115,12 +3170,7 @@ fn container_ready_condition(
31153170
ContainerSummaryStateEnum::PAUSED => {
31163171
("False", "ContainerPaused", "Container is paused", false)
31173172
}
3118-
ContainerSummaryStateEnum::EXITED => (
3119-
"False",
3120-
openshell_core::driver_utils::CONDITION_EXITED,
3121-
"Container exited",
3122-
false,
3123-
),
3173+
ContainerSummaryStateEnum::EXITED => ("False", CONDITION_EXITED, "Container exited", false),
31243174
ContainerSummaryStateEnum::DEAD => ("False", "ContainerDead", "Container is dead", false),
31253175
}
31263176
}

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2647,3 +2647,88 @@ fn lifecycle_fence_rejects_polled_exit_from_before_restart() {
26472647
fences.remove("sandbox-1");
26482648
assert!(fences.previous_exit("sandbox-1").is_none());
26492649
}
2650+
2651+
fn exited_sandbox_with_ready_reason(reason: &str) -> DriverSandbox {
2652+
DriverSandbox {
2653+
id: "sbx-exit".to_string(),
2654+
name: "demo".to_string(),
2655+
namespace: String::new(),
2656+
spec: None,
2657+
status: Some(DriverSandboxStatus {
2658+
sandbox_name: "demo".to_string(),
2659+
instance_id: "container-1".to_string(),
2660+
agent_fd: String::new(),
2661+
sandbox_fd: String::new(),
2662+
conditions: vec![DriverCondition {
2663+
r#type: "Ready".to_string(),
2664+
status: "False".to_string(),
2665+
reason: reason.to_string(),
2666+
message: "Container exited".to_string(),
2667+
last_transition_time: String::new(),
2668+
}],
2669+
deleting: false,
2670+
}),
2671+
workspace: String::new(),
2672+
}
2673+
}
2674+
2675+
fn ready_reason(sandbox: &DriverSandbox) -> &str {
2676+
sandbox
2677+
.status
2678+
.as_ref()
2679+
.and_then(|status| status.conditions.iter().find(|c| c.r#type == "Ready"))
2680+
.map(|c| c.reason.as_str())
2681+
.expect("Ready condition present")
2682+
}
2683+
2684+
#[test]
2685+
fn docker_signal_kill_reclassified_as_runtime_restart() {
2686+
// 137 (128+SIGKILL) and 143 (128+SIGTERM) mark an external termination —
2687+
// the signature of a machine/daemon restart — and become recoverable
2688+
// `ContainerRuntimeRestart`.
2689+
for exit_code in [137, 143] {
2690+
let mut sandbox = exited_sandbox_with_ready_reason(CONDITION_EXITED);
2691+
let state = ContainerState {
2692+
status: Some(ContainerStateStatusEnum::EXITED),
2693+
oom_killed: Some(false),
2694+
exit_code: Some(exit_code),
2695+
..Default::default()
2696+
};
2697+
apply_docker_exit_classification(&mut sandbox, &state);
2698+
assert_eq!(
2699+
ready_reason(&sandbox),
2700+
CONDITION_RUNTIME_RESTART,
2701+
"exit code {exit_code} should reclassify as runtime restart"
2702+
);
2703+
}
2704+
}
2705+
2706+
#[test]
2707+
fn docker_ordinary_exit_stays_terminal() {
2708+
// An application exit (non-zero error code) stays `ContainerExited` so its
2709+
// failure signal survives instead of being relaunched on startup.
2710+
let mut sandbox = exited_sandbox_with_ready_reason(CONDITION_EXITED);
2711+
let state = ContainerState {
2712+
status: Some(ContainerStateStatusEnum::EXITED),
2713+
oom_killed: Some(false),
2714+
exit_code: Some(1),
2715+
..Default::default()
2716+
};
2717+
apply_docker_exit_classification(&mut sandbox, &state);
2718+
assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
2719+
}
2720+
2721+
#[test]
2722+
fn docker_oom_kill_stays_terminal_despite_137() {
2723+
// An OOM kill reports exit 137 but must NOT be treated as a recoverable
2724+
// restart — it is a genuine failure and stays terminal.
2725+
let mut sandbox = exited_sandbox_with_ready_reason(CONDITION_EXITED);
2726+
let state = ContainerState {
2727+
status: Some(ContainerStateStatusEnum::EXITED),
2728+
oom_killed: Some(true),
2729+
exit_code: Some(137),
2730+
..Default::default()
2731+
};
2732+
apply_docker_exit_classification(&mut sandbox, &state);
2733+
assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
2734+
}

0 commit comments

Comments
 (0)