Skip to content

Commit 1984a72

Browse files
fix(podman): keep a polled stale exit from failing an in-progress start
The gateway's reconcile sweep applies polled snapshots without waiting for an in-progress start. During a stop/start restart, get_sandbox and list_sandboxes built the stopped workload from its list entry, so the previous run's exit was reported as ContainerExited and moved the sandbox from Starting to Error. While a start is in progress, inspect a stopped workload before reporting it. If it is running again, or still shows the exit recorded before the start, report it as ContainerStarting. A new exit, and any exit once the start has returned, is still reported. The start is tracked by a guard that clears the in-progress mark on every return path. Closes #5258 Signed-off-by: Matthew Grossman <[email protected]>
1 parent dbc0517 commit 1984a72

3 files changed

Lines changed: 391 additions & 35 deletions

File tree

‎crates/openshell-driver-podman/src/driver.rs‎

Lines changed: 166 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2002,6 +2002,7 @@ impl PodmanComputeDriver {
20022002
durable_sandbox: Option<&DriverSandbox>,
20032003
) -> Result<(), ComputeDriverError> {
20042004
let span_status = openshell_otel::ErrorStatusGuard::current();
2005+
let _start = self.lifecycle_event_fences.begin_start(sandbox_id);
20052006
// Verify before inspecting or modifying either member of the pair.
20062007
// The durable snapshot is used only if a stopped generation must be
20072008
// reconciled; it never comes from Podman's mutable inspect response.
@@ -2328,14 +2329,55 @@ impl PodmanComputeDriver {
23282329
.and_then(|inspect| driver_sandbox_from_inspect(&inspect))
23292330
.or_else(|| driver_sandbox_from_list_entry(entry)))
23302331
} else {
2331-
Ok(driver_sandbox_from_list_entry(entry))
2332+
Ok(self.driver_sandbox_from_polled_entry(entry).await)
23322333
}
23332334
}
23342335

2336+
/// Build a polled snapshot from a list entry that was not inspected as a
2337+
/// running workload. A workload whose start is in progress is reported as
2338+
/// starting unless it shows a new exit, because the gateway applies polled
2339+
/// snapshots without waiting for the start to finish.
2340+
async fn driver_sandbox_from_polled_entry(
2341+
&self,
2342+
entry: &ContainerListEntry,
2343+
) -> Option<DriverSandbox> {
2344+
if matches!(entry.state.as_str(), "exited" | "stopped")
2345+
&& let Some(sandbox_id) = entry.labels.get(LABEL_SANDBOX_ID)
2346+
&& self.lifecycle_event_fences.start_in_progress(sandbox_id)
2347+
{
2348+
match self.client.inspect_container(&entry.id).await {
2349+
Ok(inspect)
2350+
if self
2351+
.lifecycle_event_fences
2352+
.polled_workload_still_starting(sandbox_id, &inspect.state) =>
2353+
{
2354+
debug!(
2355+
sandbox_id = %sandbox_id,
2356+
container = %entry.id,
2357+
status = %inspect.state.status,
2358+
"Reporting polled sandbox as starting while its start is in progress"
2359+
);
2360+
return watcher::starting_driver_sandbox_from_list_entry(entry);
2361+
}
2362+
Ok(_) => {}
2363+
Err(error) => {
2364+
warn!(
2365+
sandbox_id = %sandbox_id,
2366+
container = %entry.id,
2367+
error = %error,
2368+
"Failed to inspect stopped container during start, falling back to list entry"
2369+
);
2370+
}
2371+
}
2372+
}
2373+
driver_sandbox_from_list_entry(entry)
2374+
}
2375+
23352376
/// List all managed sandboxes.
23362377
///
2337-
/// Only inspects running containers (to get health status). Non-running
2338-
/// containers are built directly from the list entry data.
2378+
/// Inspects running containers (to get health status) and stopped
2379+
/// containers whose start is in progress. Other containers are built
2380+
/// directly from the list entry data.
23392381
pub async fn list_sandboxes(&self) -> Result<Vec<DriverSandbox>, ComputeDriverError> {
23402382
let entries = self
23412383
.client
@@ -2365,7 +2407,7 @@ impl PodmanComputeDriver {
23652407
}
23662408
}
23672409
// Non-running containers (or inspect fallback): build from list data.
2368-
if let Some(sandbox) = driver_sandbox_from_list_entry(entry) {
2410+
if let Some(sandbox) = self.driver_sandbox_from_polled_entry(entry).await {
23692411
sandboxes.push(sandbox);
23702412
}
23712413
}
@@ -3064,6 +3106,126 @@ mod tests {
30643106
let _ = fs::remove_file(start_socket);
30653107
}
30663108

3109+
const PREVIOUS_EXIT: &str = "2026-08-12T16:39:13.123456789Z";
3110+
const EXITED_WORKLOAD_LIST: &str = r#"[{"Id":"ctr-1","Names":["openshell-sandbox-demo"],"State":"exited","Labels":{"openshell.ai/sandbox-id":"sandbox-1","openshell.ai/sandbox-name":"demo","openshell.ai/sandbox-workspace":"default"}}]"#;
3111+
3112+
fn exited_workload_inspect(finished_at: &str) -> StubResponse {
3113+
StubResponse::new(
3114+
StatusCode::OK,
3115+
format!(
3116+
r#"{{"Id":"ctr-1","Name":"openshell-sandbox-demo","State":{{"Status":"exited","Running":false,"ExitCode":0,"FinishedAt":"{finished_at}"}},"Config":{{}}}}"#
3117+
),
3118+
)
3119+
}
3120+
3121+
fn ready_condition(sandbox: Option<DriverSandbox>) -> (String, String) {
3122+
let condition = sandbox
3123+
.expect("sandbox should be reported")
3124+
.status
3125+
.expect("sandbox should have a status")
3126+
.conditions
3127+
.into_iter()
3128+
.find(|condition| condition.r#type == "Ready")
3129+
.expect("sandbox should have a Ready condition");
3130+
(condition.status, condition.reason)
3131+
}
3132+
3133+
#[tokio::test]
3134+
async fn polled_snapshot_during_restart_does_not_report_the_previous_exit() {
3135+
let (socket, requests, handle) = spawn_podman_stub(
3136+
"poll-during-start",
3137+
vec![
3138+
StubResponse::new(StatusCode::OK, EXITED_WORKLOAD_LIST),
3139+
exited_workload_inspect(PREVIOUS_EXIT),
3140+
StubResponse::new(StatusCode::OK, EXITED_WORKLOAD_LIST),
3141+
exited_workload_inspect(PREVIOUS_EXIT),
3142+
// The workload starts between the list and the inspect.
3143+
StubResponse::new(StatusCode::OK, EXITED_WORKLOAD_LIST),
3144+
StubResponse::new(
3145+
StatusCode::OK,
3146+
r#"{"Id":"ctr-1","Name":"openshell-sandbox-demo","State":{"Status":"running","Running":true,"StartedAt":"2026-08-12T16:39:14Z","FinishedAt":"2026-08-12T16:39:13.123456789Z"},"Config":{}}"#,
3147+
),
3148+
],
3149+
);
3150+
let driver = test_driver(socket.clone());
3151+
driver
3152+
.lifecycle_event_fences
3153+
.record_previous_exit("sandbox-1", Some(PREVIOUS_EXIT));
3154+
let _start = driver.lifecycle_event_fences.begin_start("sandbox-1");
3155+
let starting = ("False".to_string(), "ContainerStarting".to_string());
3156+
3157+
assert_eq!(
3158+
ready_condition(driver.get_sandbox("sandbox-1").await.unwrap()),
3159+
starting
3160+
);
3161+
let mut listed = driver.list_sandboxes().await.unwrap();
3162+
assert_eq!(listed.len(), 1);
3163+
assert_eq!(ready_condition(listed.pop()), starting);
3164+
assert_eq!(
3165+
ready_condition(driver.get_sandbox("sandbox-1").await.unwrap()),
3166+
starting
3167+
);
3168+
3169+
handle.await.unwrap();
3170+
assert_eq!(requests.lock().unwrap().len(), 6);
3171+
let _ = fs::remove_file(socket);
3172+
}
3173+
3174+
#[tokio::test]
3175+
async fn polled_snapshot_reports_a_new_exit_and_any_exit_after_start() {
3176+
let (socket, requests, handle) = spawn_podman_stub(
3177+
"poll-new-exit",
3178+
vec![
3179+
StubResponse::new(StatusCode::OK, EXITED_WORKLOAD_LIST),
3180+
exited_workload_inspect("2026-08-12T16:40:00.5Z"),
3181+
// Once the start has returned, the list entry is reported
3182+
// without an inspect, including a failed start's previous exit.
3183+
StubResponse::new(StatusCode::OK, EXITED_WORKLOAD_LIST),
3184+
StubResponse::new(StatusCode::OK, EXITED_WORKLOAD_LIST),
3185+
],
3186+
);
3187+
let driver = test_driver(socket.clone());
3188+
driver
3189+
.lifecycle_event_fences
3190+
.record_previous_exit("sandbox-1", Some(PREVIOUS_EXIT));
3191+
let exited = ("False".to_string(), "ContainerExited".to_string());
3192+
3193+
let start = driver.lifecycle_event_fences.begin_start("sandbox-1");
3194+
assert_eq!(
3195+
ready_condition(driver.get_sandbox("sandbox-1").await.unwrap()),
3196+
exited
3197+
);
3198+
drop(start);
3199+
assert_eq!(
3200+
ready_condition(driver.get_sandbox("sandbox-1").await.unwrap()),
3201+
exited
3202+
);
3203+
let mut listed = driver.list_sandboxes().await.unwrap();
3204+
assert_eq!(ready_condition(listed.pop()), exited);
3205+
3206+
handle.await.unwrap();
3207+
assert_eq!(requests.lock().unwrap().len(), 4);
3208+
let _ = fs::remove_file(socket);
3209+
}
3210+
3211+
#[tokio::test]
3212+
async fn failed_start_is_no_longer_in_progress() {
3213+
let (socket, _requests, handle) = spawn_podman_stub(
3214+
"failed-start",
3215+
vec![StubResponse::new(StatusCode::OK, "[]")],
3216+
);
3217+
let driver = test_driver(socket.clone());
3218+
3219+
let result = driver
3220+
.start_sandbox("sandbox-1", "generation-1", &[], None)
3221+
.await;
3222+
3223+
assert!(matches!(result, Err(ComputeDriverError::NotFound)));
3224+
assert!(!driver.lifecycle_event_fences.start_in_progress("sandbox-1"));
3225+
handle.await.unwrap();
3226+
let _ = fs::remove_file(socket);
3227+
}
3228+
30673229
#[tokio::test]
30683230
async fn stop_waits_for_the_container_to_leave_stopping_state() {
30693231
let (socket, requests, handle) = spawn_podman_stub(

0 commit comments

Comments
 (0)