Skip to content

Commit 50c43f1

Browse files
committed
fix(podman): tolerate shutdown transport closes
Signed-off-by: Evan Lezar <[email protected]>
1 parent d4cd37b commit 50c43f1

13 files changed

Lines changed: 593 additions & 70 deletions

File tree

‎crates/openshell-core/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ pub mod secrets;
3939
pub mod settings;
4040
pub mod telemetry;
4141
pub mod time;
42+
pub mod transport_errors;
4243

4344
pub use config::{
4445
ComputeDriverKind, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride,
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Shared classification for transport closes that are expected during shutdown.
5+
6+
use std::io::ErrorKind;
7+
8+
/// Returns `true` when a gRPC status represents the peer closing transport.
9+
///
10+
/// Callers must still decide whether shutdown is already in progress before
11+
/// downgrading the error. Outside shutdown these statuses remain failures.
12+
#[must_use]
13+
pub fn is_expected_transport_close_status(status: &tonic::Status) -> bool {
14+
matches!(
15+
status.code(),
16+
tonic::Code::Cancelled
17+
| tonic::Code::Unavailable
18+
| tonic::Code::Unknown
19+
| tonic::Code::Internal
20+
) && is_expected_transport_close_message(&status.to_string())
21+
}
22+
23+
/// Returns `true` when an error looks like an expected transport close.
24+
///
25+
/// This helper intentionally matches only transport-lifecycle failures such as
26+
/// HTTP/2 close races, broken pipes, and reset connections.
27+
#[must_use]
28+
pub fn is_expected_transport_close_error(error: &(dyn std::error::Error + 'static)) -> bool {
29+
if let Some(status) = error.downcast_ref::<tonic::Status>() {
30+
return is_expected_transport_close_status(status);
31+
}
32+
if let Some(io) = error.downcast_ref::<std::io::Error>() {
33+
return matches!(
34+
io.kind(),
35+
ErrorKind::BrokenPipe
36+
| ErrorKind::ConnectionAborted
37+
| ErrorKind::ConnectionReset
38+
| ErrorKind::UnexpectedEof
39+
);
40+
}
41+
is_expected_transport_close_message(&error.to_string())
42+
}
43+
44+
/// Returns `true` for known text fragments emitted by tonic, hyper, h2, and
45+
/// OS I/O errors when the peer closes a stream or connection.
46+
#[must_use]
47+
pub fn is_expected_transport_close_message(message: &str) -> bool {
48+
let msg = message.to_ascii_lowercase();
49+
msg.contains("h2 protocol error")
50+
|| msg.contains("broken pipe")
51+
|| msg.contains("connection reset")
52+
|| msg.contains("connection aborted")
53+
|| msg.contains("connection closed")
54+
|| msg.contains("connection error")
55+
|| msg.contains("error reading a body from connection")
56+
|| msg.contains("peer closed connection")
57+
|| msg.contains("unexpected eof")
58+
|| msg.contains("cancelled")
59+
|| msg.contains("canceled")
60+
}
61+
62+
#[cfg(test)]
63+
mod tests {
64+
use super::*;
65+
66+
#[test]
67+
fn classifies_expected_transport_close_statuses() {
68+
for status in [
69+
tonic::Status::unknown("h2 protocol error: error reading a body from connection"),
70+
tonic::Status::unavailable("connection reset by peer"),
71+
tonic::Status::cancelled("operation cancelled"),
72+
tonic::Status::internal("broken pipe"),
73+
] {
74+
assert!(
75+
is_expected_transport_close_status(&status),
76+
"status should be expected during shutdown: {status}"
77+
);
78+
}
79+
}
80+
81+
#[test]
82+
fn keeps_unexpected_statuses_out_of_shutdown_filter() {
83+
for status in [
84+
tonic::Status::internal("policy evaluation failed"),
85+
tonic::Status::invalid_argument("bad relay frame"),
86+
tonic::Status::permission_denied("cross-sandbox relay"),
87+
] {
88+
assert!(
89+
!is_expected_transport_close_status(&status),
90+
"status should stay unexpected: {status}"
91+
);
92+
}
93+
}
94+
95+
#[test]
96+
fn classifies_io_transport_closes() {
97+
for kind in [
98+
ErrorKind::BrokenPipe,
99+
ErrorKind::ConnectionAborted,
100+
ErrorKind::ConnectionReset,
101+
ErrorKind::UnexpectedEof,
102+
] {
103+
let error = std::io::Error::new(kind, "peer closed");
104+
assert!(is_expected_transport_close_error(&error));
105+
}
106+
}
107+
}

‎crates/openshell-driver-podman/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ Podman resources after out-of-band container removal or label drift.
339339
| `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name. |
340340
| `OPENSHELL_PODMAN_HOST_GATEWAY_IP` | `--host-gateway-ip` | empty on Linux, `192.168.127.254` on macOS | Host gateway IP used for sandbox host aliases. Empty uses Podman's `host-gateway` resolver. |
341341
| `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Supervisor Unix socket path in `PodmanComputeConfig`. |
342-
| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `10` | Container stop timeout in seconds. |
342+
| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `45` | Container stop timeout in seconds. |
343343
| `OPENSHELL_SANDBOX_PIDS_LIMIT` | `--sandbox-pids-limit` | `2048` | Podman cgroup PID limit for sandbox containers. Set `0` to inherit Podman's runtime/default PID limit. |
344344
| `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `ghcr.io/nvidia/openshell/supervisor:latest` through the gateway, required standalone | OCI image containing the supervisor binary. |
345345
| `OPENSHELL_PODMAN_TLS_CA` | `--podman-tls-ca` | unset | Host path to the CA certificate mounted for sandbox mTLS. |

‎crates/openshell-driver-podman/src/config.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
// SPDX-License-Identifier: Apache-2.0
33

4-
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
54
use std::net::IpAddr;
65
use std::path::PathBuf;
76
use std::str::FromStr;
87

98
/// Default Podman bridge network name.
109
pub const DEFAULT_NETWORK_NAME: &str = "openshell";
1110
pub const MACOS_PODMAN_MACHINE_HOST_GATEWAY_IP: &str = "192.168.127.254";
11+
/// Default Podman stop timeout in seconds (SIGTERM → SIGKILL).
12+
pub const DEFAULT_PODMAN_STOP_TIMEOUT_SECS: u32 = 45;
1213

1314
// Re-export the shared default so existing imports inside this crate keep working.
1415
pub use openshell_core::config::DEFAULT_SANDBOX_PIDS_LIMIT;
@@ -366,7 +367,7 @@ impl Default for PodmanComputeConfig {
366367
sandbox_ssh_socket_path: "/run/openshell/ssh.sock".to_string(),
367368
network_name: DEFAULT_NETWORK_NAME.to_string(),
368369
host_gateway_ip: Self::default_host_gateway_ip(),
369-
stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS,
370+
stop_timeout_secs: DEFAULT_PODMAN_STOP_TIMEOUT_SECS,
370371
supervisor_image: openshell_core::config::default_supervisor_image(),
371372
guest_tls_ca: None,
372373
guest_tls_cert: None,
@@ -428,6 +429,12 @@ mod tests {
428429
);
429430
}
430431

432+
#[test]
433+
fn default_config_sets_podman_stop_timeout() {
434+
let cfg = PodmanComputeConfig::default();
435+
assert_eq!(cfg.stop_timeout_secs, DEFAULT_PODMAN_STOP_TIMEOUT_SECS);
436+
}
437+
431438
#[test]
432439
fn default_config_sets_driver_owned_pids_limit() {
433440
let cfg = PodmanComputeConfig::default();

‎crates/openshell-driver-podman/src/main.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ use tracing::info;
99
use tracing_subscriber::EnvFilter;
1010

1111
use openshell_core::VERSION;
12-
use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
1312
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
1413
use openshell_driver_podman::config::{
15-
DEFAULT_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, ImagePullPolicy,
14+
DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT,
15+
ImagePullPolicy,
1616
};
1717
use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver};
1818

@@ -76,7 +76,7 @@ struct Args {
7676
network_name: String,
7777

7878
/// Container stop timeout in seconds (SIGTERM → SIGKILL).
79-
#[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_STOP_TIMEOUT_SECS)]
79+
#[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)]
8080
stop_timeout: u32,
8181

8282
/// Container cgroup PID limit for sandbox containers. Set 0 to inherit

‎crates/openshell-sandbox/src/lib.rs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ mod sidecar_control;
1717
use miette::{IntoDiagnostic, Result, WrapErr};
1818
use std::future::Future;
1919
use std::sync::Arc;
20-
use std::sync::atomic::AtomicU32;
20+
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
2121
use std::time::Duration;
2222
use tracing::{debug, info, warn};
2323

@@ -95,7 +95,7 @@ pub async fn run_sandbox(
9595
_health_check: bool,
9696
_health_port: u16,
9797
inference_routes: Option<String>,
98-
ocsf_enabled: Arc<std::sync::atomic::AtomicBool>,
98+
ocsf_enabled: Arc<AtomicBool>,
9999
network_enabled: bool,
100100
process_enabled: bool,
101101
upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs,
@@ -942,8 +942,9 @@ fn spawn_sidecar_entrypoint_handler(
942942
tokio::spawn(async move {
943943
let mut session_started = false;
944944
let mut trusted_supervisor_pid = None;
945+
let terminating = Arc::new(AtomicBool::new(false));
945946
while let Some(started) = entrypoint_rx.recv().await {
946-
entrypoint_pid.store(started.pid, std::sync::atomic::Ordering::Release);
947+
entrypoint_pid.store(started.pid, Ordering::Release);
947948
if started.start_session {
948949
info!(
949950
pid = started.pid,
@@ -990,11 +991,13 @@ fn spawn_sidecar_entrypoint_handler(
990991
trusted_ssh_socket_path.clone(),
991992
None,
992993
Some(supervisor_pid),
994+
Arc::clone(&terminating),
993995
);
994996
session_started = true;
995997
info!("sidecar supervisor session task spawned");
996998
}
997999
}
1000+
terminating.store(true, Ordering::Release);
9981001
});
9991002
}
10001003

@@ -2521,7 +2524,7 @@ struct PolicyPollLoopContext {
25212524
loaded_policy_origin: LoadedPolicyOrigin,
25222525
entrypoint_pid: Arc<AtomicU32>,
25232526
interval_secs: u64,
2524-
ocsf_enabled: Arc<std::sync::atomic::AtomicBool>,
2527+
ocsf_enabled: Arc<AtomicBool>,
25252528
provider_credentials: ProviderCredentialState,
25262529
policy_local_ctx: Option<Arc<openshell_supervisor_network::policy_local::PolicyLocalContext>>,
25272530
agent_proposals: AgentProposals,
@@ -2970,7 +2973,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> {
29702973
}
29712974

29722975
fn apply_ocsf_json_setting(
2973-
enabled: &std::sync::atomic::AtomicBool,
2976+
enabled: &AtomicBool,
29742977
settings: &std::collections::HashMap<String, openshell_core::proto::EffectiveSetting>,
29752978
) {
29762979
use std::sync::atomic::Ordering;

‎crates/openshell-server/src/grpc/mod.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -716,8 +716,7 @@ impl OpenShell for OpenShellService {
716716
&self,
717717
request: Request<tonic::Streaming<RelayFrame>>,
718718
) -> Result<Response<Self::RelayStreamStream>, Status> {
719-
crate::supervisor_session::handle_relay_stream(&self.state.supervisor_sessions, request)
720-
.await
719+
crate::supervisor_session::handle_relay_stream_for_state(&self.state, request).await
721720
}
722721

723722
// --- Workspace management ---

‎crates/openshell-server/src/lib.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -171,12 +171,7 @@ fn is_benign_tls_handshake_failure(error: &std::io::Error) -> bool {
171171
}
172172

173173
fn is_benign_connection_close(error: &(dyn std::error::Error + 'static)) -> bool {
174-
let msg = error.to_string();
175-
msg.contains("connection closed")
176-
|| msg.contains("connection reset")
177-
|| msg.contains("connection error")
178-
|| msg.contains("error reading a body from connection")
179-
|| msg.contains("broken pipe")
174+
openshell_core::transport_errors::is_expected_transport_close_error(error)
180175
}
181176

182177
impl ServerState {

0 commit comments

Comments
 (0)