Skip to content

Commit c1f0dd3

Browse files
committed
fix(gateway): configurable startup timeout for slow runtimes (NVIDIA#531)
Add `--timeout <seconds>` flag to `openshell gateway start` (also available via `OPENSHELL_GATEWAY_TIMEOUT` env var) that controls how long the health-check polling loop waits for the gateway to become healthy. Defaults to 180 s (unchanged behavior). K3s on Apple Silicon with Colima can take over two minutes to start; this flag lets users increase the timeout without code changes. Plumbing: `DeployOptions.startup_timeout_secs` → `wait_for_gateway_ready()` which now computes the iteration count from the timeout and the 2 s polling interval. Signed-off-by: David Gil <[email protected]> Signed-off-by: David Gil - Klaus <[email protected]>
1 parent c762e3c commit c1f0dd3

4 files changed

Lines changed: 56 additions & 5 deletions

File tree

‎crates/openshell-bootstrap/src/lib.rs‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,18 @@ pub struct DeployOptions {
119119
/// When false, an existing gateway is left as-is and deployment is
120120
/// skipped (the caller is responsible for prompting the user first).
121121
pub recreate: bool,
122+
/// Maximum time (in seconds) to wait for the gateway health check to
123+
/// report healthy. Defaults to [`DEFAULT_STARTUP_TIMEOUT_SECS`].
124+
/// Increase for slow container runtimes (e.g. Colima on Apple Silicon).
125+
pub startup_timeout_secs: u64,
122126
}
123127

128+
/// Default gateway startup timeout in seconds. Each polling iteration sleeps
129+
/// 2 s, so 180 s ≈ 90 iterations. This is generous for most Docker Desktop
130+
/// and native Linux setups but may be insufficient for emulated or
131+
/// resource-constrained runtimes.
132+
pub const DEFAULT_STARTUP_TIMEOUT_SECS: u64 = 180;
133+
124134
impl DeployOptions {
125135
pub fn new(name: impl Into<String>) -> Self {
126136
Self {
@@ -135,6 +145,7 @@ impl DeployOptions {
135145
registry_token: None,
136146
gpu: false,
137147
recreate: false,
148+
startup_timeout_secs: DEFAULT_STARTUP_TIMEOUT_SECS,
138149
}
139150
}
140151

@@ -200,6 +211,13 @@ impl DeployOptions {
200211
self.recreate = recreate;
201212
self
202213
}
214+
215+
/// Set the maximum time (in seconds) to wait for the gateway to become healthy.
216+
#[must_use]
217+
pub fn with_startup_timeout_secs(mut self, secs: u64) -> Self {
218+
self.startup_timeout_secs = secs;
219+
self
220+
}
203221
}
204222

205223
#[derive(Debug, Clone)]
@@ -264,6 +282,7 @@ where
264282
let registry_token = options.registry_token;
265283
let gpu = options.gpu;
266284
let recreate = options.recreate;
285+
let startup_timeout_secs = options.startup_timeout_secs;
267286

268287
// Wrap on_log in Arc<Mutex<>> so we can share it with pull_remote_image
269288
// which needs a 'static callback for the bollard streaming pull.
@@ -500,7 +519,8 @@ where
500519
f(msg);
501520
}
502521
};
503-
wait_for_gateway_ready(&target_docker, &name, &mut gateway_log).await?;
522+
wait_for_gateway_ready(&target_docker, &name, startup_timeout_secs, &mut gateway_log)
523+
.await?;
504524
}
505525

506526
// Create and store gateway metadata.
@@ -1077,4 +1097,16 @@ mod tests {
10771097
);
10781098
}
10791099
}
1100+
1101+
#[test]
1102+
fn deploy_options_default_startup_timeout() {
1103+
let opts = DeployOptions::new("test");
1104+
assert_eq!(opts.startup_timeout_secs, DEFAULT_STARTUP_TIMEOUT_SECS);
1105+
}
1106+
1107+
#[test]
1108+
fn deploy_options_custom_startup_timeout() {
1109+
let opts = DeployOptions::new("test").with_startup_timeout_secs(300);
1110+
assert_eq!(opts.startup_timeout_secs, 300);
1111+
}
10801112
}

‎crates/openshell-bootstrap/src/runtime.rs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,15 @@ const DNS_FAILURE_GRACE_ITERATIONS: u32 = 5;
4141
/// transient pressure can occur during image extraction on startup.
4242
const NODE_PRESSURE_GRACE_ITERATIONS: u32 = 8;
4343

44-
pub async fn wait_for_gateway_ready<F>(docker: &Docker, name: &str, mut on_log: F) -> Result<()>
44+
/// Polling interval between health check attempts.
45+
const HEALTH_POLL_INTERVAL: Duration = Duration::from_secs(2);
46+
47+
pub async fn wait_for_gateway_ready<F>(
48+
docker: &Docker,
49+
name: &str,
50+
timeout_secs: u64,
51+
mut on_log: F,
52+
) -> Result<()>
4553
where
4654
F: FnMut(String) + Send,
4755
{
@@ -54,7 +62,7 @@ where
5462
});
5563

5664
let mut recent_logs = VecDeque::with_capacity(15);
57-
let attempts = 180;
65+
let attempts = (timeout_secs / HEALTH_POLL_INTERVAL.as_secs()).max(1) as usize;
5866
let mut result = None;
5967
let mut dns_failure_seen_count: u32 = 0;
6068
let mut node_pressure_seen_count: u32 = 0;
@@ -209,7 +217,7 @@ where
209217
_ => {}
210218
}
211219

212-
tokio::time::sleep(Duration::from_secs(2)).await;
220+
tokio::time::sleep(HEALTH_POLL_INTERVAL).await;
213221
}
214222

215223
if result.is_none() {

‎crates/openshell-cli/src/main.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -809,6 +809,13 @@ enum GatewayCommands {
809809
/// NVIDIA Container Toolkit on the host.
810810
#[arg(long)]
811811
gpu: bool,
812+
813+
/// Maximum time (in seconds) to wait for the gateway to become healthy.
814+
///
815+
/// Increase for slow container runtimes such as Colima on Apple
816+
/// Silicon where K3s startup can take over two minutes.
817+
#[arg(long, default_value_t = openshell_bootstrap::DEFAULT_STARTUP_TIMEOUT_SECS, env = "OPENSHELL_GATEWAY_TIMEOUT")]
818+
timeout: u64,
812819
},
813820

814821
/// Stop the gateway (preserves state).
@@ -1561,6 +1568,7 @@ async fn main() -> Result<()> {
15611568
registry_username,
15621569
registry_token,
15631570
gpu,
1571+
timeout,
15641572
} => {
15651573
run::gateway_admin_deploy(
15661574
&name,
@@ -1574,6 +1582,7 @@ async fn main() -> Result<()> {
15741582
registry_username.as_deref(),
15751583
registry_token.as_deref(),
15761584
gpu,
1585+
timeout,
15771586
)
15781587
.await?;
15791588
}

‎crates/openshell-cli/src/run.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1356,6 +1356,7 @@ pub async fn gateway_admin_deploy(
13561356
registry_username: Option<&str>,
13571357
registry_token: Option<&str>,
13581358
gpu: bool,
1359+
startup_timeout_secs: u64,
13591360
) -> Result<()> {
13601361
let location = if remote.is_some() { "remote" } else { "local" };
13611362

@@ -1420,7 +1421,8 @@ pub async fn gateway_admin_deploy(
14201421
.with_disable_tls(disable_tls)
14211422
.with_disable_gateway_auth(disable_gateway_auth)
14221423
.with_gpu(gpu)
1423-
.with_recreate(should_recreate);
1424+
.with_recreate(should_recreate)
1425+
.with_startup_timeout_secs(startup_timeout_secs);
14241426
if let Some(opts) = remote_opts {
14251427
options = options.with_remote(opts);
14261428
}

0 commit comments

Comments
 (0)