Skip to content

Commit 22209de

Browse files
shiju-nveformat
andauthored
feat(compute)!: configure provisioning and bootstrap timeouts (NVIDIA#5321)
* feat(compute)!: configure provisioning and bootstrap timeouts Continue Mike Hepburn's configuration contribution from NVIDIA#4575 (de949b7), preserving its attribution and original DCO provenance. Adapt it to current session admission, registered and shared backends, bound configuration input, and use the configured budget for the initial routed connection. Add focused timeout and configuration regressions. These adaptations are Shiju's work and are not represented as endorsed by the original contributor. BREAKING CHANGE: External Rust configuration struct literals must supply the new timeout fields. PreparedBackend discovery implementors and callers of public supervisor/runtime policy-discovery methods must accept or pass the timeout argument. Target the next qualified minor release, not a patch release or backport. Existing TOML defaults and normal reconnect limits are retained. Co-authored-by: Mike Hepburn <[email protected]> Signed-off-by: Mike Hepburn <[email protected]> Signed-off-by: Shiju <[email protected]> * fix(compute): correct configurable startup window diagnostics Signed-off-by: Shiju <[email protected]> * fix(compute): use seconds for operator startup timeouts Signed-off-by: Shiju <[email protected]> * docs(gateway): relocate startup timeout guidance Signed-off-by: Shiju <[email protected]> --------- Signed-off-by: Mike Hepburn <[email protected]> Signed-off-by: Shiju <[email protected]> Co-authored-by: Mike Hepburn <[email protected]>
1 parent f3d9765 commit 22209de

33 files changed

Lines changed: 934 additions & 166 deletions

File tree

‎crates/openshell-core/src/config.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,11 @@ pub struct Config {
339339
/// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive.
340340
pub image_preparation_timeout_seconds: u32,
341341

342+
/// Provisioning repair window, in seconds. The first authenticated
343+
/// supervisor configuration report starts admission repair, which stays
344+
/// armed until Ready. Must be between 1 and 86400, inclusive.
345+
pub provisioning_repair_window_seconds: i64,
346+
342347
/// Maximum gRPC requests allowed per rate-limit window.
343348
///
344349
/// When paired with [`Self::grpc_rate_limit_window_secs`], positive values
@@ -1061,6 +1066,7 @@ impl Config {
10611066
max_ssh_connections_per_sandbox: DEFAULT_MAX_CONNECTIONS_PER_SANDBOX,
10621067
max_service_connections_per_sandbox: DEFAULT_MAX_CONNECTIONS_PER_SANDBOX,
10631068
image_preparation_timeout_seconds: 1800,
1069+
provisioning_repair_window_seconds: 300,
10641070
grpc_rate_limit_requests: None,
10651071
grpc_rate_limit_window_secs: None,
10661072
service_routing: ServiceRoutingConfig::default(),

‎crates/openshell-driver-kubernetes/src/config.rs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,17 @@ pub struct KubernetesComputeConfig {
346346
pub supervisor_image_pull_policy: Option<KubernetesImagePullPolicy>,
347347
/// Cross-pod sandbox/supervisor settings.
348348
pub sandbox_runtime: KubernetesSandboxRuntimeConfig,
349+
/// Bound on how long a crash-interrupted, fail-closed sandbox runtime
350+
/// bootstrap may remain stranded before the driver reclaims it, in seconds. Slow
351+
/// VM-backed runtimes (CoCo/peer-pods) need longer than the default.
352+
pub runtime_bootstrap_grace_seconds: u64,
353+
/// Supervisor attach/policy-discover request timeout, in seconds.
354+
/// When set, the driver copies the value into the supervisor pod
355+
/// environment (`OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS`) so slow VM-backed
356+
/// runtimes have time to boot the remote boundary before the supervisor
357+
/// gives up.
358+
#[serde(default, skip_serializing_if = "Option::is_none")]
359+
pub sandbox_attach_request_timeout_seconds: Option<u64>,
349360
/// Corporate HTTP forward proxy used by the network supervisor for
350361
/// policy-approved TLS CONNECT egress.
351362
pub https_proxy: Option<String>,
@@ -513,6 +524,8 @@ impl Default for KubernetesComputeConfig {
513524
supervisor_image: config::default_supervisor_image(),
514525
supervisor_image_pull_policy: None,
515526
sandbox_runtime: KubernetesSandboxRuntimeConfig::default(),
527+
runtime_bootstrap_grace_seconds: 300,
528+
sandbox_attach_request_timeout_seconds: None,
516529
https_proxy: None,
517530
no_proxy: None,
518531
proxy_auth_secret_name: None,
@@ -544,6 +557,7 @@ impl Default for KubernetesComputeConfig {
544557
impl KubernetesComputeConfig {
545558
/// Validate startup configuration without connecting to Kubernetes.
546559
pub fn validate_configuration(&self) -> Result<(), String> {
560+
self.validate_bootstrap_windows()?;
547561
self.validate_supervisor_topology()?;
548562
self.validate_workspace_mode()?;
549563
self.validate_provider_spiffe_workload_api_socket_path()?;
@@ -554,6 +568,24 @@ impl KubernetesComputeConfig {
554568
self.validate_allowed_pod_labels()
555569
}
556570

571+
fn validate_bootstrap_windows(&self) -> Result<(), String> {
572+
for (name, value) in [
573+
(
574+
"runtime_bootstrap_grace_seconds",
575+
Some(self.runtime_bootstrap_grace_seconds),
576+
),
577+
(
578+
"sandbox_attach_request_timeout_seconds",
579+
self.sandbox_attach_request_timeout_seconds,
580+
),
581+
] {
582+
if value.is_some_and(|value| !(1..=86_400).contains(&value)) {
583+
return Err(format!("{name} must be between 1 and 86400 seconds"));
584+
}
585+
}
586+
Ok(())
587+
}
588+
557589
/// The shared supervisor serves only sandbox instances it can isolate: it
558590
/// holds no per-sandbox workload identity, upstream proxy credential, or
559591
/// proxy CA.
@@ -1249,6 +1281,43 @@ mod tests {
12491281
}
12501282
}
12511283

1284+
#[test]
1285+
fn bootstrap_grace_defaults_and_parses_from_toml() {
1286+
assert_eq!(
1287+
KubernetesComputeConfig::default().runtime_bootstrap_grace_seconds,
1288+
300
1289+
);
1290+
assert!(
1291+
KubernetesComputeConfig::default()
1292+
.sandbox_attach_request_timeout_seconds
1293+
.is_none()
1294+
);
1295+
let config: KubernetesComputeConfig = toml::from_str(
1296+
"runtime_bootstrap_grace_seconds = 900\nsandbox_attach_request_timeout_seconds = 900\n",
1297+
)
1298+
.unwrap();
1299+
assert_eq!(config.runtime_bootstrap_grace_seconds, 900);
1300+
assert_eq!(config.sandbox_attach_request_timeout_seconds, Some(900));
1301+
assert!(config.validate_configuration().is_ok());
1302+
let partial: KubernetesComputeConfig = toml::from_str("namespace = 'custom'").unwrap();
1303+
assert_eq!(partial.runtime_bootstrap_grace_seconds, 300);
1304+
for field in [
1305+
"runtime_bootstrap_grace_seconds",
1306+
"sandbox_attach_request_timeout_seconds",
1307+
] {
1308+
for value in [0, 86_401] {
1309+
let config: KubernetesComputeConfig =
1310+
toml::from_str(&format!("{field} = {value}")).unwrap();
1311+
assert!(config.validate_configuration().unwrap_err().contains(field));
1312+
}
1313+
for value in [1, 86_400] {
1314+
let config: KubernetesComputeConfig =
1315+
toml::from_str(&format!("{field} = {value}")).unwrap();
1316+
assert!(config.validate_configuration().is_ok());
1317+
}
1318+
}
1319+
}
1320+
12521321
#[test]
12531322
fn published_kubernetes_example_is_valid_toml() {
12541323
let docs = include_str!("../../../docs/how-it-works/gateways/configuration.mdx");

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -612,10 +612,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError {
612612
}
613613
}
614614
const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
615-
/// Unsupervised shared sandbox instances removed at once by one reconcile pass.
616-
/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded.
617-
const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5);
618-
619615
fn random_sandbox_runtime_token() -> Result<String, KubernetesDriverError> {
620616
use std::fmt::Write as _;
621617

@@ -3582,6 +3578,7 @@ impl KubernetesComputeDriver {
35823578
.map(|generation| (generation.name.as_str(), generation.digest.as_str())),
35833579
owner,
35843580
&self.supervisor_otlp_environment(),
3581+
self.config.sandbox_attach_request_timeout_seconds,
35853582
)
35863583
.map_err(KubernetesDriverError::Message)?,
35873584
&self.config.sandbox_runtime.app_listener,
@@ -4103,6 +4100,7 @@ impl KubernetesComputeDriver {
41034100
sandbox_api.api.clone(),
41044101
cr_name.to_string(),
41054102
Some(cr_uid.to_string()),
4103+
Duration::from_secs(self.config.runtime_bootstrap_grace_seconds),
41064104
);
41074105
Ok(())
41084106
}
@@ -5544,7 +5542,7 @@ impl KubernetesComputeDriver {
55445542
if !sandbox_runtime_bootstrap_is_stale(
55455543
object,
55465544
SystemTime::now(),
5547-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
5545+
Duration::from_secs(self.config.runtime_bootstrap_grace_seconds),
55485546
) {
55495547
return;
55505548
}
@@ -5987,9 +5985,10 @@ fn spawn_sandbox_runtime_bootstrap_completion(
59875985
sandboxes: Api<DynamicObject>,
59885986
sandbox_name: String,
59895987
expected_sandbox_uid: Option<String>,
5988+
bootstrap_grace: Duration,
59905989
) {
59915990
tokio::spawn(async move {
5992-
let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE;
5991+
let deadline = tokio::time::Instant::now() + bootstrap_grace;
59935992
match tokio::time::timeout_at(deadline, control.wait_available()).await {
59945993
Ok(true) => {}
59955994
Ok(false) | Err(_) => return,
@@ -10514,6 +10513,7 @@ mod tests {
1051410513
#[test]
1051510514
fn sandbox_runtime_bootstrap_marker_and_age_gate_rollback() {
1051610515
let started = Duration::from_hours(490_896);
10516+
let bootstrap_grace = Duration::from_mins(5);
1051710517
let mut object: DynamicObject = serde_json::from_value(serde_json::json!({
1051810518
"apiVersion": "agents.x-k8s.io/v1beta1",
1051910519
"kind": "Sandbox",
@@ -10536,13 +10536,13 @@ mod tests {
1053610536
let started_at = SystemTime::UNIX_EPOCH + started;
1053710537
assert!(!sandbox_runtime_bootstrap_is_stale(
1053810538
&object,
10539-
started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1),
10540-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE
10539+
started_at + bootstrap_grace - Duration::from_secs(1),
10540+
bootstrap_grace
1054110541
));
1054210542
assert!(sandbox_runtime_bootstrap_is_stale(
1054310543
&object,
10544-
started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
10545-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE
10544+
started_at + bootstrap_grace,
10545+
bootstrap_grace
1054610546
));
1054710547
object
1054810548
.metadata
@@ -10553,7 +10553,7 @@ mod tests {
1055310553
assert!(sandbox_runtime_bootstrap_is_stale(
1055410554
&object,
1055510555
started_at,
10556-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE
10556+
bootstrap_grace
1055710557
));
1055810558
object
1055910559
.metadata

‎crates/openshell-driver-kubernetes/src/main.rs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@ struct Args {
2929
default_value = "{}"
3030
)]
3131
admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig,
32+
/// Runtime bootstrap budget in seconds.
33+
#[arg(long, env = "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS", default_value = "300",
34+
value_parser = clap::value_parser!(u64).range(1..=86_400))]
35+
runtime_bootstrap_grace_seconds: u64,
36+
/// Initial attachment budget passed to dedicated supervisor Pods.
37+
#[arg(long, env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS",
38+
value_parser = clap::value_parser!(u64).range(1..=86_400))]
39+
sandbox_attach_request_timeout_seconds: Option<u64>,
3240
/// Public compute-driver Unix socket used by an external gateway.
3341
#[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")]
3442
bind_socket: Option<PathBuf>,
@@ -286,6 +294,8 @@ async fn main() -> Result<()> {
286294
openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(),
287295
boundary_port: args.sandbox_runtime_boundary_port,
288296
},
297+
runtime_bootstrap_grace_seconds: args.runtime_bootstrap_grace_seconds,
298+
sandbox_attach_request_timeout_seconds: args.sandbox_attach_request_timeout_seconds,
289299
https_proxy: args.https_proxy,
290300
no_proxy: args.no_proxy,
291301
proxy_auth_secret_name: args.proxy_auth_secret_name,
@@ -369,6 +379,33 @@ async fn main() -> Result<()> {
369379

370380
#[cfg(test)]
371381
mod tests {
382+
#[test]
383+
fn bootstrap_timeout_arguments_reject_invalid_values() {
384+
let defaults = Args::try_parse_from(["openshell-driver-kubernetes"]).unwrap();
385+
assert_eq!(defaults.runtime_bootstrap_grace_seconds, 300);
386+
assert!(defaults.sandbox_attach_request_timeout_seconds.is_none());
387+
for flag in [
388+
"--runtime-bootstrap-grace-seconds",
389+
"--sandbox-attach-request-timeout-seconds",
390+
] {
391+
for value in ["0", "86401", "invalid"] {
392+
assert!(
393+
Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).is_err()
394+
);
395+
}
396+
for value in ["1", "900", "86400"] {
397+
let args =
398+
Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).unwrap();
399+
let seconds = value.parse::<u64>().unwrap();
400+
if flag == "--runtime-bootstrap-grace-seconds" {
401+
assert_eq!(args.runtime_bootstrap_grace_seconds, seconds);
402+
} else {
403+
assert_eq!(args.sandbox_attach_request_timeout_seconds, Some(seconds));
404+
}
405+
}
406+
}
407+
}
408+
372409
use super::*;
373410

374411
#[test]

‎crates/openshell-driver-kubernetes/src/sandbox_runtime.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,7 @@ pub fn supervisor_pod(
363363
network_additional_ca: Option<(&str, &str)>,
364364
owner: OwnerReference,
365365
tracing_environment: &[(&str, String)],
366+
attach_request_timeout_seconds: Option<u64>,
366367
) -> Result<Pod, String> {
367368
let labels = control_labels(sandbox_id, gateway_id);
368369
let mut environment = vec![
@@ -404,6 +405,12 @@ pub fn supervisor_pod(
404405
.iter()
405406
.map(|(name, value)| env_var(name, value)),
406407
);
408+
if let Some(timeout_seconds) = attach_request_timeout_seconds {
409+
environment.push(env_var(
410+
"OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS",
411+
&timeout_seconds.to_string(),
412+
));
413+
}
407414
if let Some(server_name) = gateway_tls_server_name(grpc_endpoint) {
408415
environment.push(env_var(
409416
openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME,
@@ -1055,6 +1062,7 @@ mod tests {
10551062
None,
10561063
owner(),
10571064
&[],
1065+
None,
10581066
)
10591067
.expect("render supervisor Pod")
10601068
}
@@ -1140,6 +1148,7 @@ mod tests {
11401148
),
11411149
("TRACEPARENT", TRACEPARENT.to_string()),
11421150
],
1151+
None,
11431152
)
11441153
.expect("render supervisor Pod");
11451154
let environment = pod.spec.expect("Pod spec").containers[0]
@@ -1156,6 +1165,56 @@ mod tests {
11561165
assert_eq!(environment["TRACEPARENT"], TRACEPARENT);
11571166
}
11581167

1168+
#[test]
1169+
fn supervisor_pod_sets_attach_request_timeout_only_when_configured() {
1170+
let env = |timeout: Option<u64>| {
1171+
supervisor_pod(
1172+
"sandbox",
1173+
&SandboxRuntimeNames::new("pair"),
1174+
"pair",
1175+
"demo",
1176+
"gateway",
1177+
"supervisor:latest",
1178+
None,
1179+
"sandbox-sa",
1180+
1000,
1181+
1000,
1182+
&[],
1183+
"https://gateway:8080",
1184+
SupervisorClientTls::Disabled,
1185+
"{}",
1186+
"info",
1187+
600,
1188+
None,
1189+
None,
1190+
None,
1191+
false,
1192+
false,
1193+
false,
1194+
None,
1195+
None,
1196+
owner(),
1197+
&[],
1198+
timeout,
1199+
)
1200+
.unwrap()
1201+
.spec
1202+
.expect("Pod spec")
1203+
.containers[0]
1204+
.env
1205+
.clone()
1206+
.unwrap_or_default()
1207+
.into_iter()
1208+
.map(|variable| (variable.name, variable.value.unwrap_or_default()))
1209+
.collect::<BTreeMap<_, _>>()
1210+
};
1211+
assert!(!env(None).contains_key("OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS"));
1212+
assert_eq!(
1213+
env(Some(900))["OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS"],
1214+
"900"
1215+
);
1216+
}
1217+
11591218
#[test]
11601219
fn supervisor_pod_reads_client_tls_from_the_bootstrap_secret() {
11611220
let (env, volumes) = tls_env_and_volumes(&supervisor_pod_with_client_tls(
@@ -1261,6 +1320,7 @@ mod tests {
12611320
None,
12621321
owner(),
12631322
&[],
1323+
None,
12641324
)
12651325
.expect("render supervisor Pod");
12661326
let pod_spec = pod.spec.as_ref().expect("Pod spec");
@@ -1430,6 +1490,7 @@ mod tests {
14301490
Some(("openshell-additional-ca-gateway-generation", digest)),
14311491
owner(),
14321492
&[],
1493+
None,
14331494
)
14341495
.expect("render supervisor Pod with destination trust");
14351496
let pod = crate::app_listener::configure_supervisor_pod(
@@ -1622,6 +1683,7 @@ mod tests {
16221683
None,
16231684
owner(),
16241685
&[],
1686+
None,
16251687
)
16261688
.expect("render supervisor Pod")
16271689
}

0 commit comments

Comments
 (0)