Skip to content

Commit de949b7

Browse files
committed
feat(compute): make provisioning and bootstrap deadlines configurable
Signed-off-by: Mike Hepburn <[email protected]>
1 parent 6e69360 commit de949b7

20 files changed

Lines changed: 632 additions & 131 deletions

File tree

‎crates/openshell-core/src/config.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,11 @@ pub struct Config {
286286
/// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive.
287287
pub image_preparation_timeout_seconds: u32,
288288

289+
/// Provisioning repair window, in milliseconds. The first authenticated
290+
/// supervisor configuration report starts admission repair, which stays
291+
/// armed until Ready. Must be between 1000 and 86400000, inclusive.
292+
pub provisioning_repair_window_ms: i64,
293+
289294
/// Maximum gRPC requests allowed per rate-limit window.
290295
///
291296
/// When paired with [`Self::grpc_rate_limit_window_secs`], positive values
@@ -918,6 +923,7 @@ impl Config {
918923
default_credential_driver: None,
919924
ssh_session_ttl_secs: default_ssh_session_ttl_secs(),
920925
image_preparation_timeout_seconds: 1800,
926+
provisioning_repair_window_ms: 300_000,
921927
grpc_rate_limit_requests: None,
922928
grpc_rate_limit_window_secs: None,
923929
service_routing: ServiceRoutingConfig::default(),

‎crates/openshell-driver-kubernetes/src/config.rs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,18 @@ pub struct KubernetesComputeConfig {
205205
pub supervisor_image_pull_policy: Option<KubernetesImagePullPolicy>,
206206
/// Cross-pod sandbox/supervisor settings.
207207
pub sandbox_runtime: KubernetesSandboxRuntimeConfig,
208+
/// Bound on how long a crash-interrupted, fail-closed sandbox runtime
209+
/// bootstrap may remain stranded before the driver reclaims it. Slow
210+
/// VM-backed runtimes (CoCo/peer-pods) need longer than the default.
211+
#[serde(default)]
212+
pub runtime_bootstrap_grace_ms: u64,
213+
/// Supervisor attach/policy-discover request timeout, in milliseconds.
214+
/// When set, the driver copies the value into the supervisor pod
215+
/// environment (`OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS`) so slow VM-backed
216+
/// runtimes have time to boot the remote boundary before the supervisor
217+
/// gives up.
218+
#[serde(default, skip_serializing_if = "Option::is_none")]
219+
pub sandbox_attach_request_timeout_ms: Option<u64>,
208220
/// Corporate HTTP forward proxy used by the network supervisor for
209221
/// policy-approved TLS CONNECT egress.
210222
pub https_proxy: Option<String>,
@@ -361,6 +373,8 @@ impl Default for KubernetesComputeConfig {
361373
supervisor_image: config::default_supervisor_image(),
362374
supervisor_image_pull_policy: None,
363375
sandbox_runtime: KubernetesSandboxRuntimeConfig::default(),
376+
runtime_bootstrap_grace_ms: 300_000,
377+
sandbox_attach_request_timeout_ms: None,
364378
https_proxy: None,
365379
no_proxy: None,
366380
proxy_auth_secret_name: None,
@@ -971,6 +985,25 @@ mod tests {
971985
}
972986
}
973987

988+
#[test]
989+
fn bootstrap_grace_defaults_and_parses_from_toml() {
990+
assert_eq!(
991+
KubernetesComputeConfig::default().runtime_bootstrap_grace_ms,
992+
300_000
993+
);
994+
assert!(
995+
KubernetesComputeConfig::default()
996+
.sandbox_attach_request_timeout_ms
997+
.is_none()
998+
);
999+
let config: KubernetesComputeConfig = toml::from_str(
1000+
"runtime_bootstrap_grace_ms = 900000\nsandbox_attach_request_timeout_ms = 900000\n",
1001+
)
1002+
.unwrap();
1003+
assert_eq!(config.runtime_bootstrap_grace_ms, 900_000);
1004+
assert_eq!(config.sandbox_attach_request_timeout_ms, Some(900_000));
1005+
}
1006+
9741007
#[test]
9751008
fn published_kubernetes_example_is_valid_toml() {
9761009
let docs = include_str!("../../../docs/how-it-works/gateways/configuration.mdx");

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError {
280280
}
281281
}
282282
const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
283-
/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded.
284-
const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5);
285283

286284
fn random_sandbox_runtime_token() -> String {
287285
use std::fmt::Write as _;
@@ -2493,6 +2491,7 @@ impl KubernetesComputeDriver {
24932491
),
24942492
dependent_owner.clone(),
24952493
&self.supervisor_tracing_environment(),
2494+
self.config.sandbox_attach_request_timeout_ms,
24962495
)
24972496
.map_err(KubernetesDriverError::Message)?,
24982497
)
@@ -2727,6 +2726,7 @@ impl KubernetesComputeDriver {
27272726
names.supervisor_pod.clone(),
27282727
cr_name.to_string(),
27292728
Some(cr_uid.to_string()),
2729+
Duration::from_millis(self.config.runtime_bootstrap_grace_ms),
27302730
);
27312731
// Return while the CR remains explicitly bootstrapping. The gateway
27322732
// can now commit the sandbox configuration required by a policy-less
@@ -2984,6 +2984,7 @@ impl KubernetesComputeDriver {
29842984
names.supervisor_pod.clone(),
29852985
cr_name.to_string(),
29862986
Some(cr_uid.to_string()),
2987+
Duration::from_millis(self.config.runtime_bootstrap_grace_ms),
29872988
);
29882989
Ok(())
29892990
}
@@ -3315,6 +3316,7 @@ impl KubernetesComputeDriver {
33153316
false,
33163317
),
33173318
&self.supervisor_tracing_environment(),
3319+
self.config.sandbox_attach_request_timeout_ms,
33183320
)
33193321
.map_err(KubernetesDriverError::Message)?,
33203322
)
@@ -4211,7 +4213,7 @@ impl KubernetesComputeDriver {
42114213
if !sandbox_runtime_bootstrap_is_stale(
42124214
object,
42134215
SystemTime::now(),
4214-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
4216+
Duration::from_millis(self.config.runtime_bootstrap_grace_ms),
42154217
) {
42164218
return;
42174219
}
@@ -4538,9 +4540,10 @@ fn spawn_sandbox_runtime_bootstrap_completion(
45384540
supervisor_pod_name: String,
45394541
sandbox_name: String,
45404542
expected_sandbox_uid: Option<String>,
4543+
bootstrap_grace: Duration,
45414544
) {
45424545
tokio::spawn(async move {
4543-
let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE;
4546+
let deadline = tokio::time::Instant::now() + bootstrap_grace;
45444547
let available = |pod: Option<&Pod>| {
45454548
pod.is_some_and(|pod| {
45464549
sandbox_runtime_control_availability_from_pod(pod)
@@ -8562,6 +8565,7 @@ mod tests {
85628565
#[test]
85638566
fn sandbox_runtime_bootstrap_marker_and_age_gate_rollback() {
85648567
let started = Duration::from_hours(490_896);
8568+
let bootstrap_grace = Duration::from_mins(5);
85658569
let mut object: DynamicObject = serde_json::from_value(serde_json::json!({
85668570
"apiVersion": "agents.x-k8s.io/v1beta1",
85678571
"kind": "Sandbox",
@@ -8584,13 +8588,13 @@ mod tests {
85848588
let started_at = SystemTime::UNIX_EPOCH + started;
85858589
assert!(!sandbox_runtime_bootstrap_is_stale(
85868590
&object,
8587-
started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1),
8588-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE
8591+
started_at + bootstrap_grace - Duration::from_secs(1),
8592+
bootstrap_grace
85898593
));
85908594
assert!(sandbox_runtime_bootstrap_is_stale(
85918595
&object,
8592-
started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
8593-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE
8596+
started_at + bootstrap_grace,
8597+
bootstrap_grace
85948598
));
85958599
object
85968600
.metadata
@@ -8601,7 +8605,7 @@ mod tests {
86018605
assert!(sandbox_runtime_bootstrap_is_stale(
86028606
&object,
86038607
started_at,
8604-
SANDBOX_RUNTIME_BOOTSTRAP_GRACE
8608+
bootstrap_grace
86058609
));
86068610
object
86078611
.metadata

‎crates/openshell-driver-kubernetes/src/main.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,15 @@ async fn main() -> Result<()> {
275275
sandbox_runtime: KubernetesSandboxRuntimeConfig {
276276
boundary_port: args.sandbox_runtime_boundary_port,
277277
},
278+
runtime_bootstrap_grace_ms: std::env::var("OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS")
279+
.ok()
280+
.and_then(|value| value.parse::<u64>().ok())
281+
.filter(|value| *value > 0)
282+
.unwrap_or(300_000),
283+
sandbox_attach_request_timeout_ms: std::env::var("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS")
284+
.ok()
285+
.and_then(|value| value.parse::<u64>().ok())
286+
.filter(|value| *value > 0),
278287
https_proxy: args.https_proxy,
279288
no_proxy: args.no_proxy,
280289
proxy_auth_secret_name: args.proxy_auth_secret_name,

‎crates/openshell-driver-kubernetes/src/sandbox_runtime.rs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ pub fn supervisor_pod(
234234
provider_spiffe_socket_path: Option<&str>,
235235
owner: OwnerReference,
236236
tracing_environment: &[(&str, String)],
237+
attach_request_timeout_ms: Option<u64>,
237238
) -> Result<Pod, String> {
238239
let labels = control_labels(sandbox_id, gateway_id);
239240
let mut environment = vec![
@@ -275,6 +276,12 @@ pub fn supervisor_pod(
275276
.iter()
276277
.map(|(name, value)| env_var(name, value)),
277278
);
279+
if let Some(timeout_ms) = attach_request_timeout_ms {
280+
environment.push(env_var(
281+
"OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS",
282+
&timeout_ms.to_string(),
283+
));
284+
}
278285
if let Some(server_name) = gateway_tls_server_name(grpc_endpoint) {
279286
environment.push(env_var(
280287
openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME,
@@ -836,6 +843,7 @@ mod tests {
836843
None,
837844
owner(),
838845
&[],
846+
None,
839847
)
840848
.expect("render supervisor Pod")
841849
}
@@ -901,6 +909,7 @@ mod tests {
901909
),
902910
("TRACEPARENT", TRACEPARENT.to_string()),
903911
],
912+
None,
904913
)
905914
.expect("render supervisor Pod");
906915
let environment = pod.spec.expect("Pod spec").containers[0]
@@ -917,6 +926,55 @@ mod tests {
917926
assert_eq!(environment["TRACEPARENT"], TRACEPARENT);
918927
}
919928

929+
#[test]
930+
fn supervisor_pod_sets_attach_request_timeout_only_when_configured() {
931+
let env = |timeout: Option<u64>| {
932+
supervisor_pod(
933+
"sandbox",
934+
&SandboxRuntimeNames::new("pair"),
935+
"pair",
936+
"demo",
937+
"gateway",
938+
"supervisor:latest",
939+
None,
940+
"sandbox-sa",
941+
1000,
942+
1000,
943+
&[],
944+
"https://gateway:8080",
945+
SupervisorClientTls::Disabled,
946+
"{}",
947+
"info",
948+
600,
949+
None,
950+
None,
951+
None,
952+
false,
953+
false,
954+
false,
955+
None,
956+
owner(),
957+
&[],
958+
timeout,
959+
)
960+
.unwrap()
961+
.spec
962+
.expect("Pod spec")
963+
.containers[0]
964+
.env
965+
.clone()
966+
.unwrap_or_default()
967+
.into_iter()
968+
.map(|variable| (variable.name, variable.value.unwrap_or_default()))
969+
.collect::<BTreeMap<_, _>>()
970+
};
971+
assert!(!env(None).contains_key("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS"));
972+
assert_eq!(
973+
env(Some(900_000))["OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS"],
974+
"900000"
975+
);
976+
}
977+
920978
#[test]
921979
fn supervisor_pod_reads_client_tls_from_the_bootstrap_secret() {
922980
let (env, volumes) = tls_env_and_volumes(&supervisor_pod_with_client_tls(
@@ -1021,6 +1079,7 @@ mod tests {
10211079
None,
10221080
owner(),
10231081
&[],
1082+
None,
10241083
)
10251084
.expect("render supervisor Pod");
10261085
let pod_spec = pod.spec.as_ref().expect("Pod spec");
@@ -1279,6 +1338,7 @@ mod tests {
12791338
None,
12801339
owner(),
12811340
&[],
1341+
None,
12821342
)
12831343
.expect("render supervisor Pod")
12841344
}

‎crates/openshell-gateway/src/lib.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,32 @@ fn kubernetes_config(
293293
config.supervisor_image,
294294
std::env::var(openshell_core::config::SUPERVISOR_IMAGE_ENV).ok(),
295295
);
296+
if let Ok(raw) = std::env::var("OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS") {
297+
let grace = raw.parse::<u64>().map_err(|_| {
298+
openshell_core::Error::config(
299+
"OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS must be an integer number of milliseconds",
300+
)
301+
})?;
302+
if grace == 0 {
303+
return Err(openshell_core::Error::config(
304+
"OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS must be greater than zero",
305+
));
306+
}
307+
config.runtime_bootstrap_grace_ms = grace;
308+
}
309+
if let Ok(raw) = std::env::var("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS") {
310+
let timeout = raw.parse::<u64>().map_err(|_| {
311+
openshell_core::Error::config(
312+
"OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS must be an integer number of milliseconds",
313+
)
314+
})?;
315+
if timeout == 0 {
316+
return Err(openshell_core::Error::config(
317+
"OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS must be greater than zero",
318+
));
319+
}
320+
config.sandbox_attach_request_timeout_ms = Some(timeout);
321+
}
296322
if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") {
297323
config.workspace_default_storage_size = size;
298324
}

0 commit comments

Comments
 (0)