Skip to content

Commit 730ca3f

Browse files
committed
feat(k8s-driver): add default_runtime_class_name config for sandbox pods
Allow operators to configure a default Kubernetes runtimeClassName that is applied to sandbox pods when the CreateSandbox request does not specify one. This avoids requiring every API caller to explicitly set the runtime class for clusters that always need a specific RuntimeClass (e.g. kata-containers, nvidia). The fallback is applied in the Kubernetes driver only — per-request values still take priority, and an empty default (the built-in) preserves existing behavior (field omitted, cluster default applies).
1 parent 5f58cb0 commit 730ca3f

7 files changed

Lines changed: 121 additions & 1 deletion

File tree

‎crates/openshell-driver-kubernetes/src/config.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ pub struct KubernetesComputeConfig {
7676
pub host_gateway_ip: String,
7777
pub enable_user_namespaces: bool,
7878
pub workspace_default_storage_size: String,
79+
/// Default Kubernetes `runtimeClassName` for sandbox pods.
80+
/// Applied when a `CreateSandbox` request does not specify one.
81+
/// Empty string (default) = omit the field, using the cluster default.
82+
pub default_runtime_class_name: String,
7983
/// Lifetime (seconds) of the projected `ServiceAccount` token kubelet
8084
/// writes into each sandbox pod. Used only for the one-shot
8185
/// `IssueSandboxToken` bootstrap exchange — the gateway-minted JWT
@@ -116,6 +120,7 @@ impl Default for KubernetesComputeConfig {
116120
host_gateway_ip: String::new(),
117121
enable_user_namespaces: false,
118122
workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(),
123+
default_runtime_class_name: String::new(),
119124
sa_token_ttl_secs: 3600,
120125
}
121126
}
@@ -176,6 +181,21 @@ mod tests {
176181
assert_eq!(cfg.service_account_name, "openshell-sandbox");
177182
}
178183

184+
#[test]
185+
fn serde_override_default_runtime_class_name() {
186+
let json = serde_json::json!({
187+
"default_runtime_class_name": "nvidia"
188+
});
189+
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
190+
assert_eq!(cfg.default_runtime_class_name, "nvidia");
191+
}
192+
193+
#[test]
194+
fn default_runtime_class_name_is_empty() {
195+
let cfg = KubernetesComputeConfig::default();
196+
assert!(cfg.default_runtime_class_name.is_empty());
197+
}
198+
179199
#[test]
180200
fn serde_override_image_pull_secrets() {
181201
let json = serde_json::json!({

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,7 @@ impl KubernetesComputeDriver {
329329
host_gateway_ip: &self.config.host_gateway_ip,
330330
enable_user_namespaces: self.config.enable_user_namespaces,
331331
workspace_default_storage_size: &self.config.workspace_default_storage_size,
332+
default_runtime_class_name: &self.config.default_runtime_class_name,
332333
sa_token_ttl_secs: self.config.effective_sa_token_ttl_secs(),
333334
};
334335
obj.data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), &params);
@@ -1041,6 +1042,7 @@ struct SandboxPodParams<'a> {
10411042
host_gateway_ip: &'a str,
10421043
enable_user_namespaces: bool,
10431044
workspace_default_storage_size: &'a str,
1045+
default_runtime_class_name: &'a str,
10441046
/// Lifetime (seconds) of the projected `ServiceAccount` token used
10451047
/// for the bootstrap `IssueSandboxToken` exchange.
10461048
sa_token_ttl_secs: i64,
@@ -1064,6 +1066,7 @@ impl Default for SandboxPodParams<'_> {
10641066
host_gateway_ip: "",
10651067
enable_user_namespaces: false,
10661068
workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE,
1069+
default_runtime_class_name: "",
10671070
sa_token_ttl_secs: 3600,
10681071
}
10691072
}
@@ -1184,7 +1187,11 @@ fn sandbox_template_to_k8s(
11841187
}
11851188

11861189
let mut spec = serde_json::Map::new();
1187-
if let Some(runtime_class) = platform_config_string(template, "runtime_class_name") {
1190+
let runtime_class_name = platform_config_string(template, "runtime_class_name").or_else(|| {
1191+
(!params.default_runtime_class_name.is_empty())
1192+
.then(|| params.default_runtime_class_name.to_string())
1193+
});
1194+
if let Some(runtime_class) = runtime_class_name {
11881195
spec.insert(
11891196
"runtimeClassName".to_string(),
11901197
serde_json::json!(runtime_class),
@@ -2075,6 +2082,84 @@ mod tests {
20752082
);
20762083
}
20772084

2085+
#[test]
2086+
fn default_runtime_class_name_applied_when_template_omits_it() {
2087+
let template = SandboxTemplate::default();
2088+
let pod_template = {
2089+
let params = SandboxPodParams {
2090+
default_runtime_class_name: "kata-containers",
2091+
..SandboxPodParams::default()
2092+
};
2093+
sandbox_template_to_k8s(
2094+
&template,
2095+
false,
2096+
&std::collections::HashMap::new(),
2097+
true,
2098+
&params,
2099+
)
2100+
};
2101+
2102+
assert_eq!(
2103+
pod_template["spec"]["runtimeClassName"],
2104+
serde_json::json!("kata-containers")
2105+
);
2106+
}
2107+
2108+
#[test]
2109+
fn template_runtime_class_name_overrides_config_default() {
2110+
let template = SandboxTemplate {
2111+
platform_config: Some(Struct {
2112+
fields: std::iter::once((
2113+
"runtime_class_name".to_string(),
2114+
Value {
2115+
kind: Some(Kind::StringValue("gvisor".to_string())),
2116+
},
2117+
))
2118+
.collect(),
2119+
}),
2120+
..SandboxTemplate::default()
2121+
};
2122+
2123+
let pod_template = {
2124+
let params = SandboxPodParams {
2125+
default_runtime_class_name: "kata-containers",
2126+
..SandboxPodParams::default()
2127+
};
2128+
sandbox_template_to_k8s(
2129+
&template,
2130+
false,
2131+
&std::collections::HashMap::new(),
2132+
true,
2133+
&params,
2134+
)
2135+
};
2136+
2137+
assert_eq!(
2138+
pod_template["spec"]["runtimeClassName"],
2139+
serde_json::json!("gvisor")
2140+
);
2141+
}
2142+
2143+
#[test]
2144+
fn runtime_class_name_omitted_when_both_template_and_default_empty() {
2145+
let template = SandboxTemplate::default();
2146+
let pod_template = {
2147+
let params = SandboxPodParams::default();
2148+
sandbox_template_to_k8s(
2149+
&template,
2150+
false,
2151+
&std::collections::HashMap::new(),
2152+
true,
2153+
&params,
2154+
)
2155+
};
2156+
2157+
assert_eq!(
2158+
pod_template["spec"]["runtimeClassName"],
2159+
serde_json::json!(null)
2160+
);
2161+
}
2162+
20782163
#[test]
20792164
fn gpu_sandbox_preserves_existing_resource_limits() {
20802165
use openshell_core::proto::compute::v1::DriverResourceRequirements;

‎crates/openshell-driver-kubernetes/src/main.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ async fn main() -> Result<()> {
122122
.unwrap_or_else(|_| {
123123
openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string()
124124
}),
125+
default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME")
126+
.unwrap_or_default(),
125127
sa_token_ttl_secs: args.sa_token_ttl_secs,
126128
})
127129
.await

‎deploy/helm/openshell/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ JWT signing Secret.
187187
| securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. |
188188
| server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. |
189189
| server.dbUrl | string | `"sqlite:/var/openshell/openshell.db"` | Gateway database URL (used for the default SQLite backend). |
190+
| server.defaultRuntimeClassName | string | `""` | Default Kubernetes runtimeClassName for sandbox pods. Applied when a CreateSandbox request does not specify one. Empty (default) = omit the field, using the cluster's default RuntimeClass. Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it to all sandboxes that don't explicitly override it. |
190191
| server.disableTls | bool | `false` | Disable TLS entirely - the server listens on plaintext HTTP. Set to true when a reverse proxy / tunnel terminates TLS at the edge. |
191192
| server.enableLoopbackServiceHttp | bool | `true` | Enable plaintext HTTP routing for loopback sandbox service URLs on TLS-enabled gateways. |
192193
| server.enableUserNamespaces | bool | `false` | Enable Kubernetes user namespace isolation (hostUsers: false) for sandbox pods. Requires Kubernetes 1.33+ with user namespace support available (beta through 1.35, GA in 1.36+), plus a supporting container runtime and Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host UID and capabilities become namespaced. |

‎deploy/helm/openshell/templates/gateway-config.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,9 @@ data:
118118
{{- if .Values.server.workspaceDefaultStorageSize }}
119119
workspace_default_storage_size = {{ .Values.server.workspaceDefaultStorageSize | quote }}
120120
{{- end }}
121+
{{- if .Values.server.defaultRuntimeClassName }}
122+
default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }}
123+
{{- end }}
121124
{{- if .Values.supervisor.image.pullPolicy }}
122125
supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }}
123126
{{- end }}

‎deploy/helm/openshell/values.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,12 @@ server:
158158
# Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi").
159159
# Empty = built-in default (2Gi).
160160
workspaceDefaultStorageSize: ""
161+
# -- Default Kubernetes runtimeClassName for sandbox pods.
162+
# Applied when a CreateSandbox request does not specify one.
163+
# Empty (default) = omit the field, using the cluster's default RuntimeClass.
164+
# Set to a RuntimeClass name (e.g. "kata-containers", "nvidia") to apply it
165+
# to all sandboxes that don't explicitly override it.
166+
defaultRuntimeClassName: ""
161167
# -- gRPC endpoint sandboxes call back into the gateway. Leave empty to derive
162168
# it from the chart fullname, release namespace, service port, and
163169
# disableTls flag, for example https://openshell.openshell.svc.cluster.local:8080.

‎docs/reference/gateway-config.mdx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,9 @@ client_tls_secret_name = "openshell-client-tls"
178178
host_gateway_ip = "10.0.0.1"
179179
enable_user_namespaces = false
180180
workspace_default_storage_size = "10Gi"
181+
# Kubernetes RuntimeClass applied to sandbox pods when the API request does
182+
# not specify one. Empty (default) = omit the field, using the cluster default.
183+
# default_runtime_class_name = "kata-containers"
181184
# Kubelet clamps projected tokens below 600 seconds. The driver caps values at 86400.
182185
sa_token_ttl_secs = 3600
183186
```

0 commit comments

Comments
 (0)