Skip to content
This repository was archived by the owner on Nov 12, 2026. It is now read-only.

Commit 1eef6e3

Browse files
committed
feat(sandbox): add Platform network mode for restricted K8s platforms
Add NetworkMode::Platform for running the supervisor without elevated capabilities on Kubernetes platforms enforcing the restricted Pod Security Standard (including OpenShift restricted-v2 SCC). Platform Mode keeps Landlock filesystem isolation, seccomp syscall filtering, OPA policy evaluation, credential injection, and L7 inspection via a loopback CONNECT proxy. It replaces the network namespace (which requires CAP_SYS_ADMIN + CAP_NET_ADMIN) with: - Loopback proxy binding (127.0.0.1 instead of veth interface) - K8s driver: zero capabilities, drop ALL, non-root UID - K8s driver: shareProcessNamespace for /proc identity resolution without CAP_SYS_PTRACE - K8s driver: per-sandbox deny-all egress NetworkPolicy (loopback traffic to the in-pod proxy is unaffected by NetworkPolicy) - seccomp: block SOCK_DGRAM (UDP) in Platform mode to match the nftables UDP reject in namespace mode - Landlock scope: restrict abstract Unix sockets and signals (ABI v5+, BestEffort degrades on older kernels) Security parity with namespace mode: | Attack | Namespace mode | Platform mode | |------------------------|------------------------|--------------------------| | TCP bypass proxy | nftables REJECT | Landlock port 3128 only | | UDP exfiltration | nftables REJECT | seccomp SOCK_DGRAM block | | TCP to other-pod:3128 | nftables REJECT | egress NetworkPolicy | | Abstract Unix sockets | netns isolation | Landlock scope | | Container escape | Risk (CAP_SYS_ADMIN) | Impossible (zero caps) | Proto: add NetworkEnforcementMode enum and field to SandboxPolicy and DriverSandboxSpec. Default NAMESPACE (0) preserves existing behavior; PLATFORM (1) activates the new mode. Closes NVIDIA#899 Signed-off-by: Ladislav Smola <[email protected]>
1 parent 69d9e06 commit 1eef6e3

12 files changed

Lines changed: 405 additions & 150 deletions

File tree

‎crates/openshell-driver-docker/src/tests.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ fn test_sandbox() -> DriverSandbox {
4545
gpu: false,
4646
gpu_device: String::new(),
4747
sandbox_token: String::new(),
48+
network_enforcement: 0,
4849
}),
4950
status: None,
5051
}

‎crates/openshell-driver-kubernetes/src/driver.rs‎

Lines changed: 142 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -330,10 +330,32 @@ impl KubernetesComputeDriver {
330330
enable_user_namespaces: self.config.enable_user_namespaces,
331331
workspace_default_storage_size: &self.config.workspace_default_storage_size,
332332
sa_token_ttl_secs: self.config.effective_sa_token_ttl_secs(),
333+
is_platform_mode: sandbox
334+
.spec
335+
.as_ref()
336+
.is_some_and(|s| s.network_enforcement == 1),
333337
};
334338
obj.data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), &params);
335339
let api = self.api();
336340

341+
// Platform mode: emit a deny-all egress NetworkPolicy for this sandbox.
342+
// Loopback traffic (to the in-pod proxy on 127.0.0.1:3128) is unaffected
343+
// by NetworkPolicy. This closes the Landlock port-not-IP gap where
344+
// connect(other-pod:3128) would bypass the sandbox's OPA policy.
345+
if params.is_platform_mode {
346+
if let Err(e) = self
347+
.create_sandbox_network_policy(name, &sandbox.id)
348+
.await
349+
{
350+
warn!(
351+
sandbox_id = %sandbox.id,
352+
error = %e,
353+
"Failed to create egress NetworkPolicy for platform-mode sandbox \
354+
(sandbox will rely on Landlock + seccomp only)"
355+
);
356+
}
357+
}
358+
337359
match tokio::time::timeout(KUBE_API_TIMEOUT, api.create(&PostParams::default(), &obj)).await
338360
{
339361
Ok(Ok(_result)) => {
@@ -368,6 +390,67 @@ impl KubernetesComputeDriver {
368390
}
369391
}
370392

393+
/// Create a deny-all egress NetworkPolicy for a platform-mode sandbox pod.
394+
///
395+
/// The policy selects the specific sandbox pod by its `openshell.ai/sandbox-id`
396+
/// label and denies all egress. Loopback traffic (127.0.0.1:3128 to the
397+
/// in-pod CONNECT proxy) is unaffected by Kubernetes NetworkPolicy.
398+
async fn create_sandbox_network_policy(
399+
&self,
400+
sandbox_name: &str,
401+
sandbox_id: &str,
402+
) -> Result<(), KubernetesDriverError> {
403+
let np_name = format!("{sandbox_name}-egress");
404+
let np = serde_json::json!({
405+
"apiVersion": "networking.k8s.io/v1",
406+
"kind": "NetworkPolicy",
407+
"metadata": {
408+
"name": np_name,
409+
"namespace": self.config.namespace,
410+
"labels": {
411+
"openshell.ai/managed-by": "openshell",
412+
"openshell.ai/sandbox-id": sandbox_id,
413+
},
414+
},
415+
"spec": {
416+
"podSelector": {
417+
"matchLabels": {
418+
"openshell.ai/sandbox-id": sandbox_id,
419+
},
420+
},
421+
"policyTypes": ["Egress"],
422+
"egress": [],
423+
},
424+
});
425+
426+
let gvk = GroupVersionKind::gvk("networking.k8s.io", "v1", "NetworkPolicy");
427+
let resource = ApiResource::from_gvk(&gvk);
428+
let api: Api<DynamicObject> =
429+
Api::namespaced_with(self.client.clone(), &self.config.namespace, &resource);
430+
let obj = serde_json::from_value::<DynamicObject>(np)
431+
.map_err(|e| KubernetesDriverError::Message(format!("NetworkPolicy serialization: {e}")))?;
432+
433+
match tokio::time::timeout(KUBE_API_TIMEOUT, api.create(&PostParams::default(), &obj)).await
434+
{
435+
Ok(Ok(_)) => {
436+
info!(
437+
sandbox_id = %sandbox_id,
438+
policy = %np_name,
439+
"Created deny-all egress NetworkPolicy for platform-mode sandbox"
440+
);
441+
Ok(())
442+
}
443+
Ok(Err(KubeError::Api(err))) if err.code == 409 => {
444+
debug!(policy = %np_name, "NetworkPolicy already exists");
445+
Ok(())
446+
}
447+
Ok(Err(err)) => Err(KubernetesDriverError::from_kube(err)),
448+
Err(_) => Err(KubernetesDriverError::Message(
449+
"timed out creating NetworkPolicy".into(),
450+
)),
451+
}
452+
}
453+
371454
pub async fn delete_sandbox(&self, name: &str) -> Result<bool, String> {
372455
info!(
373456
sandbox_name = %name,
@@ -823,6 +906,7 @@ fn apply_supervisor_sideload(
823906
supervisor_image: &str,
824907
supervisor_image_pull_policy: &str,
825908
method: SupervisorSideloadMethod,
909+
is_platform_mode: bool,
826910
) {
827911
let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else {
828912
return;
@@ -882,16 +966,16 @@ fn apply_supervisor_sideload(
882966
serde_json::json!([format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH)]),
883967
);
884968

885-
// Force the supervisor to run as root (UID 0). Sandbox images may set
886-
// a non-root USER directive (e.g. `USER sandbox`), but the supervisor
887-
// needs root to create network namespaces, set up the proxy, and
888-
// configure Landlock/seccomp. The supervisor itself drops privileges
889-
// for child processes via the policy's `run_as_user`/`run_as_group`.
890-
let security_context = container
891-
.entry("securityContext")
892-
.or_insert_with(|| serde_json::json!({}));
893-
if let Some(sc) = security_context.as_object_mut() {
894-
sc.insert("runAsUser".to_string(), serde_json::json!(0));
969+
// In namespace mode, force root (UID 0) so the supervisor can create
970+
// network namespaces and drop privileges for child processes.
971+
// In platform mode, keep the image's default non-root user.
972+
if !is_platform_mode {
973+
let security_context = container
974+
.entry("securityContext")
975+
.or_insert_with(|| serde_json::json!({}));
976+
if let Some(sc) = security_context.as_object_mut() {
977+
sc.insert("runAsUser".to_string(), serde_json::json!(0));
978+
}
895979
}
896980

897981
// Add volume mount
@@ -1044,6 +1128,10 @@ struct SandboxPodParams<'a> {
10441128
/// Lifetime (seconds) of the projected `ServiceAccount` token used
10451129
/// for the bootstrap `IssueSandboxToken` exchange.
10461130
sa_token_ttl_secs: i64,
1131+
/// Platform network enforcement mode (Issue #899). When true, sandbox
1132+
/// pods are emitted without elevated capabilities, compatible with
1133+
/// restricted-v2 SCC and restricted Pod Security Standard.
1134+
is_platform_mode: bool,
10471135
}
10481136

10491137
impl Default for SandboxPodParams<'_> {
@@ -1065,6 +1153,7 @@ impl Default for SandboxPodParams<'_> {
10651153
enable_user_namespaces: false,
10661154
workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE,
10671155
sa_token_ttl_secs: 3600,
1156+
is_platform_mode: false,
10681157
}
10691158
}
10701159
}
@@ -1233,6 +1322,18 @@ fn sandbox_template_to_k8s(
12331322
serde_json::json!(false),
12341323
);
12351324

1325+
// Platform mode: share the PID namespace between supervisor and agent
1326+
// containers so the proxy can resolve process identity via /proc/<pid>/exe
1327+
// without CAP_SYS_PTRACE. Same-PID-namespace /proc reads work for
1328+
// same-UID processes; cross-UID reads work because the supervisor runs
1329+
// as the pod's non-root user alongside the agent.
1330+
if params.is_platform_mode {
1331+
spec.insert(
1332+
"shareProcessNamespace".to_string(),
1333+
serde_json::json!(true),
1334+
);
1335+
}
1336+
12361337
let mut container = serde_json::Map::new();
12371338
container.insert("name".to_string(), serde_json::json!("agent"));
12381339
// Use template image if provided, otherwise fall back to default
@@ -1265,22 +1366,32 @@ fn sandbox_template_to_k8s(
12651366

12661367
container.insert("env".to_string(), serde_json::Value::Array(env));
12671368

1268-
let mut capabilities: Vec<&str> = vec!["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"];
1269-
if use_user_namespaces {
1270-
// In a user namespace the bounding set is reset. SETUID/SETGID are
1271-
// needed for the supervisor to drop privileges to the sandbox user.
1272-
// DAC_READ_SEARCH is needed for cross-UID /proc/<pid>/fd/ access
1273-
// for process identity resolution in network policy enforcement.
1274-
capabilities.extend(["SETUID", "SETGID", "DAC_READ_SEARCH"]);
1369+
if params.is_platform_mode {
1370+
// Platform mode: zero elevated capabilities. Compatible with
1371+
// restricted-v2 SCC and restricted Pod Security Standard.
1372+
container.insert(
1373+
"securityContext".to_string(),
1374+
serde_json::json!({
1375+
"allowPrivilegeEscalation": false,
1376+
"capabilities": {
1377+
"drop": ["ALL"]
1378+
}
1379+
}),
1380+
);
1381+
} else {
1382+
let mut capabilities: Vec<&str> = vec!["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"];
1383+
if use_user_namespaces {
1384+
capabilities.extend(["SETUID", "SETGID", "DAC_READ_SEARCH"]);
1385+
}
1386+
container.insert(
1387+
"securityContext".to_string(),
1388+
serde_json::json!({
1389+
"capabilities": {
1390+
"add": capabilities
1391+
}
1392+
}),
1393+
);
12751394
}
1276-
container.insert(
1277-
"securityContext".to_string(),
1278-
serde_json::json!({
1279-
"capabilities": {
1280-
"add": capabilities
1281-
}
1282-
}),
1283-
);
12841395

12851396
// Mount client TLS secret for mTLS to the server, plus the projected
12861397
// ServiceAccount token used to bootstrap the sandbox's gateway JWT
@@ -1363,6 +1474,7 @@ fn sandbox_template_to_k8s(
13631474
params.supervisor_image,
13641475
params.supervisor_image_pull_policy,
13651476
params.supervisor_sideload_method,
1477+
params.is_platform_mode,
13661478
);
13671479

13681480
// Inject workspace persistence (init container + PVC volume mount) so
@@ -1750,6 +1862,7 @@ mod tests {
17501862
"custom-image:latest",
17511863
"IfNotPresent",
17521864
SupervisorSideloadMethod::InitContainer,
1865+
false,
17531866
);
17541867

17551868
let sc = &pod_template["spec"]["containers"][0]["securityContext"];
@@ -1779,6 +1892,7 @@ mod tests {
17791892
"supervisor-image:latest",
17801893
"IfNotPresent",
17811894
SupervisorSideloadMethod::InitContainer,
1895+
false,
17821896
);
17831897

17841898
let sc = &pod_template["spec"]["containers"][0]["securityContext"];
@@ -1804,6 +1918,7 @@ mod tests {
18041918
"supervisor-image:latest",
18051919
"IfNotPresent",
18061920
SupervisorSideloadMethod::InitContainer,
1921+
false,
18071922
);
18081923

18091924
// Volume should be an emptyDir
@@ -1878,6 +1993,7 @@ mod tests {
18781993
"supervisor-image:latest",
18791994
"IfNotPresent",
18801995
SupervisorSideloadMethod::ImageVolume,
1996+
false,
18811997
);
18821998

18831999
let volumes = pod_template["spec"]["volumes"]
@@ -1932,6 +2048,7 @@ mod tests {
19322048
"supervisor-image:latest",
19332049
"",
19342050
SupervisorSideloadMethod::ImageVolume,
2051+
false,
19352052
);
19362053

19372054
let volume = &pod_template["spec"]["volumes"][0];

‎crates/openshell-policy/src/lib.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy {
378378
run_as_group: p.run_as_group,
379379
}),
380380
network_policies,
381+
network_enforcement: 0,
381382
}
382383
}
383384

@@ -649,6 +650,7 @@ pub fn restrictive_default_policy() -> SandboxPolicy {
649650
run_as_group: "sandbox".into(),
650651
}),
651652
network_policies: HashMap::new(),
653+
network_enforcement: 0, // NAMESPACE (default)
652654
}
653655
}
654656

@@ -1262,6 +1264,7 @@ network_policies:
12621264
filesystem: None,
12631265
landlock: None,
12641266
network_policies: HashMap::new(),
1267+
network_enforcement: 0,
12651268
};
12661269
assert!(validate_sandbox_policy(&policy).is_ok());
12671270
}

0 commit comments

Comments
 (0)