Skip to content

Commit 061ae1e

Browse files
test(features): check host.openshell.internal from a Python sandbox
The host-gateway suite starts a server on the host and checks that a sandbox from the Python workload image reaches it through host.openshell.internal. CI runs it on Docker rootful, Podman rootful and rootless, k3s and k3s-ha. Closes #5275 Signed-off-by: Matthew Grossman <[email protected]>
1 parent 8441388 commit 061ae1e

6 files changed

Lines changed: 242 additions & 1 deletion

File tree

‎.github/workflows/e2e-suites.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ jobs:
7777
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
7878
{"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
7979
{"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
80+
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"},
81+
{"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"},
82+
{"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"},
83+
{"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"},
84+
{"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"},
8085
{"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
8186
{"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
8287
{"environment":"fedora-podman-rootful-userns-default","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},

‎TESTING.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,13 @@ mise run e2e:oci-image:podman
595595
Both tasks set `OPENSHELL_TEST_CONTAINER_ENGINE`, the command that builds test
596596
images into the gateway's image store.
597597

598+
The `host-gateway` feature suite (`tests/suites/features/host-gateway`) checks
599+
that a sandbox from the Python test image reaches a server the test runs on the
600+
host through `host.openshell.internal`. CI runs it on Docker rootful, Podman
601+
rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on
602+
the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to
603+
the node's address.
604+
598605
Run the VM-backed Rust CLI e2e suite:
599606

600607
```shell

‎tests/suites/features/Cargo.lock‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/suites/features/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33

44
[workspace]
55
resolver = "2"
6-
members = ["keycloak-provider-refresh", "kubernetes-updates", "oci-image"]
6+
members = ["host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"]
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
[package]
5+
name = "host-gateway"
6+
description = "Features suite: sandboxes reach a host server through host.openshell.internal"
7+
version = "0.0.0"
8+
edition = "2024"
9+
rust-version = "1.94"
10+
license = "Apache-2.0"
11+
repository = "https://github.com/NVIDIA/OpenShell"
12+
publish = false
13+
14+
[dependencies]
15+
openshell-testkit = { path = "../../../testkit/testkit" }
16+
tempfile = "3"
17+
tokio = { version = "1.43", features = ["io-util", "net", "rt"] }
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! `host.openshell.internal` names the host from inside a sandbox, on every
5+
//! compute driver.
6+
//!
7+
//! The test starts an HTTP server where the tests run, which is the host the
8+
//! gateway runs its sandboxes on, and creates a sandbox from the Python test
9+
//! image ([`testkit::python_image`]) whose main process requests a page from
10+
//! that server through the name.
11+
12+
use testkit::{Context, Error, Result, ResultExt as _};
13+
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
14+
use tokio::net::{TcpListener, TcpStream};
15+
use tokio::task::JoinHandle;
16+
17+
/// The Python test image's virtual environment, which links to a uv-managed
18+
/// Python under `/sandbox/.uv/python`.
19+
const VENV_PYTHON: &str = "/sandbox/.venv/bin/python3";
20+
21+
/// Prints the interpreter's prefix, then requests `/` from
22+
/// `host.openshell.internal` on the port in `argv[1]` and prints the
23+
/// response. A failed connection is retried for 20 seconds, so a slow
24+
/// start doesn't fail the test, and reported on stderr.
25+
const CLIENT: &str = r#"
26+
import socket, sys, time
27+
28+
print(f"prefix={sys.prefix}", flush=True)
29+
host, port = "host.openshell.internal", int(sys.argv[1])
30+
deadline = time.monotonic() + 20
31+
while True:
32+
try:
33+
conn = socket.create_connection((host, port), timeout=10)
34+
break
35+
except OSError as error:
36+
print(f"connect {host}:{port}: {error}", file=sys.stderr, flush=True)
37+
if time.monotonic() > deadline:
38+
sys.exit(1)
39+
time.sleep(1)
40+
with conn:
41+
conn.sendall(f"GET / HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n".encode())
42+
response = b""
43+
while chunk := conn.recv(4096):
44+
response += chunk
45+
print(response.decode(errors="replace"))
46+
"#;
47+
48+
/// A sandbox from the Python test image resolves `host.openshell.internal`
49+
/// and gets a response from a server the test runs on the host. The image's
50+
/// virtual environment runs the client, so its workspace content reached the
51+
/// sandbox, and the policy lets only that Python reach the server's port.
52+
#[testkit::test]
53+
async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result {
54+
let marker = format!("host-gateway-{}", cx.run_id());
55+
let server = HostServer::start(marker.clone()).await?;
56+
let policy = PolicyFile::new(&policy(server.port))?;
57+
let port = server.port.to_string();
58+
let image = testkit::python_image();
59+
60+
let name = cx.name("hg");
61+
cx.track(&name);
62+
let create = cx
63+
.cli([
64+
"sandbox",
65+
"create",
66+
"--name",
67+
&name,
68+
"--from",
69+
&image,
70+
"--policy",
71+
policy.path()?,
72+
"--no-tty",
73+
"--",
74+
VENV_PYTHON,
75+
"-c",
76+
CLIENT,
77+
&port,
78+
])
79+
.await?
80+
.success()
81+
.context(format!("sandbox from {image} runs the client"))?;
82+
testkit::ensure!(
83+
create.stdout.contains("prefix=/sandbox/.venv"),
84+
"the client did not run in the image's virtual environment:\n{}",
85+
create.stdout
86+
);
87+
testkit::ensure!(
88+
create.stdout.contains(&marker),
89+
"the client did not get the host server's response:\n{}{}",
90+
create.stdout,
91+
create.stderr
92+
);
93+
Ok(())
94+
}
95+
96+
/// A sandbox policy for the Python test image that allows its uv-managed
97+
/// Python, which [`VENV_PYTHON`] links to, to connect to
98+
/// `host.openshell.internal` on `port`. The endpoint names only the host and
99+
/// port, as the policy docs describe, so the driver's address for the name
100+
/// is what makes it reachable, not an `allowed_ips` range.
101+
fn policy(port: u16) -> String {
102+
format!(
103+
r#"version: 1
104+
105+
filesystem_policy:
106+
include_workdir: true
107+
read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc]
108+
read_write: [/sandbox, /tmp, /dev/null]
109+
landlock:
110+
compatibility: best_effort
111+
112+
process:
113+
run_as_user: sandbox
114+
run_as_group: sandbox
115+
116+
network_policies:
117+
host_server:
118+
name: host_server
119+
endpoints:
120+
- host: host.openshell.internal
121+
port: {port}
122+
binaries:
123+
- path: /sandbox/.uv/python/**/python*
124+
"#
125+
)
126+
}
127+
128+
/// An HTTP server on the host that answers every request with `body`. It
129+
/// stops when dropped.
130+
struct HostServer {
131+
port: u16,
132+
task: JoinHandle<()>,
133+
}
134+
135+
impl HostServer {
136+
/// Listen on every host address, since drivers map
137+
/// `host.openshell.internal` to different ones.
138+
async fn start(body: String) -> Result<Self> {
139+
let listener = TcpListener::bind(("0.0.0.0", 0))
140+
.await
141+
.map_err(|error| Error::from_error(&error))
142+
.context("bind the host server")?;
143+
let port = listener
144+
.local_addr()
145+
.map_err(|error| Error::from_error(&error))
146+
.context("read the host server's address")?
147+
.port();
148+
let task = tokio::spawn(async move {
149+
while let Ok((stream, _)) = listener.accept().await {
150+
tokio::spawn(respond(stream, body.clone()));
151+
}
152+
});
153+
Ok(Self { port, task })
154+
}
155+
}
156+
157+
impl Drop for HostServer {
158+
fn drop(&mut self) {
159+
self.task.abort();
160+
}
161+
}
162+
163+
/// Read one request's headers from `stream` and answer with `body`.
164+
async fn respond(mut stream: TcpStream, body: String) {
165+
let _ = stream.set_nodelay(true);
166+
let mut request = Vec::new();
167+
let mut buf = [0_u8; 1024];
168+
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
169+
match stream.read(&mut buf).await {
170+
Ok(read) if read > 0 && request.len() < 64 * 1024 => {
171+
request.extend_from_slice(&buf[..read]);
172+
}
173+
_ => return,
174+
}
175+
}
176+
let response = format!(
177+
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
178+
body.len()
179+
);
180+
let _ = stream.write_all(response.as_bytes()).await;
181+
let _ = stream.shutdown().await;
182+
}
183+
184+
/// A sandbox policy written to a temporary file for `sandbox create --policy`.
185+
struct PolicyFile(tempfile::NamedTempFile);
186+
187+
impl PolicyFile {
188+
fn new(contents: &str) -> Result<Self> {
189+
let file = tempfile::NamedTempFile::new()
190+
.map_err(|error| Error::from_error(&error))
191+
.context("create policy file")?;
192+
std::fs::write(file.path(), contents)
193+
.map_err(|error| Error::from_error(&error))
194+
.context(format!("write {}", file.path().display()))?;
195+
Ok(Self(file))
196+
}
197+
198+
fn path(&self) -> Result<&str> {
199+
let path = self.0.path();
200+
path.to_str()
201+
.ok_or_else(|| Error::msg(format!("{} is not UTF-8", path.display())))
202+
}
203+
}

0 commit comments

Comments
 (0)