Skip to content

Commit c55df65

Browse files
committed
test(e2e): run the edge tunnel test over TLS
ws_tunnel_status_through_edge_proxy returned early, and so reported success, whenever it could not set up its own preconditions. It could not in any lane that starts its own gateway, so it never exercised the WebSocket tunnel: - It looked for the gateway URL on a `gateway info` line that mentions "endpoint" or "gateway", but the URL is printed on the `Server:` line. - It required a plaintext endpoint, but the CLI only tunnels over https://, and those lanes register an mTLS gateway at an https:// endpoint. Read the endpoint from the registered gateway metadata instead of scraping `gateway info`. For a gateway with a registered CA, trust it by pointing the CLI at it with SSL_CERT_FILE, since the tunnel client uses the system trust store; otherwise use the system store. Fail, instead of returning early, when the gateway is unhealthy or not https://, and require the CLI's debug log to show the completed WebSocket handshake so a status that bypassed the tunnel cannot pass. The test needs a gateway with the WebSocket tunnel enabled, so move it behind a new e2e-edge-tunnel feature that the docker and podman lanes enable. The Kubernetes and VM lanes cannot exercise it and no longer compile it. Contributes to NVIDIA#3712 Signed-off-by: politerealism <[email protected]>
1 parent 0d4d22e commit c55df65

2 files changed

Lines changed: 156 additions & 68 deletions

File tree

‎e2e/rust/Cargo.toml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,19 @@ e2e = []
2626
# when it has resolved an IP.
2727
e2e-host-gateway = ["e2e"]
2828
e2e-local-container-driver = ["e2e"]
29-
e2e-docker = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"]
29+
# Selects tests that need a gateway with the WebSocket tunnel enabled, reached
30+
# over `https://`. The lanes that start their own gateway (docker, podman)
31+
# configure it; the Kubernetes and VM lanes do not.
32+
e2e-edge-tunnel = ["e2e"]
33+
e2e-docker = ["e2e", "e2e-host-gateway", "e2e-local-container-driver", "e2e-edge-tunnel"]
3034
e2e-gpu = ["e2e"]
3135
e2e-docker-gpu = ["e2e-docker", "e2e-gpu"]
3236
e2e-kubernetes = ["e2e"]
3337
e2e-kubernetes-ha = ["e2e-kubernetes"]
3438
e2e-kubernetes-credential-drivers = ["e2e-kubernetes"]
3539
e2e-kubernetes-workspace-managed = ["e2e-kubernetes"]
3640
e2e-kubernetes-workspace-operator = ["e2e-kubernetes"]
37-
e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"]
41+
e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver", "e2e-edge-tunnel"]
3842
e2e-podman-gpu = ["e2e-podman", "e2e-gpu"]
3943
e2e-oidc-pkce = []
4044
e2e-vm = ["e2e", "e2e-host-gateway"]

‎e2e/rust/tests/edge_tunnel_e2e.rs‎

Lines changed: 150 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,35 @@
55

66
//! E2E tests for edge tunnel auth flow against a running gateway.
77
//!
8-
//! Prerequisites:
9-
//! - A running openshell gateway
10-
//! - For WS tunnel coverage, the gateway's HTTP endpoint is accessible (no TLS)
8+
//! `gateway_status_reports_healthy` runs wherever the common `e2e` suite runs.
9+
//! The tunnel test needs a gateway with special configuration, so it is a
10+
//! feature test behind `e2e-edge-tunnel`, which the Docker and Podman lanes
11+
//! enable.
12+
//!
13+
//! Prerequisites for the tunnel test:
14+
//! - A running openshell gateway started with the WebSocket tunnel enabled
15+
//! (`--enable-websocket-tunnel true`) and reachable over `https://`. The CLI
16+
//! only routes through the tunnel for `https://` gateways, so a plaintext
17+
//! gateway cannot exercise it.
1118
//! - The `openshell` binary (built automatically from the workspace)
1219
//!
13-
//! These tests exercise the full CLI → WS tunnel → gRPC flow.
20+
//! The tunnel test reads the active gateway's endpoint from the user config
21+
//! the target registered (`$XDG_CONFIG_HOME/openshell/gateways/<name>`), not
22+
//! from the system gateway directory. When that gateway has a registered CA
23+
//! (`mtls/ca.crt`) the test trusts it by pointing the CLI at it with
24+
//! `SSL_CERT_FILE`, because the tunnel client uses the system trust store.
25+
//! Without one it relies on the system trust store, as a gateway with a
26+
//! publicly trusted certificate would.
27+
//!
28+
//! The tunnel client presents no client certificate. The gateway only requests
29+
//! client certificates, so the WebSocket upgrade does not need one.
1430
//!
1531
//! Environment variables:
1632
//! - `OPENSHELL_GATEWAY`: Name of the active gateway (standard e2e var)
17-
//! - `OPENSHELL_GATEWAY_ENDPOINT`: Optional direct plaintext endpoint.
18-
//!
19-
//! The edge-tunnel path requires a gateway endpoint that accepts plaintext HTTP.
33+
//! - `XDG_CONFIG_HOME`: Config home holding the registered gateways
2034
35+
#[cfg(feature = "e2e-edge-tunnel")]
36+
use std::path::{Path, PathBuf};
2137
use std::process::Stdio;
2238

2339
use openshell_e2e::harness::binary::openshell_cmd;
@@ -38,15 +54,28 @@ async fn run_cli(args: &[&str]) -> (String, i32) {
3854

3955
/// Run `openshell <args>` with a custom config directory so the CLI reads
4056
/// our seeded gateway metadata and edge token instead of the real config.
41-
async fn run_cli_with_config(config_dir: &std::path::Path, args: &[&str]) -> (String, i32) {
57+
///
58+
/// Debug logging is on so the CLI records the tunnel handshake. When
59+
/// `ssl_cert_file` is set, the CLI trusts that CA bundle instead of the system
60+
/// store.
61+
#[cfg(feature = "e2e-edge-tunnel")]
62+
async fn run_cli_with_config(
63+
config_dir: &Path,
64+
ssl_cert_file: Option<&Path>,
65+
args: &[&str],
66+
) -> (String, i32) {
4267
let mut cmd = openshell_cmd();
4368
cmd.args(args)
4469
.env("XDG_CONFIG_HOME", config_dir)
4570
.env("HOME", config_dir)
71+
.env("RUST_LOG", "openshell_cli=debug,openshell_sdk=debug")
4672
.env_remove("OPENSHELL_GATEWAY")
4773
.env_remove("OPENSHELL_GATEWAY_ENDPOINT")
4874
.stdout(Stdio::piped())
4975
.stderr(Stdio::piped());
76+
if let Some(ca) = ssl_cert_file {
77+
cmd.env("SSL_CERT_FILE", ca);
78+
}
5079

5180
let output = cmd.output().await.expect("spawn openshell");
5281
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
@@ -59,8 +88,9 @@ async fn run_cli_with_config(config_dir: &std::path::Path, args: &[&str]) -> (St
5988
/// Seed a temporary config directory with gateway metadata that has
6089
/// `auth_mode: "cloudflare_jwt"`, a stored edge token, and an active gateway
6190
/// pointing at the given endpoint.
91+
#[cfg(feature = "e2e-edge-tunnel")]
6292
fn seed_edge_gateway_config(
63-
config_dir: &std::path::Path,
93+
config_dir: &Path,
6494
gateway_name: &str,
6595
gateway_endpoint: &str,
6696
edge_token: &str,
@@ -123,6 +153,65 @@ async fn gateway_status_reports_healthy() {
123153
);
124154
}
125155

156+
/// Where the active e2e gateway is and how to trust it.
157+
#[cfg(feature = "e2e-edge-tunnel")]
158+
struct GatewayTarget {
159+
endpoint: String,
160+
/// CA that signed the gateway's server certificate, when the target
161+
/// registered one. Gateways with a publicly trusted certificate have none.
162+
ca_cert: Option<PathBuf>,
163+
}
164+
165+
/// Resolve the active gateway from the config the target registered, rather
166+
/// than scraping `gateway info` output.
167+
#[cfg(feature = "e2e-edge-tunnel")]
168+
fn active_gateway_target() -> GatewayTarget {
169+
let config_home = std::env::var_os("XDG_CONFIG_HOME")
170+
.map(PathBuf::from)
171+
.or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".config")))
172+
.expect("XDG_CONFIG_HOME or HOME must be set");
173+
let openshell_dir = config_home.join("openshell");
174+
175+
let name = std::env::var("OPENSHELL_GATEWAY")
176+
.ok()
177+
.filter(|name| !name.is_empty())
178+
.or_else(|| {
179+
std::fs::read_to_string(openshell_dir.join("active_gateway"))
180+
.ok()
181+
.map(|name| name.trim().to_string())
182+
})
183+
.expect("no active gateway: set OPENSHELL_GATEWAY or register one");
184+
let gateway_dir = openshell_dir.join("gateways").join(&name);
185+
186+
let metadata_path = gateway_dir.join("metadata.json");
187+
let metadata = std::fs::read_to_string(&metadata_path)
188+
.unwrap_or_else(|e| panic!("read {}: {e}", metadata_path.display()));
189+
let metadata: serde_json::Value =
190+
serde_json::from_str(&metadata).expect("parse gateway metadata");
191+
let endpoint = metadata["gateway_endpoint"]
192+
.as_str()
193+
.unwrap_or_else(|| panic!("no gateway_endpoint in {}", metadata_path.display()))
194+
.to_string();
195+
196+
let ca_cert = Some(gateway_dir.join("mtls").join("ca.crt")).filter(|ca| ca.is_file());
197+
GatewayTarget { endpoint, ca_cert }
198+
}
199+
200+
/// Whether `openshell status` output reports a healthy gateway: a `Status:`
201+
/// line whose value is exactly "Connected" or "Healthy". Substring matches
202+
/// would accept "Disconnected" and "unhealthy".
203+
#[cfg(feature = "e2e-edge-tunnel")]
204+
fn reports_healthy(output: &str) -> bool {
205+
output.lines().any(|line| {
206+
line.trim().strip_prefix("Status:").is_some_and(|value| {
207+
matches!(
208+
value.trim().to_lowercase().as_str(),
209+
"connected" | "healthy"
210+
)
211+
})
212+
})
213+
}
214+
126215
// -------------------------------------------------------------------
127216
// Test 13: gRPC through the WS tunnel proxy (edge token path)
128217
// -------------------------------------------------------------------
@@ -134,87 +223,82 @@ async fn gateway_status_reports_healthy() {
134223
/// CLI → local TCP proxy → WebSocket → /_ws_tunnel → loopback TCP → gRPC
135224
///
136225
/// The test seeds a temporary config directory with edge auth metadata and a
137-
/// dummy token, then runs `openshell status` against the live plaintext
226+
/// dummy token, then runs `openshell status` against the live `https://`
138227
/// gateway.
139228
///
140-
/// Note: The dummy token won't be validated (no edge auth middleware on
141-
/// the plaintext gateway), but it triggers the CLI's tunnel proxy codepath.
229+
/// Note: The dummy token won't be validated (no edge auth middleware on the
230+
/// gateway), but it triggers the CLI's tunnel proxy codepath. The CLI only
231+
/// tunnels over `https://`, so the test fails for a plaintext gateway instead
232+
/// of passing without using the tunnel. It also requires the CLI's debug log
233+
/// to show the completed WebSocket handshake, so a healthy status that did not
234+
/// use the tunnel cannot pass. That depends on a diagnostic line, which the
235+
/// testing strategy RFC (0016) discourages for conformance tests; it is kept
236+
/// here because nothing else observable separates the two paths.
237+
#[cfg(feature = "e2e-edge-tunnel")]
142238
#[tokio::test]
143239
async fn ws_tunnel_status_through_edge_proxy() {
144-
let (original_status, _) = run_cli(&["status"]).await;
240+
let (original_status, status_code) = run_cli(&["status"]).await;
145241
let clean_status = strip_ansi(&original_status);
146242

147-
// Only run this test if we have a healthy gateway to test against.
148-
if !clean_status.to_lowercase().contains("healthy")
149-
&& !clean_status.to_lowercase().contains("running")
150-
&& !clean_status.to_lowercase().contains("connected")
151-
&& !clean_status.contains("✓")
152-
{
153-
eprintln!("Skipping ws_tunnel test: no healthy gateway available");
154-
return;
155-
}
243+
// This test needs a healthy gateway to test against.
244+
assert_eq!(
245+
status_code, 0,
246+
"ws_tunnel test needs a healthy gateway, but status failed:\n{clean_status}"
247+
);
248+
assert!(
249+
reports_healthy(&clean_status),
250+
"ws_tunnel test needs a healthy gateway:\n{clean_status}"
251+
);
156252

157-
// Get the gateway endpoint from the gateway metadata.
158-
let (info_output, info_code) = run_cli(&["gateway", "info"]).await;
159-
assert_eq!(info_code, 0, "gateway info should succeed:\n{info_output}");
160-
161-
let info_clean = strip_ansi(&info_output);
162-
163-
// Extract the gateway endpoint from the info output.
164-
// The format varies, but it should contain a URL-like string.
165-
let endpoint = info_clean.lines().find_map(|line| {
166-
if line.to_lowercase().contains("endpoint") || line.to_lowercase().contains("gateway") {
167-
// Try to extract a URL from the line
168-
line.split_whitespace()
169-
.find(|word| word.starts_with("http://") || word.starts_with("https://"))
170-
.map(String::from)
171-
} else {
172-
None
173-
}
174-
});
253+
let GatewayTarget { endpoint, ca_cert } = active_gateway_target();
175254

176-
let Some(endpoint) = endpoint else {
177-
eprintln!(
178-
"Skipping ws_tunnel test: could not extract gateway endpoint from:\n{info_clean}"
179-
);
180-
return;
181-
};
182-
183-
// For the WS tunnel test, we need the endpoint to be HTTP (plaintext).
184-
// If it's HTTPS, the WS tunnel test requires TLS negotiation which
185-
// complicates things. Skip if the gateway isn't plaintext.
186-
if !endpoint.starts_with("http://") {
187-
eprintln!(
188-
"Skipping ws_tunnel test: gateway endpoint is not plaintext HTTP: {endpoint}\n\
189-
Use a plaintext local gateway such as `mise run gateway:docker` for this test."
190-
);
191-
return;
192-
}
255+
// The CLI only routes through the tunnel for https:// gateways.
256+
assert!(
257+
endpoint.starts_with("https://"),
258+
"ws_tunnel test needs an https:// gateway with the WebSocket tunnel enabled, got: {endpoint}\n\
259+
A plaintext gateway, such as one reused through OPENSHELL_GATEWAY_ENDPOINT endpoint mode, \
260+
cannot exercise the tunnel. Use a lane-started gateway."
261+
);
262+
263+
// A gateway with a lane-local CA is not trusted by the system store, and
264+
// the tunnel client uses the system store, so point it at that CA. With no
265+
// registered CA, rely on the system store.
266+
let tmpdir = tempfile::tempdir().expect("create temp config dir");
193267

194268
// Seed a temporary config directory with edge auth metadata pointing at
195269
// the live gateway. The dummy token triggers the WS tunnel codepath
196270
// without requiring real edge auth middleware.
197-
let tmpdir = tempfile::tempdir().expect("create temp config dir");
198271
seed_edge_gateway_config(
199272
tmpdir.path(),
200273
"edge-tunnel-test",
201274
&endpoint,
202275
"dummy-test-jwt",
203276
);
204277

205-
let (output, code) =
206-
run_cli_with_config(tmpdir.path(), &["--gateway", "edge-tunnel-test", "status"]).await;
278+
let (output, code) = run_cli_with_config(
279+
tmpdir.path(),
280+
ca_cert.as_deref(),
281+
&["--gateway", "edge-tunnel-test", "status"],
282+
)
283+
.await;
207284

208285
let clean = strip_ansi(&output);
209286
assert_eq!(
210-
code, 0,
211-
"openshell status through WS tunnel should exit 0:\n{clean}"
287+
code,
288+
0,
289+
"openshell status through WS tunnel should exit 0 (trusted CA: {}):\n{clean}",
290+
ca_cert
291+
.as_deref()
292+
.map_or_else(|| "system store".to_string(), |ca| ca.display().to_string())
212293
);
294+
// A healthy status alone does not show the tunnel carried it, so require
295+
// the CLI's own record of the completed WebSocket handshake.
213296
assert!(
214-
clean.to_lowercase().contains("healthy")
215-
|| clean.to_lowercase().contains("running")
216-
|| clean.to_lowercase().contains("connected")
217-
|| clean.contains("✓"),
297+
clean.contains("WebSocket connected to edge"),
298+
"status should have gone through the WS tunnel:\n{clean}"
299+
);
300+
assert!(
301+
reports_healthy(&clean),
218302
"status through WS tunnel should report healthy:\n{clean}"
219303
);
220304
}

0 commit comments

Comments
 (0)