Skip to content

Commit b7f666c

Browse files
committed
test(e2e): verify host-only VM trust delivery and drop curl dependency
The additional-CA lane's VM assertions read the destination CA out of the guest overlay and a guest-side supervisor-argv marker — artifacts from the guest-supervisor design that no longer exist now that the VM driver stages the CA host-only and runs the supervisor on the host. Inspect the driver-staged host artifact (digest-compared against the source), assert the guest overlay contains no CA material at the old mount paths, and keep the serial-console no-disclosure check. The workload validation scripts also ran curl, which exists only in the container-engine fixture image; the Kubernetes and VM lanes use the uv-based workload image and died instantly with "canonical main process already finished". Rewrite the main-process, exec, and lifecycle validation scripts in python3 (present in every lane's workload image). The network-policy binary identity is the resolved /proc/<pid>/exe target, so the versioned python3.12 binaries are allowlisted rather than the unversioned symlinks. The hostname mismatch check keeps its semantics: same reachable fixture, mismatched TLS SNI, and the failure must specifically be hostname verification. Addresses GATOR-291f43b2-04. Signed-off-by: Jesse Jaggars <[email protected]>
1 parent bd881f5 commit b7f666c

1 file changed

Lines changed: 159 additions & 104 deletions

File tree

‎e2e/rust/tests/additional_ca.rs‎

Lines changed: 159 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,13 @@ network_policies:
9393
access: full
9494
enforcement: enforce
9595
binaries:
96-
- path: /usr/bin/curl
96+
# The network-policy binary identity is the resolved /proc/<pid>/exe
97+
# target, so the versioned real binaries must be allowlisted rather
98+
# than the unversioned symlinks: python3.12 on the Noble fixture image
99+
# (/usr/bin/python3 -> python3.12) and on the uv-based Kubernetes/VM
100+
# workload image (/usr/local/bin/python3 -> python3.12).
101+
- path: /usr/bin/python3.12
102+
- path: /usr/local/bin/python3.12
97103
"#
98104
);
99105
file.write_all(policy.as_bytes())
@@ -455,23 +461,40 @@ fn inspect_vm_overlay() -> Result<(), String> {
455461
std::env::var("OPENSHELL_E2E_ADDITIONAL_CA_CERT")
456462
.map_err(|error| format!("additional CA source path missing: {error}"))?,
457463
);
458-
let mut overlays = std::fs::read_dir(state_dir.join("sandboxes"))
464+
let mut sandboxes = std::fs::read_dir(state_dir.join("sandboxes"))
459465
.map_err(|error| format!("read VM sandbox state: {error}"))?
460466
.filter_map(Result::ok)
461-
.map(|entry| entry.path().join("overlay.ext4"))
462-
.filter(|path| path.is_file())
467+
.map(|entry| entry.path())
468+
.filter(|path| path.join("overlay.ext4").is_file())
463469
.collect::<Vec<_>>();
464-
overlays.sort();
465-
let [overlay] = overlays.as_slice() else {
470+
sandboxes.sort();
471+
let [sandbox_state] = sandboxes.as_slice() else {
466472
return Err(format!(
467-
"expected one VM sandbox overlay, found {overlays:?}"
473+
"expected one VM sandbox overlay, found {sandboxes:?}"
468474
));
469475
};
476+
let overlay = sandbox_state.join("overlay.ext4");
477+
478+
// Host-only delivery: the driver stages the normalized destination CA as
479+
// a driver-owned artifact beside the sandbox state, where the host
480+
// supervisor consumes it. The guest never receives CA material through
481+
// its overlay, rootfs, argv, or console.
482+
let staged = std::fs::read(sandbox_state.join("additional-ca.crt"))
483+
.map_err(|error| format!("read host-staged VM destination CA: {error}"))?;
484+
let source = std::fs::read(&source_ca)
485+
.map_err(|error| format!("read additional CA source for digest comparison: {error}"))?;
486+
if Sha256::digest(&staged) != Sha256::digest(&source) {
487+
return Err(
488+
"host-staged VM destination CA digest does not match normalized source".to_string(),
489+
);
490+
}
470491

471-
let debugfs_read = |guest_path: &str| -> Result<Vec<u8>, String> {
492+
// debugfs exits successfully even for missing inodes, so presence is
493+
// detected from the "Inode:" stat header rather than the exit status.
494+
let overlay_has = |guest_path: &str| -> Result<bool, String> {
472495
let output = Command::new("debugfs")
473-
.args(["-R", &format!("cat {guest_path}")])
474-
.arg(overlay)
496+
.args(["-R", &format!("stat {guest_path}")])
497+
.arg(&overlay)
475498
.output()
476499
.map_err(|error| format!("run debugfs overlay inspection: {error}"))?;
477500
if !output.status.success() {
@@ -480,36 +503,24 @@ fn inspect_vm_overlay() -> Result<(), String> {
480503
String::from_utf8_lossy(&output.stderr)
481504
));
482505
}
483-
Ok(output.stdout)
506+
let stdout = String::from_utf8_lossy(&output.stdout);
507+
Ok(stdout.contains("Inode:"))
484508
};
485-
486-
let staged = debugfs_read("/upper/etc/openshell-tls/additional-ca.crt")?;
487-
let source = std::fs::read(&source_ca)
488-
.map_err(|error| format!("read additional CA source for digest comparison: {error}"))?;
489-
if Sha256::digest(&staged) != Sha256::digest(&source) {
490-
return Err(
491-
"VM overlay destination CA digest does not match normalized source".to_string(),
492-
);
493-
}
494-
495-
let args = String::from_utf8(debugfs_read("/upper/opt/openshell/supervisor-args")?)
496-
.map_err(|error| format!("VM supervisor argument marker is not UTF-8: {error}"))?;
497-
let lines = args.lines().collect::<Vec<_>>();
498-
if lines != ["--additional-ca-bundle", ADDITIONAL_CA_MOUNT_PATH] {
499-
return Err(format!(
500-
"unexpected VM supervisor argument marker: {lines:?}"
501-
));
509+
for guest_path in [
510+
"/upper/etc/openshell-tls/additional-ca.crt",
511+
"/etc/openshell-tls/additional-ca.crt",
512+
] {
513+
if overlay_has(guest_path)? {
514+
return Err(format!(
515+
"VM guest overlay received CA material at {guest_path}; the \
516+
destination CA must stay host-only"
517+
));
518+
}
502519
}
503520

504-
let console = overlay
505-
.parent()
506-
.expect("overlay has sandbox state parent")
507-
.join("rootfs-console.log");
521+
let console = sandbox_state.join("rootfs-console.log");
508522
let console = std::fs::read_to_string(&console)
509523
.map_err(|error| format!("read VM serial log: {error}"))?;
510-
if !console.contains("supervisor arguments from driver: 2 entries") {
511-
return Err("VM serial log did not confirm driver-owned supervisor argv".to_string());
512-
}
513524
if console.contains("-----BEGIN CERTIFICATE-----") {
514525
return Err("VM serial log disclosed certificate material".to_string());
515526
}
@@ -936,39 +947,82 @@ fn delete_kubernetes_config_map(identity: &str) -> Result<(), String> {
936947
Ok(())
937948
}
938949

939-
fn hostname_validation_script(
940-
matching_url: &str,
941-
mismatch_url: &str,
942-
mismatch_connect: &str,
943-
) -> String {
950+
/// Exec argv that fetches `url` with the workload's default TLS verification
951+
/// and prints the body. Exits nonzero when validation fails, so a failed exec
952+
/// surfaces the verification error. Uses python3 because the Kubernetes and
953+
/// VM workload images do not ship curl.
954+
fn python_fetch_argv(driver: &str, url: &str) -> Vec<String> {
955+
let script = format!(
956+
"import sys, urllib.request\ntry:\n with urllib.request.urlopen({url:?}, timeout=30) as response:\n sys.stdout.write(response.read().decode())\nexcept Exception as error:\n sys.exit(str(error))\n"
957+
);
958+
vec![
959+
python_binary_for_driver(driver).to_string(),
960+
"-c".to_string(),
961+
script,
962+
]
963+
}
964+
965+
fn python_binary_for_driver(driver: &str) -> &'static str {
966+
match driver {
967+
// The container-engine lanes run the Noble fixture image.
968+
"docker" | "podman" => "/usr/bin/python3",
969+
// The Kubernetes and VM lanes run the uv-based workload image, where
970+
// python3 lives under /usr/local/bin and curl is not present.
971+
_ => "/usr/local/bin/python3",
972+
}
973+
}
974+
975+
fn hostname_validation_script(matching_host: &str, port: u16) -> String {
944976
format!(
945-
r#"set -eu
946-
response="$(curl --fail --silent --show-error --max-time 30 '{matching_url}')"
947-
test "$response" = '{{"additional_ca":"trusted"}}'
948-
set +e
949-
curl --fail --silent --show-error --verbose --max-time 15 --connect-to '{mismatch_connect}' '{mismatch_url}' >/tmp/mismatch.out 2>/tmp/mismatch.err
950-
mismatch_status=$?
951-
set -e
952-
# --connect-to proves the mismatch uses the same reachable fixture as the
953-
# matching request, while preserving the mismatch URL and TLS SNI. The
954-
# fixture's private CA is trusted, so curl status 60 must specifically be its
955-
# hostname-verification failure, not an untrusted issuer or routing error.
956-
if [ "$mismatch_status" -ne 60 ]; then
957-
echo "hostname mismatch returned curl status $mismatch_status, expected 60" >&2
958-
cat /tmp/mismatch.err >&2
959-
exit 1
960-
fi
961-
if ! grep -Fq "no alternative certificate subject name matches target host name '{MISMATCH_HOST}'" /tmp/mismatch.err; then
962-
echo "hostname mismatch did not report hostname verification failure" >&2
963-
cat /tmp/mismatch.err >&2
964-
exit 1
965-
fi
966-
curl --fail --silent --show-error --max-time 30 https://example.com/ >/dev/null
967-
echo matching-host-trusted
968-
echo hostname-mismatch-rejected
969-
echo public-root-trusted
970-
echo {READY_MARKER}
971-
while true; do sleep 1; done"#,
977+
r#"
978+
import json
979+
import socket
980+
import ssl
981+
import sys
982+
import time
983+
import urllib.request
984+
985+
port = {port}
986+
matching_host = {matching_host:?}
987+
mismatch_host = {MISMATCH_HOST:?}
988+
989+
# 1. The matching destination must validate through the workload trust
990+
# store (system roots plus the provisioned destination CA).
991+
with urllib.request.urlopen("https://" + matching_host + ":%d/" % port, timeout=30) as response:
992+
body = response.read()
993+
if json.loads(body) != {{"additional_ca": "trusted"}}:
994+
sys.exit("unexpected matching-host body: %r" % (body,))
995+
print("matching-host-trusted", flush=True)
996+
997+
# 2. Hostname mismatch against the same reachable fixture: connect to the
998+
# matching host's address while presenting the mismatch hostname as TLS
999+
# SNI. The fixture's private CA is trusted, so the failure must be the
1000+
# hostname check itself, not an untrusted issuer or a routing error.
1001+
address = socket.getaddrinfo(matching_host, port, proto=socket.IPPROTO_TCP)[0][4]
1002+
sock = socket.create_connection(address, timeout=15)
1003+
try:
1004+
context = ssl.create_default_context()
1005+
try:
1006+
context.wrap_socket(sock, server_hostname=mismatch_host)
1007+
except ssl.SSLCertVerificationError as error:
1008+
message = str(error).lower()
1009+
if "mismatch" not in message and "doesn't match" not in message:
1010+
sys.exit("unexpected hostname verification error: %s" % error)
1011+
else:
1012+
sys.exit("hostname mismatch was accepted by the destination trust")
1013+
finally:
1014+
sock.close()
1015+
print("hostname-mismatch-rejected", flush=True)
1016+
1017+
# 3. Public roots must keep validating alongside the destination CA.
1018+
with urllib.request.urlopen("https://example.com/", timeout=30) as response:
1019+
response.read()
1020+
print("public-root-trusted", flush=True)
1021+
1022+
print({READY_MARKER:?}, flush=True)
1023+
while True:
1024+
time.sleep(1)
1025+
"#,
9721026
)
9731027
}
9741028

@@ -982,13 +1036,11 @@ async fn additional_ca_trusts_matching_and_public_hosts_and_rejects_mismatch() {
9821036
let policy = write_policy(port).expect("write additional CA e2e policy");
9831037
let policy_path = policy.path().to_str().expect("policy path must be UTF-8");
9841038
let matching_url = format!("https://host.openshell.internal:{port}/");
985-
let mismatch_url = format!("https://{MISMATCH_HOST}:{port}/");
986-
let mismatch_connect = format!("{MISMATCH_HOST}:{port}:host.openshell.internal:{port}");
987-
let script = hostname_validation_script(&matching_url, &mismatch_url, &mismatch_connect);
1039+
let script = hostname_validation_script("host.openshell.internal", port);
9881040

9891041
let mut sandbox = SandboxGuard::create_keep_with_args(
9901042
&["--policy", policy_path],
991-
&["sh", "-lc", &script],
1043+
&[python_binary_for_driver(driver), "-c", script.as_str()],
9921044
READY_MARKER,
9931045
)
9941046
.await
@@ -1027,16 +1079,10 @@ async fn additional_ca_trusts_matching_and_public_hosts_and_rejects_mismatch() {
10271079
replace_staged_material(driver, kubernetes_identity.as_deref())
10281080
.expect("replace staged destination trust with invalid material");
10291081
if driver == "kubernetes" || driver == "vm" {
1082+
let fetch_argv = python_fetch_argv(driver, &matching_url);
1083+
let fetch_argv = fetch_argv.iter().map(String::as_str).collect::<Vec<_>>();
10301084
let output = sandbox
1031-
.exec(&[
1032-
"curl",
1033-
"--fail",
1034-
"--silent",
1035-
"--show-error",
1036-
"--max-time",
1037-
"30",
1038-
&matching_url,
1039-
])
1085+
.exec(&fetch_argv)
10401086
.await
10411087
.expect("running supervisor retains startup trust after staged material changes");
10421088
assert!(
@@ -1052,7 +1098,7 @@ async fn additional_ca_trusts_matching_and_public_hosts_and_rejects_mismatch() {
10521098
} else if driver == "vm" {
10531099
let mut retained_snapshot_sandbox = SandboxGuard::create_keep_with_args(
10541100
&["--policy", policy_path],
1055-
&["sh", "-lc", &script],
1101+
&[python_binary_for_driver(driver), "-c", script.as_str()],
10561102
READY_MARKER,
10571103
)
10581104
.await
@@ -1086,16 +1132,13 @@ async fn additional_ca_trusts_matching_and_public_hosts_and_rejects_mismatch() {
10861132
)
10871133
.await
10881134
.expect("create lifecycle sandbox using restored destination CA");
1135+
let lifecycle_fetch_argv = python_fetch_argv(driver, &matching_url);
1136+
let lifecycle_fetch_argv = lifecycle_fetch_argv
1137+
.iter()
1138+
.map(String::as_str)
1139+
.collect::<Vec<_>>();
10891140
let private_ca_output = lifecycle_sandbox
1090-
.exec(&[
1091-
"curl",
1092-
"--fail",
1093-
"--silent",
1094-
"--show-error",
1095-
"--max-time",
1096-
"30",
1097-
&matching_url,
1098-
])
1141+
.exec(&lifecycle_fetch_argv)
10991142
.await
11001143
.expect("lifecycle sandbox trusts the private CA before removal");
11011144
assert!(
@@ -1146,22 +1189,34 @@ async fn additional_ca_trusts_matching_and_public_hosts_and_rejects_mismatch() {
11461189
);
11471190

11481191
let removed_validation_script = format!(
1149-
r#"set -eu
1150-
test "$(cat {LIFECYCLE_DURABLE_PATH})" = "{LIFECYCLE_DURABLE_MARKER}"
1151-
test ! -e '{ADDITIONAL_CA_MOUNT_PATH}'
1152-
set +e
1153-
curl --fail --silent --show-error --max-time 30 '{matching_url}' >/tmp/removed.out 2>/tmp/removed.err
1154-
removed_status=$?
1155-
set -e
1156-
if [ "$removed_status" -eq 0 ]; then
1157-
echo "private CA remained trusted after configuration removal" >&2
1158-
exit 1
1159-
fi
1160-
echo gateway-authenticated-exec-works
1161-
echo same-sandbox-private-ca-rejected"#
1192+
r#"import os
1193+
import sys
1194+
import urllib.request
1195+
1196+
marker = open({LIFECYCLE_DURABLE_PATH:?}).read().strip()
1197+
if marker != {LIFECYCLE_DURABLE_MARKER:?}:
1198+
sys.exit("lifecycle durable marker was lost across the restart")
1199+
if os.path.exists({ADDITIONAL_CA_MOUNT_PATH:?}):
1200+
sys.exit("destination CA mount path survived configuration removal")
1201+
try:
1202+
urllib.request.urlopen({matching_url:?}, timeout=30).read()
1203+
except Exception:
1204+
print("same-sandbox-private-ca-rejected", flush=True)
1205+
else:
1206+
sys.exit("private CA remained trusted after configuration removal")
1207+
print("gateway-authenticated-exec-works", flush=True)"#
11621208
);
1209+
let removed_validation_argv = [
1210+
python_binary_for_driver(driver).to_string(),
1211+
"-c".to_string(),
1212+
removed_validation_script,
1213+
];
1214+
let removed_validation_argv = removed_validation_argv
1215+
.iter()
1216+
.map(String::as_str)
1217+
.collect::<Vec<_>>();
11631218
let removed_validation_output = lifecycle_sandbox
1164-
.exec(&["sh", "-lc", &removed_validation_script])
1219+
.exec(&removed_validation_argv)
11651220
.await
11661221
.expect("gateway-authenticated exec should work after lifecycle restart");
11671222
assert!(

0 commit comments

Comments
 (0)