Skip to content

Commit 729061e

Browse files
committed
fix(podman): trust an operator-supplied additional CA for direct egress
Since v0.1.2 the Podman supervisor runs as its own container, separate from the workload image, so it can no longer inherit a private CA baked into the workload's system trust store. Policy-allowed, inspected HTTPS requests to destinations signed by that CA fail upstream TLS establishment, and the only workaround is rebuilding the supervisor image with the CA baked in. Add a Podman-only additional_ca_bundle setting, independent of the existing proxy_ca_bundle (which requires https_proxy), that bind-mounts an operator PEM into the supervisor container and folds it into the upstream trust store alongside the system CA bundle. Validated on the gateway host at sandbox-create time, matching the existing corporate proxy CA bundle's validation, so a misconfigured path or certificate-free bundle fails closed immediately rather than producing a failed sandbox once the supervisor container starts. Scoped to Podman only, reusing the existing additive-bundle mechanism without new gateway.toml schema, proto fields, or cross-driver plumbing -- a narrower alternative to the in-flight cross-driver implementation in #3292, which remains the intended long-term direction. Fixes #3781. Signed-off-by: politerealism <[email protected]>
1 parent a9a635a commit 729061e

13 files changed

Lines changed: 421 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-core/src/driver_utils.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,18 @@ pub const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = "/etc/openshell/auth/upstream-p
134134
/// mount is used rather than a driver secret.
135135
pub const PROXY_CA_MOUNT_PATH: &str = "/etc/openshell/tls/proxy/ca-bundle.pem";
136136

137+
/// Container-side mount path for an operator-supplied additional destination
138+
/// CA bundle.
139+
///
140+
/// Drivers with an `additional_ca_bundle` operator setting bind-mount the
141+
/// host PEM file here (read-only) and pass the path on the supervisor's argv
142+
/// via `--additional-ca-bundle`. The supervisor folds it into its upstream
143+
/// TLS trust store alongside the system CA bundle, for direct (non-proxied)
144+
/// inspected HTTPS egress to destinations signed by a private CA — unlike
145+
/// [`PROXY_CA_MOUNT_PATH`], this applies regardless of whether a corporate
146+
/// forward proxy is configured.
147+
pub const ADDITIONAL_CA_MOUNT_PATH: &str = "/etc/openshell/tls/additional/ca-bundle.pem";
148+
137149
/// A validated corporate upstream-proxy address.
138150
///
139151
/// Produced by [`parse_upstream_proxy_url`], which is the single source of

‎crates/openshell-driver-podman/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ url = { workspace = true }
5050
openshell-otel-test-support = { path = "../openshell-otel-test-support" }
5151
opentelemetry_sdk = { workspace = true, features = ["testing"] }
5252
prost-types = { workspace = true }
53+
rcgen = { workspace = true }
5354
temp-env = "0.3"
5455
tempfile = "3"
5556
tokio = { workspace = true, features = ["test-util"] }

‎crates/openshell-driver-podman/src/config.rs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,23 @@ pub struct PodmanComputeConfig {
178178
/// and upstream verification. Only meaningful with `https_proxy` set; the
179179
/// bundle must exist and contain at least one certificate.
180180
pub proxy_ca_bundle: Option<String>,
181+
/// Path (on the gateway host) to a PEM CA bundle trusted for direct
182+
/// (non-proxied), policy-inspected HTTPS egress to destinations signed by
183+
/// a private CA.
184+
///
185+
/// Unlike [`proxy_ca_bundle`](Self::proxy_ca_bundle), this applies
186+
/// regardless of whether `https_proxy` is configured: it addresses
187+
/// supervisors that run as their own container, separate from the
188+
/// workload image, and therefore cannot inherit a private CA baked into
189+
/// the workload's system trust store. The gateway bind-mounts this file
190+
/// read-only into the supervisor container (at
191+
/// [`ADDITIONAL_CA_MOUNT_PATH`](openshell_core::driver_utils::ADDITIONAL_CA_MOUNT_PATH))
192+
/// and passes its path via `--additional-ca-bundle`. The supervisor folds
193+
/// it into its upstream TLS trust store alongside (not instead of) the
194+
/// system CA bundle. The bundle must exist and contain at least one
195+
/// certificate; this is validated on the gateway host at sandbox-create
196+
/// time, not deferred until the supervisor container starts.
197+
pub additional_ca_bundle: Option<String>,
181198
/// User namespace mode for sandbox containers (e.g. `auto`, `private`).
182199
/// When unset, containers use the default user namespace.
183200
pub userns: Option<String>,
@@ -332,6 +349,21 @@ impl PodmanComputeConfig {
332349
));
333350
}
334351
}
352+
// additional_ca_bundle is independent of the corporate proxy setting
353+
// above: it applies to direct, inspected egress and has no
354+
// https_proxy dependency. Read and validate the actual host file
355+
// here (this path is not yet mounted into any container), rather
356+
// than only checking it is non-empty: an operator with a mistyped
357+
// path or a certificate-free bundle should see a config-validation
358+
// error at sandbox-create time, not a failed/crash-looping sandbox
359+
// once the supervisor container starts and tries to read it.
360+
if let Some(path) = self.additional_ca_bundle.as_deref() {
361+
openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(
362+
path,
363+
"additional_ca_bundle",
364+
)
365+
.map_err(crate::client::PodmanApiError::InvalidInput)?;
366+
}
335367
Ok(())
336368
}
337369

@@ -519,6 +551,7 @@ impl Default for PodmanComputeConfig {
519551
proxy_auth_allow_insecure: None,
520552
proxy_connect_by_hostname: None,
521553
proxy_ca_bundle: None,
554+
additional_ca_bundle: None,
522555
userns: None,
523556
uidmap: Vec::new(),
524557
gidmap: Vec::new(),
@@ -832,6 +865,65 @@ mod tests {
832865
assert!(err.to_string().contains("proxy_ca_bundle"), "{err}");
833866
}
834867

868+
#[test]
869+
fn validate_configuration_accepts_additional_ca_bundle_without_proxy() {
870+
// #3781: unlike proxy_ca_bundle, this must work with no corporate
871+
// proxy configured at all.
872+
let directory = tempfile::tempdir().unwrap();
873+
let ca_path = directory.path().join("additional-ca.pem");
874+
std::fs::write(&ca_path, generate_test_ca_pem()).unwrap();
875+
876+
let cfg = PodmanComputeConfig {
877+
additional_ca_bundle: Some(ca_path.to_str().unwrap().to_string()),
878+
..PodmanComputeConfig::default()
879+
};
880+
assert!(cfg.validate_proxy_config().is_ok());
881+
assert!(cfg.https_proxy.is_none());
882+
}
883+
884+
#[test]
885+
fn validate_configuration_rejects_empty_additional_ca_bundle() {
886+
let cfg = PodmanComputeConfig {
887+
additional_ca_bundle: Some(" ".to_string()),
888+
..PodmanComputeConfig::default()
889+
};
890+
let err = cfg.validate_proxy_config().unwrap_err();
891+
assert!(err.to_string().contains("additional_ca_bundle"), "{err}");
892+
}
893+
894+
#[test]
895+
fn validate_configuration_rejects_nonexistent_additional_ca_bundle() {
896+
// #3781 fix requirement: an operator with a mistyped path must see
897+
// this at config-validation time, not a failed sandbox later.
898+
let cfg = PodmanComputeConfig {
899+
additional_ca_bundle: Some("/nonexistent/additional-ca.pem".to_string()),
900+
..PodmanComputeConfig::default()
901+
};
902+
let err = cfg.validate_proxy_config().unwrap_err();
903+
assert!(err.to_string().contains("additional_ca_bundle"), "{err}");
904+
}
905+
906+
#[test]
907+
fn validate_configuration_rejects_certificate_free_additional_ca_bundle() {
908+
let directory = tempfile::tempdir().unwrap();
909+
let ca_path = directory.path().join("not-a-cert.pem");
910+
std::fs::write(&ca_path, "not a certificate\n").unwrap();
911+
912+
let cfg = PodmanComputeConfig {
913+
additional_ca_bundle: Some(ca_path.to_str().unwrap().to_string()),
914+
..PodmanComputeConfig::default()
915+
};
916+
let err = cfg.validate_proxy_config().unwrap_err();
917+
assert!(err.to_string().contains("additional_ca_bundle"), "{err}");
918+
}
919+
920+
/// Generate a real, valid self-signed CA certificate PEM for tests.
921+
fn generate_test_ca_pem() -> String {
922+
let key_pair = rcgen::KeyPair::generate().unwrap();
923+
let params = rcgen::CertificateParams::new(Vec::new()).unwrap();
924+
params.self_signed(&key_pair).unwrap().pem()
925+
}
926+
835927
#[test]
836928
fn validate_proxy_config_rejects_url_components() {
837929
for url in [

‎crates/openshell-driver-podman/src/container.rs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str =
6969
openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH;
7070
const RESOLV_CONF_PATH: &str = "/etc/resolv.conf";
7171
const PROXY_CA_MOUNT_PATH: &str = openshell_core::driver_utils::PROXY_CA_MOUNT_PATH;
72+
const ADDITIONAL_CA_MOUNT_PATH: &str = openshell_core::driver_utils::ADDITIONAL_CA_MOUNT_PATH;
7273
const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str =
7374
openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR;
7475

@@ -612,6 +613,19 @@ fn upstream_proxy_cli_args(config: &PodmanComputeConfig) -> Vec<String> {
612613
args
613614
}
614615

616+
/// Build the additional-destination-CA command-line argument passed to the
617+
/// supervisor. Independent of `upstream_proxy_cli_args`: this applies to
618+
/// direct, inspected egress regardless of whether a corporate proxy is
619+
/// configured.
620+
fn additional_ca_cli_args(config: &PodmanComputeConfig) -> Vec<String> {
621+
let mut args = Vec::new();
622+
if config.additional_ca_bundle.is_some() {
623+
args.push("--additional-ca-bundle".to_string());
624+
args.push(ADDITIONAL_CA_MOUNT_PATH.to_string());
625+
}
626+
args
627+
}
628+
615629
fn build_env(
616630
sandbox: &DriverSandbox,
617631
config: &PodmanComputeConfig,
@@ -1272,6 +1286,7 @@ fn build_base_spec(
12721286
image_volumes.extend(user_mounts.image_volumes);
12731287
let mut command = vec!["--workdir".to_string(), image.workspace_root.clone()];
12741288
command.extend(upstream_proxy_cli_args(config));
1289+
command.extend(additional_ca_cli_args(config));
12751290

12761291
let container_spec = ContainerSpec {
12771292
name,
@@ -1438,6 +1453,24 @@ fn build_base_spec(
14381453
options: ro,
14391454
});
14401455
}
1456+
// Bind-mount an operator-supplied additional destination CA
1457+
// bundle read-only when configured. Independent of the corporate
1458+
// proxy CA above: this applies to direct, inspected egress. The
1459+
// supervisor reads it via the --additional-ca-bundle argv path
1460+
// (see additional_ca_cli_args) and folds it into its upstream
1461+
// trust store alongside the system bundle.
1462+
if let Some(ca_bundle) = &config.additional_ca_bundle {
1463+
let mut ro = vec!["ro".into(), "rbind".into()];
1464+
if is_selinux_enabled() {
1465+
ro.push("z".into());
1466+
}
1467+
m.push(Mount {
1468+
kind: "bind".into(),
1469+
source: ca_bundle.clone(),
1470+
destination: ADDITIONAL_CA_MOUNT_PATH.into(),
1471+
options: ro,
1472+
});
1473+
}
14411474
if let Some(bin_path) = supervisor_bin_path {
14421475
let mut opts = vec!["ro".into(), "rbind".into()];
14431476
if is_selinux_enabled() {
@@ -2870,6 +2903,70 @@ mod tests {
28702903
);
28712904
}
28722905

2906+
#[test]
2907+
fn container_spec_binds_additional_ca_bundle_and_passes_argv_without_proxy() {
2908+
// #3781: the additional CA must work for direct, inspected egress
2909+
// with no corporate proxy configured at all — unlike proxy_ca_bundle,
2910+
// which requires https_proxy.
2911+
let sandbox = test_sandbox("additional-ca-id", "additional-ca-name");
2912+
let mut config = test_config();
2913+
assert!(
2914+
config.https_proxy.is_none(),
2915+
"test must exercise the no-proxy case"
2916+
);
2917+
config.additional_ca_bundle = Some("/host/additional-ca.pem".to_string());
2918+
2919+
let spec = build_container_spec(&sandbox, &config);
2920+
let command = spec_command(&spec);
2921+
2922+
let idx = command
2923+
.iter()
2924+
.position(|a| a == "--additional-ca-bundle")
2925+
.expect("additional CA bundle flag present");
2926+
assert_eq!(
2927+
command.get(idx + 1).map(String::as_str),
2928+
Some("/etc/openshell/tls/additional/ca-bundle.pem")
2929+
);
2930+
2931+
let mounts = spec["mounts"]
2932+
.as_array()
2933+
.expect("mounts should be an array");
2934+
let ca_mount = mounts
2935+
.iter()
2936+
.find(|m| {
2937+
m["type"].as_str() == Some("bind")
2938+
&& m["destination"].as_str()
2939+
== Some("/etc/openshell/tls/additional/ca-bundle.pem")
2940+
})
2941+
.expect("additional CA bundle bind mount present");
2942+
assert_eq!(ca_mount["source"].as_str(), Some("/host/additional-ca.pem"));
2943+
assert!(
2944+
ca_mount["options"]
2945+
.as_array()
2946+
.expect("options array")
2947+
.iter()
2948+
.any(|o| o.as_str() == Some("ro")),
2949+
"additional CA bundle mount must be read-only"
2950+
);
2951+
}
2952+
2953+
#[test]
2954+
fn container_spec_omits_additional_ca_bundle_when_unconfigured() {
2955+
let sandbox = test_sandbox("test-id", "test-name");
2956+
let spec = build_container_spec(&sandbox, &test_config());
2957+
let command = spec_command(&spec);
2958+
assert!(
2959+
!command.iter().any(|a| a == "--additional-ca-bundle"),
2960+
"no additional-ca-bundle flag without operator config"
2961+
);
2962+
let mounts = spec["mounts"].as_array().expect("mounts array");
2963+
assert!(
2964+
!mounts.iter().any(|m| m["destination"].as_str()
2965+
== Some("/etc/openshell/tls/additional/ca-bundle.pem")),
2966+
"no additional CA bundle mount without operator config"
2967+
);
2968+
}
2969+
28732970
#[test]
28742971
fn container_spec_sandbox_env_cannot_influence_proxy_argv() {
28752972
use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate};

‎crates/openshell-driver-podman/src/main.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,14 @@ struct Args {
174174
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CA_BUNDLE")]
175175
sandbox_proxy_ca_bundle: Option<String>,
176176

177+
/// Path (on the gateway host) to a PEM CA bundle trusted for direct
178+
/// (non-proxied), policy-inspected HTTPS egress to destinations signed by
179+
/// a private CA. Unlike `--sandbox-proxy-ca-bundle`, this applies
180+
/// regardless of whether a corporate proxy is configured. Bind-mounted
181+
/// read-only into the supervisor container.
182+
#[arg(long, env = "OPENSHELL_SANDBOX_ADDITIONAL_CA_BUNDLE")]
183+
sandbox_additional_ca_bundle: Option<String>,
184+
177185
/// User namespace mode for sandbox containers (e.g. `auto`).
178186
/// When unset, containers use the default user namespace.
179187
#[arg(long, env = "OPENSHELL_PODMAN_USERNS")]
@@ -241,6 +249,7 @@ async fn main() -> Result<()> {
241249
proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure,
242250
proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname,
243251
proxy_ca_bundle: args.sandbox_proxy_ca_bundle,
252+
additional_ca_bundle: args.sandbox_additional_ca_bundle,
244253
userns: args.userns,
245254
uidmap: args.uidmap,
246255
gidmap: args.gidmap,

‎crates/openshell-supervisor-network/src/l7/tls.rs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,11 +585,80 @@ mod tests {
585585
assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()]);
586586
}
587587

588+
/// Reproduces issue #3781 end to end with a real TLS handshake: an
589+
/// upstream client trying to reach a destination signed by an
590+
/// operator-owned private CA fails when that CA is unknown, and succeeds
591+
/// once it is appended to the bundle passed to
592+
/// `build_upstream_client_config` — the same mechanism the Podman driver
593+
/// fix (`additional_ca_bundle`) feeds into. No fakes: a real rustls
594+
/// `TlsAcceptor`/`TlsConnector` pair over a real loopback TCP connection.
595+
#[tokio::test]
596+
async fn additional_ca_enables_trust_of_privately_signed_upstream() {
597+
let hostname = "private.example";
598+
599+
// A CA the system bundle has never heard of, standing in for the
600+
// operator-owned private CA from #3781.
601+
let private_ca = SandboxCa::generate().unwrap();
602+
let private_ca_pem = private_ca.cert_pem().to_string();
603+
let cert_cache = CertCache::new(private_ca);
604+
let leaf = cert_cache.get_or_generate(hostname).unwrap();
605+
606+
let server_config = Arc::new(
607+
ServerConfig::builder()
608+
.with_no_client_auth()
609+
.with_single_cert(leaf.cert_chain.clone(), leaf.private_key.clone_key())
610+
.unwrap(),
611+
);
612+
let acceptor = TlsAcceptor::from(server_config);
613+
614+
// --- Without the private CA: must fail. This is the literal bug in
615+
// #3781 — "Upstream TLS establishment failed" against a
616+
// policy-allowed, private-CA-signed destination. ---
617+
let addr = serve_one(acceptor.clone()).await;
618+
let client_without = build_upstream_client_config("").unwrap();
619+
let tcp = tokio::net::TcpStream::connect(addr).await.unwrap();
620+
let result = tls_connect_upstream(tcp, hostname, &client_without).await;
621+
assert!(
622+
result.is_err(),
623+
"connection to a privately-signed destination must fail when the \
624+
private CA is not trusted, reproducing #3781"
625+
);
626+
627+
// --- With the private CA appended to the bundle: must succeed. This
628+
// is what additional_ca_bundle plumbs through in the Podman driver
629+
// fix. ---
630+
let addr = serve_one(acceptor).await;
631+
let client_with = build_upstream_client_config(&private_ca_pem).unwrap();
632+
let tcp = tokio::net::TcpStream::connect(addr).await.unwrap();
633+
let result = tls_connect_upstream(tcp, hostname, &client_with).await;
634+
assert!(
635+
result.is_ok(),
636+
"connection must succeed once the private CA is trusted: {:?}",
637+
result.err()
638+
);
639+
}
640+
588641
/// Helper: generate a self-signed CA and return its PEM string.
589642
fn generate_ca_pem() -> String {
590643
SandboxCa::generate().unwrap().ca_cert_pem
591644
}
592645

646+
/// Accept exactly one TLS connection on a fresh loopback listener and
647+
/// return its address. Used by `additional_ca_enables_trust_of_...` to
648+
/// give each connection attempt its own listener.
649+
async fn serve_one(acceptor: TlsAcceptor) -> std::net::SocketAddr {
650+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
651+
let addr = listener.local_addr().unwrap();
652+
tokio::spawn(async move {
653+
let (stream, _) = listener.accept().await.unwrap();
654+
// Don't unwrap: when the client (correctly) rejects an untrusted
655+
// certificate, the server observes a failed handshake too. Only
656+
// the client's outcome matters here.
657+
let _ = acceptor.accept(stream).await;
658+
});
659+
addr
660+
}
661+
593662
#[test]
594663
fn load_pem_certs_single_ca() {
595664
let pem = generate_ca_pem();

0 commit comments

Comments
 (0)