Skip to content

Commit d9083da

Browse files
committed
feat(supervisor-network): serve reserved destinations inside the proxy
Add a `ReservedStreamHandler` hook to the transparent-open path so a staged workload connection to a reserved, unroutable address is served by the supervisor itself instead of being policy-evaluated and dialed upstream. The branch sits at the top of `preauthorize_transparent_open`, ahead of host mapping, OPA, SSRF checks, and destination validation, because the address is a label the supervisor switches on rather than a place anything could reach. The handler is consulted before `RelayReady` is sent, so an exhausted handler refuses the open with `ResourceExhausted` and the sandbox never commits the workload socket. A connect to the OTLP relay address with no handler installed is denied outright rather than falling through. `ReservedDestination` is threaded through `ProxyHandle::start_with_bind_addr` and `run_networking`; both supervisor call sites pass `None` until the relay is wired in the next commit. No hyper or OTLP dependency enters this crate. Unit tests cover the four cases: served without policy (bytes from the workload side reach the handler), refused at capacity, denied without a handler, and other destinations unaffected. Refs NVIDIA#2641 Signed-off-by: Roland Huß <[email protected]>
1 parent 06cd5c4 commit d9083da

4 files changed

Lines changed: 295 additions & 4 deletions

File tree

‎crates/openshell-supervisor-network/src/host.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ pub async fn start_host_proxy(config: HostProxyConfig) -> Result<HostProxyHandle
220220
None,
221221
None,
222222
Some(direct_listener_identity),
223+
None,
223224
)
224225
.await?;
225226

‎crates/openshell-supervisor-network/src/proxy.rs‎

Lines changed: 290 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,14 @@ use openshell_ocsf::{
3838
ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, HttpResponse,
3939
NetworkActivityBuilder, Process, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit,
4040
};
41+
use std::future::Future;
4142
#[cfg(target_os = "linux")]
4243
use std::mem::size_of;
4344
use std::net::{IpAddr, SocketAddr};
4445
use std::path::PathBuf;
45-
use std::sync::Arc;
46+
use std::pin::Pin;
4647
use std::sync::atomic::{AtomicU32, Ordering};
48+
use std::sync::{Arc, LazyLock};
4749
use tokio::io::{
4850
AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite,
4951
AsyncWriteExt,
@@ -73,6 +75,41 @@ enum ProxyAcceptError {
7375
Source(openshell_isolation_interface::contract::BackendError),
7476
}
7577

78+
/// A future that serves one reserved-destination stream to completion.
79+
pub type ReservedStreamFuture = Pin<Box<dyn Future<Output = ()> + Send + 'static>>;
80+
81+
/// Serves workload connections to a reserved, unroutable destination inside
82+
/// the supervisor instead of dialing upstream.
83+
///
84+
/// The proxy consults the handler before the sandbox commits the workload
85+
/// socket. `None` refuses the open, which the workload observes as `EAGAIN`;
86+
/// `Some` accepts it, after which the proxy answers `RelayReady` and drives
87+
/// the returned future on its own task. Nothing may be read from `stream`
88+
/// until that future is polled.
89+
pub trait ReservedStreamHandler: Send + Sync {
90+
fn serve(&self, stream: BoundaryDuplexStream) -> Option<ReservedStreamFuture>;
91+
}
92+
93+
/// A reserved destination the proxy serves locally.
94+
#[derive(Clone)]
95+
pub struct ReservedDestination {
96+
pub addr: SocketAddr,
97+
pub handler: Arc<dyn ReservedStreamHandler>,
98+
}
99+
100+
/// The OTLP relay address, known to the proxy even when no handler is
101+
/// installed so a workload connect to it never falls through to policy
102+
/// evaluation and upstream dialing.
103+
static OTLP_RELAY_ADDR: LazyLock<SocketAddr> = LazyLock::new(|| {
104+
openshell_core::sandbox_env::OTLP_RELAY_ADDR
105+
.parse()
106+
.expect("OTLP_RELAY_ADDR is a valid socket address")
107+
});
108+
109+
fn is_reserved_relay_destination(destination: SocketAddr) -> bool {
110+
destination == *OTLP_RELAY_ADDR
111+
}
112+
76113
use self::destination::{
77114
DestinationDenial, DestinationDenialKind, DestinationRequest, build_pinned_validation_plan,
78115
build_validation_plan, validate_destination,
@@ -222,6 +259,7 @@ impl ProxyHandle {
222259
network_mediation_source: Option<Arc<dyn NetworkMediationSource>>,
223260
policy_dns_store: Option<Arc<ResolvedEndpointStore>>,
224261
direct_listener_identity: Option<ContractBinaryIdentity>,
262+
reserved_destination: Option<ReservedDestination>,
225263
) -> Result<Self> {
226264
// Use override bind_addr, fall back to policy http_addr, then default
227265
// to loopback:3128. The default allows the proxy to function when no
@@ -383,13 +421,15 @@ impl ProxyHandle {
383421
let opa = opa_engine.clone();
384422
let backend_gateway = *backend_host_gateway;
385423
let trusted_gateway = *trusted_host_gateway;
424+
let reserved = reserved_destination.clone();
386425
tokio::spawn(async move {
387426
if let Some(connection) = preauthorize_transparent_open(
388427
connection,
389428
dns_store.as_ref(),
390429
&opa,
391430
backend_gateway,
392431
trusted_gateway,
432+
reserved.as_ref(),
393433
)
394434
.await
395435
{
@@ -558,6 +598,7 @@ async fn preauthorize_transparent_open(
558598
opa_engine: &OpaEngine,
559599
backend_host_gateway: Option<IpAddr>,
560600
trusted_host_gateway: Option<IpAddr>,
601+
reserved: Option<&ReservedDestination>,
561602
) -> Option<AcceptedProxyConnection> {
562603
let PendingTcpOpen {
563604
stream,
@@ -572,6 +613,40 @@ async fn preauthorize_transparent_open(
572613
timing,
573614
operation: "tcp",
574615
};
616+
// A reserved destination is served inside the supervisor. It is decided
617+
// before host mapping, policy, and SSRF validation because the address is
618+
// an unroutable label, not a place anything could dial.
619+
if let Some(reserved) = reserved.filter(|reserved| reserved.addr == destination) {
620+
if let Some(served) = reserved.handler.serve(stream) {
621+
debug!(%destination, "Serving staged connection on reserved destination");
622+
// If the sandbox side is already gone, dropping `served` closes
623+
// the stream and releases the handler's slot.
624+
if completion.send(TcpOpenDecision::RelayReady).is_ok() {
625+
tokio::spawn(served);
626+
}
627+
} else {
628+
warn!(
629+
%destination,
630+
"Refused staged reserved-destination connection: handler at capacity"
631+
);
632+
let _ = completion.send(TcpOpenDecision::Denied(TcpOpenDenial::ResourceExhausted));
633+
}
634+
return None;
635+
}
636+
if is_reserved_relay_destination(destination) {
637+
warn!(
638+
%destination,
639+
"Denied staged reserved-destination connection: no handler installed"
640+
);
641+
emit_staged_transparent_denial(
642+
destination,
643+
&binary_identity,
644+
"reserved destination has no handler",
645+
"transparent_tcp_reserved_denied",
646+
);
647+
let _ = completion.send(TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied));
648+
return None;
649+
}
575650
let host = match transparent_destination_host(destination, policy_dns_store, opa_engine) {
576651
Ok(host) => host,
577652
Err(error) => {
@@ -6553,15 +6628,15 @@ process:
65536628

65546629
let (allowed, allowed_result) = pending("203.0.113.7:443");
65556630
assert!(
6556-
preauthorize_transparent_open(allowed, None, &engine, None, None)
6631+
preauthorize_transparent_open(allowed, None, &engine, None, None, None)
65576632
.await
65586633
.is_some()
65596634
);
65606635
assert_eq!(allowed_result.await.unwrap(), TcpOpenDecision::RelayReady);
65616636

65626637
let (unsafe_destination, unsafe_result) = pending("169.254.169.254:80");
65636638
assert!(
6564-
preauthorize_transparent_open(unsafe_destination, None, &engine, None, None)
6639+
preauthorize_transparent_open(unsafe_destination, None, &engine, None, None, None)
65656640
.await
65666641
.is_none()
65676642
);
@@ -6572,7 +6647,7 @@ process:
65726647

65736648
let (denied, denied_result) = pending("203.0.113.8:443");
65746649
assert!(
6575-
preauthorize_transparent_open(denied, None, &engine, None, None)
6650+
preauthorize_transparent_open(denied, None, &engine, None, None, None)
65766651
.await
65776652
.is_none()
65786653
);
@@ -6582,6 +6657,216 @@ process:
65826657
);
65836658
}
65846659

6660+
// ---- reserved destinations ----
6661+
6662+
/// Policy that allows one public endpoint and nothing else. It never
6663+
/// mentions the relay address, so a served relay stream proves the
6664+
/// reserved path bypassed policy.
6665+
fn reserved_test_engine() -> OpaEngine {
6666+
OpaEngine::from_strings(
6667+
include_str!("../data/sandbox-policy.rego"),
6668+
r#"
6669+
network_policies:
6670+
allowed:
6671+
name: allowed
6672+
endpoints:
6673+
- host: 203.0.113.7
6674+
port: 443
6675+
binaries:
6676+
- path: /usr/bin/curl
6677+
filesystem_policy:
6678+
include_workdir: true
6679+
read_only: []
6680+
read_write: []
6681+
landlock:
6682+
compatibility: best_effort
6683+
process:
6684+
run_as_user: sandbox
6685+
run_as_group: sandbox
6686+
"#,
6687+
)
6688+
.unwrap()
6689+
}
6690+
6691+
/// A staged open from `/usr/bin/curl` to `destination`, plus the
6692+
/// decision receiver and the workload-side peer of the staged stream.
6693+
fn staged_open(
6694+
engine: &OpaEngine,
6695+
destination: &str,
6696+
) -> (
6697+
PendingTcpOpen,
6698+
tokio::sync::oneshot::Receiver<TcpOpenDecision>,
6699+
tokio::io::DuplexStream,
6700+
) {
6701+
let (stream, peer) = tokio::io::duplex(256);
6702+
let (decision, completion) = tokio::sync::oneshot::channel();
6703+
let open = PendingTcpOpen {
6704+
stream: Box::new(stream),
6705+
binary_identity: Ok(ContractBinaryIdentity {
6706+
binary_path: PathBuf::from("/usr/bin/curl"),
6707+
binary_digest: Some("00".repeat(32).parse().unwrap()),
6708+
ancestors: Vec::new(),
6709+
cmdline_paths: Vec::new(),
6710+
}),
6711+
destination: destination.parse().unwrap(),
6712+
socket: openshell_isolation_interface::contract::NetworkSocketMetadata {
6713+
socket_cookie: 7,
6714+
nonblocking: false,
6715+
process_generation: 1,
6716+
},
6717+
policy_generation: engine.current_generation(),
6718+
timing: MediationTiming::default(),
6719+
decision,
6720+
};
6721+
(open, completion, peer)
6722+
}
6723+
6724+
/// Handler double: counts calls, optionally refuses, and hands every
6725+
/// accepted stream back to the test.
6726+
struct RecordingReservedHandler {
6727+
accept: bool,
6728+
served: mpsc::UnboundedSender<BoundaryDuplexStream>,
6729+
calls: Arc<std::sync::atomic::AtomicUsize>,
6730+
}
6731+
6732+
impl ReservedStreamHandler for RecordingReservedHandler {
6733+
fn serve(&self, stream: BoundaryDuplexStream) -> Option<ReservedStreamFuture> {
6734+
self.calls.fetch_add(1, Ordering::SeqCst);
6735+
if !self.accept {
6736+
return None;
6737+
}
6738+
let served = self.served.clone();
6739+
Some(Box::pin(async move {
6740+
let _ = served.send(stream);
6741+
}))
6742+
}
6743+
}
6744+
6745+
fn reserved_relay(
6746+
accept: bool,
6747+
) -> (
6748+
ReservedDestination,
6749+
mpsc::UnboundedReceiver<BoundaryDuplexStream>,
6750+
Arc<std::sync::atomic::AtomicUsize>,
6751+
) {
6752+
let (served, served_rx) = mpsc::unbounded_channel();
6753+
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
6754+
let handler = RecordingReservedHandler {
6755+
accept,
6756+
served,
6757+
calls: Arc::clone(&calls),
6758+
};
6759+
let reserved = ReservedDestination {
6760+
addr: *OTLP_RELAY_ADDR,
6761+
handler: Arc::new(handler),
6762+
};
6763+
(reserved, served_rx, calls)
6764+
}
6765+
6766+
#[tokio::test]
6767+
async fn reserved_destination_is_served_without_policy_evaluation() {
6768+
let engine = reserved_test_engine();
6769+
let (reserved, mut served_rx, calls) = reserved_relay(true);
6770+
let (open, decision, mut peer) =
6771+
staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR);
6772+
6773+
assert!(
6774+
preauthorize_transparent_open(open, None, &engine, None, None, Some(&reserved))
6775+
.await
6776+
.is_none(),
6777+
"a reserved open never becomes a proxy connection"
6778+
);
6779+
assert_eq!(decision.await.unwrap(), TcpOpenDecision::RelayReady);
6780+
assert_eq!(calls.load(Ordering::SeqCst), 1);
6781+
6782+
// The handler holds the workload's stream: bytes written by the
6783+
// workload side arrive at the handler.
6784+
let mut stream = tokio::time::timeout(std::time::Duration::from_secs(1), served_rx.recv())
6785+
.await
6786+
.expect("handler future ran")
6787+
.expect("handler received the stream");
6788+
peer.write_all(b"POST /v1/traces").await.unwrap();
6789+
let mut head = [0u8; 15];
6790+
stream.read_exact(&mut head).await.unwrap();
6791+
assert_eq!(&head, b"POST /v1/traces");
6792+
}
6793+
6794+
#[tokio::test]
6795+
async fn reserved_destination_is_refused_when_handler_is_at_capacity() {
6796+
let engine = reserved_test_engine();
6797+
let (reserved, _served_rx, calls) = reserved_relay(false);
6798+
let (open, decision, _peer) =
6799+
staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR);
6800+
6801+
assert!(
6802+
preauthorize_transparent_open(open, None, &engine, None, None, Some(&reserved))
6803+
.await
6804+
.is_none()
6805+
);
6806+
assert_eq!(
6807+
decision.await.unwrap(),
6808+
TcpOpenDecision::Denied(TcpOpenDenial::ResourceExhausted),
6809+
"refusal happens before the sandbox commits the socket"
6810+
);
6811+
assert_eq!(calls.load(Ordering::SeqCst), 1);
6812+
}
6813+
6814+
#[tokio::test]
6815+
async fn reserved_relay_destination_is_denied_without_a_handler() {
6816+
let engine = reserved_test_engine();
6817+
let (open, decision, _peer) =
6818+
staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR);
6819+
6820+
assert!(
6821+
preauthorize_transparent_open(open, None, &engine, None, None, None)
6822+
.await
6823+
.is_none()
6824+
);
6825+
assert_eq!(
6826+
decision.await.unwrap(),
6827+
TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied),
6828+
"the relay address must never reach policy evaluation or dialing"
6829+
);
6830+
}
6831+
6832+
#[tokio::test]
6833+
async fn reserved_handler_ignores_other_destinations() {
6834+
let engine = reserved_test_engine();
6835+
let (reserved, _served_rx, calls) = reserved_relay(true);
6836+
6837+
let (denied, denied_result) = {
6838+
let (open, decision, _peer) = staged_open(&engine, "203.0.113.8:443");
6839+
(open, decision)
6840+
};
6841+
assert!(
6842+
preauthorize_transparent_open(denied, None, &engine, None, None, Some(&reserved))
6843+
.await
6844+
.is_none()
6845+
);
6846+
assert_eq!(
6847+
denied_result.await.unwrap(),
6848+
TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied)
6849+
);
6850+
6851+
let (allowed, allowed_result) = {
6852+
let (open, decision, _peer) = staged_open(&engine, "203.0.113.7:443");
6853+
(open, decision)
6854+
};
6855+
assert!(
6856+
preauthorize_transparent_open(allowed, None, &engine, None, None, Some(&reserved))
6857+
.await
6858+
.is_some(),
6859+
"policy-allowed destinations still flow to the proxy"
6860+
);
6861+
assert_eq!(allowed_result.await.unwrap(), TcpOpenDecision::RelayReady);
6862+
6863+
assert_eq!(
6864+
calls.load(Ordering::SeqCst),
6865+
0,
6866+
"the handler is consulted only for its own address"
6867+
);
6868+
}
6869+
65856870
struct FailedMediationSource;
65866871

65876872
#[tokio::test]
@@ -6853,6 +7138,7 @@ network_policies: {}
68537138
Some(Arc::new(FailedMediationSource)),
68547139
None,
68557140
None,
7141+
None,
68567142
)
68577143
.await
68587144
.expect("proxy starts before source accept");

0 commit comments

Comments
 (0)