Skip to content

Commit e12c36a

Browse files
committed
feat(supervisor): OTLP relay served on a reserved mediated destination
Relocate the supervisor-side OTLP relay onto the RFC 0012 split boundary. The supervisor no longer shares a network namespace with the agent, so the receiver cannot bind a loopback port the agent can reach. Instead the agent exports to a reserved, unroutable destination (192.0.0.8:4318, the RFC 7600 dummy address). The sandbox seccomp broker stages every non-loopback connect for the supervisor, whose proxy will hand the staged stream to the relay rather than dialing upstream. No boundary-protocol or isolation-contract change is required. `openshell-supervisor-process::otlp` keeps the PR's bounded buffer, span enrichment, and OTLP HTTP parsing unchanged. The accept loop becomes `OtlpConnectionServer::serve`, which runs one hyper connection on a stream the caller supplies; `try_reserve` lets the proxy refuse an open before the sandbox commits the workload socket. Shutdown keeps the previous semantics (refuse new streams, disable keep-alive, bounded grace, cut stragglers) with hyper-util's graceful watcher plus an abort signal instead of a task set. `RelayLifecycle` drops the lazy `Pending` state: the relay starts before networking, so there is no window where the agent sees a refused connection. Receiver tests run over `tokio::io::duplex`, standing in for the boundary stream. A constant test pins the reserved address as non-loopback and outside the policy DNS synthetic pool. The proxy hook and session wiring follow in separate commits. Refs NVIDIA#2641 Signed-off-by: Roland Huß <[email protected]>
1 parent 43a25db commit e12c36a

8 files changed

Lines changed: 1471 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-core/src/sandbox_env.rs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,11 +266,67 @@ pub const DEFAULT_SANDBOX_GID: u32 = 1000;
266266
/// OCI only for the former contract.
267267
pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
268268

269+
/// Standard OpenTelemetry environment variable for the OTLP exporter endpoint.
270+
///
271+
/// The gateway sets it in the sandbox environment when
272+
/// `[openshell.gateway.otlp]` is configured, pointing agent SDKs at
273+
/// [`OTLP_RELAY_ENDPOINT`]. A value the user declared explicitly wins.
274+
pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
275+
276+
/// Standard OpenTelemetry environment variable for the OTLP exporter protocol.
277+
///
278+
/// Set to `http/protobuf` alongside [`OTEL_EXPORTER_OTLP_ENDPOINT`].
279+
pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
280+
281+
/// Reserved destination agent processes export OTLP to.
282+
///
283+
/// This address is never routed. A workload `connect()` to any non-loopback
284+
/// address is intercepted by the sandbox seccomp broker and staged for the
285+
/// supervisor, which recognises this destination and serves the OTLP
286+
/// receiver on the staged stream itself instead of dialing upstream. So the
287+
/// value is a label the supervisor switches on, and it must stay outside
288+
/// loopback (loopback connects complete locally without mediation) and
289+
/// outside `198.18.0.0/15`, which the policy DNS runtime uses for synthetic
290+
/// answers. `192.0.0.8` is the IPv4 dummy address (RFC 7600): reserved,
291+
/// unroutable, and never assigned to a real service.
292+
pub const OTLP_RELAY_ADDR: &str = "192.0.0.8:4318";
293+
294+
/// [`OTLP_RELAY_ADDR`] as the URL injected through
295+
/// [`OTEL_EXPORTER_OTLP_ENDPOINT`].
296+
pub const OTLP_RELAY_ENDPOINT: &str = "http://192.0.0.8:4318";
297+
269298
// The corporate upstream-proxy configuration deliberately has no reserved
270299
// environment variables: it travels on the supervisor's argv
271300
// (`--upstream-proxy` and friends), which a sandbox image cannot forge the
272301
// way it could bake `ENV` values.
273302

303+
#[cfg(test)]
304+
mod otlp_relay_address_tests {
305+
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
306+
307+
use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT};
308+
309+
#[test]
310+
fn relay_address_is_an_unroutable_non_loopback_label() {
311+
let addr: SocketAddr = OTLP_RELAY_ADDR.parse().expect("relay address parses");
312+
assert_eq!(addr.port(), 4318, "OTLP HTTP default port");
313+
let IpAddr::V4(ip) = addr.ip() else {
314+
panic!("relay address must be IPv4");
315+
};
316+
assert!(
317+
!ip.is_loopback(),
318+
"loopback would bypass seccomp mediation and never reach the supervisor"
319+
);
320+
assert_eq!(ip, Ipv4Addr::new(192, 0, 0, 8), "RFC 7600 dummy address");
321+
let [a, b, ..] = ip.octets();
322+
assert!(
323+
!(a == 198 && (b == 18 || b == 19)),
324+
"198.18.0.0/15 is the policy DNS synthetic answer pool"
325+
);
326+
assert_eq!(OTLP_RELAY_ENDPOINT, format!("http://{OTLP_RELAY_ADDR}"));
327+
}
328+
}
329+
274330
#[cfg(test)]
275331
mod tests {
276332
use super::*;

‎crates/openshell-supervisor-process/Cargo.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,18 @@ async-trait = "0.1"
2020
base64 = { workspace = true }
2121
bytes = { workspace = true }
2222
hex = "0.4"
23+
http-body-util = { workspace = true }
24+
hyper = { workspace = true }
25+
hyper-util = { workspace = true, features = ["tokio", "http1", "server-graceful"] }
2326
miette = { workspace = true }
2427
nix = { workspace = true }
28+
opentelemetry-proto = { workspace = true }
29+
prost = { workspace = true }
2530
rand = "0.10"
2631
russh = "0.63.1"
2732
serde_json = { workspace = true }
2833
sha2 = { workspace = true }
34+
thiserror = { workspace = true }
2935
tokio = { workspace = true }
3036
tokio-stream = { workspace = true }
3137
tonic = { workspace = true, features = ["channel", "tls-native-roots"] }

‎crates/openshell-supervisor-process/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ pub mod debug_rpc;
1111
pub mod delegated;
1212
pub mod log_push;
1313
pub mod main_session;
14+
pub mod otlp;
1415
pub mod skills;
1516
pub mod ssh;
1617
pub mod supervisor_session;
Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Bounded telemetry buffer with ring-buffer drop semantics.
5+
6+
use std::sync::Arc;
7+
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
8+
9+
use tokio::sync::mpsc;
10+
11+
/// Distinguishes trace data from OCSF events in the shared buffer.
12+
#[derive(Debug)]
13+
pub enum TelemetryItem {
14+
Trace(Vec<u8>),
15+
Ocsf(Vec<u8>),
16+
}
17+
18+
/// Shared drop/depth counters for the buffer.
19+
#[derive(Debug, Clone)]
20+
pub struct BufferMetrics {
21+
drop_count: Arc<AtomicU64>,
22+
queue_depth: Arc<AtomicUsize>,
23+
}
24+
25+
impl BufferMetrics {
26+
pub fn drops(&self) -> u64 {
27+
self.drop_count.load(Ordering::Relaxed)
28+
}
29+
30+
pub fn depth(&self) -> usize {
31+
self.queue_depth.load(Ordering::Relaxed)
32+
}
33+
}
34+
35+
/// Sender half of the telemetry buffer. Sends never block: when the buffer
36+
/// is full, the new item is dropped and the drop counter increments.
37+
#[derive(Clone)]
38+
pub struct TelemetrySender {
39+
tx: mpsc::Sender<TelemetryItem>,
40+
metrics: BufferMetrics,
41+
}
42+
43+
impl TelemetrySender {
44+
/// Send a telemetry item into the buffer. If the channel is full, the
45+
/// item is dropped and the drop counter is incremented.
46+
pub fn send(&self, item: TelemetryItem) {
47+
match self.tx.try_send(item) {
48+
Ok(()) => {
49+
self.metrics.queue_depth.fetch_add(1, Ordering::Relaxed);
50+
}
51+
Err(_) => {
52+
self.metrics.drop_count.fetch_add(1, Ordering::Relaxed);
53+
}
54+
}
55+
}
56+
57+
pub fn send_trace(&self, data: Vec<u8>) {
58+
self.send(TelemetryItem::Trace(data));
59+
}
60+
61+
pub fn send_ocsf(&self, data: Vec<u8>) {
62+
self.send(TelemetryItem::Ocsf(data));
63+
}
64+
65+
pub fn metrics(&self) -> &BufferMetrics {
66+
&self.metrics
67+
}
68+
}
69+
70+
/// Receiver half of the telemetry buffer.
71+
pub struct TelemetryReceiver {
72+
rx: mpsc::Receiver<TelemetryItem>,
73+
metrics: BufferMetrics,
74+
}
75+
76+
impl TelemetryReceiver {
77+
/// Receive the next buffered entry, or `None` if all senders are dropped.
78+
pub async fn recv(&mut self) -> Option<TelemetryItem> {
79+
let item = self.rx.recv().await;
80+
if item.is_some() {
81+
self.metrics.queue_depth.fetch_sub(1, Ordering::Relaxed);
82+
}
83+
item
84+
}
85+
86+
/// Drain all currently buffered entries without waiting.
87+
pub fn drain(&mut self) -> Vec<TelemetryItem> {
88+
let mut items = Vec::new();
89+
while let Ok(item) = self.rx.try_recv() {
90+
self.metrics.queue_depth.fetch_sub(1, Ordering::Relaxed);
91+
items.push(item);
92+
}
93+
items
94+
}
95+
96+
pub fn metrics(&self) -> &BufferMetrics {
97+
&self.metrics
98+
}
99+
}
100+
101+
/// Create a new telemetry buffer pair with the given capacity.
102+
pub fn new_telemetry_buffer(capacity: usize) -> (TelemetrySender, TelemetryReceiver) {
103+
let (tx, rx) = mpsc::channel(capacity);
104+
let metrics = BufferMetrics {
105+
drop_count: Arc::new(AtomicU64::new(0)),
106+
queue_depth: Arc::new(AtomicUsize::new(0)),
107+
};
108+
(
109+
TelemetrySender {
110+
tx,
111+
metrics: metrics.clone(),
112+
},
113+
TelemetryReceiver { rx, metrics },
114+
)
115+
}
116+
117+
#[cfg(test)]
118+
mod tests {
119+
use super::*;
120+
121+
#[tokio::test]
122+
async fn buffer_drops_when_full() {
123+
let (tx, mut rx) = new_telemetry_buffer(2);
124+
125+
tx.send_trace(vec![1]);
126+
tx.send_trace(vec![2]);
127+
tx.send_trace(vec![3]);
128+
129+
assert_eq!(tx.metrics().drops(), 1);
130+
assert_eq!(tx.metrics().depth(), 2);
131+
132+
let first = rx.recv().await.unwrap();
133+
assert!(matches!(first, TelemetryItem::Trace(v) if v == vec![1]));
134+
assert_eq!(rx.metrics().depth(), 1);
135+
}
136+
137+
#[tokio::test]
138+
async fn drain_empties_buffer() {
139+
let (tx, mut rx) = new_telemetry_buffer(16);
140+
141+
tx.send_trace(vec![1]);
142+
tx.send_ocsf(vec![2]);
143+
tx.send_trace(vec![3]);
144+
145+
let items = rx.drain();
146+
assert_eq!(items.len(), 3);
147+
assert!(matches!(&items[0], TelemetryItem::Trace(_)));
148+
assert!(matches!(&items[1], TelemetryItem::Ocsf(_)));
149+
assert_eq!(rx.metrics().depth(), 0);
150+
}
151+
152+
#[tokio::test]
153+
async fn depth_tracks_send_and_recv() {
154+
let (tx, mut rx) = new_telemetry_buffer(16);
155+
156+
tx.send_trace(vec![1]);
157+
tx.send_trace(vec![2]);
158+
tx.send_trace(vec![3]);
159+
assert_eq!(tx.metrics().depth(), 3);
160+
161+
rx.recv().await.unwrap();
162+
assert_eq!(rx.metrics().depth(), 2);
163+
164+
let remaining = rx.drain();
165+
assert_eq!(remaining.len(), 2);
166+
assert_eq!(rx.metrics().depth(), 0);
167+
}
168+
169+
#[tokio::test]
170+
async fn drop_count_increments_on_each_overflow() {
171+
let (tx, _rx) = new_telemetry_buffer(2);
172+
173+
tx.send_trace(vec![1]);
174+
tx.send_trace(vec![2]);
175+
assert_eq!(tx.metrics().drops(), 0);
176+
177+
for _ in 0..5 {
178+
tx.send_trace(vec![99]);
179+
}
180+
assert_eq!(tx.metrics().drops(), 5);
181+
assert_eq!(tx.metrics().depth(), 2);
182+
}
183+
184+
#[tokio::test]
185+
async fn metrics_shared_across_clones() {
186+
let (tx, mut rx) = new_telemetry_buffer(16);
187+
let tx2 = tx.clone();
188+
189+
tx.send_trace(vec![1]);
190+
tx2.send_trace(vec![2]);
191+
tx.send_ocsf(vec![3]);
192+
193+
assert_eq!(tx.metrics().depth(), 3);
194+
assert_eq!(tx2.metrics().depth(), 3);
195+
196+
rx.recv().await.unwrap();
197+
assert_eq!(tx.metrics().depth(), 2);
198+
assert_eq!(tx2.metrics().depth(), 2);
199+
}
200+
201+
#[tokio::test]
202+
async fn recv_returns_none_when_all_senders_dropped() {
203+
let (tx, mut rx) = new_telemetry_buffer(16);
204+
tx.send_trace(vec![1]);
205+
drop(tx);
206+
207+
assert!(rx.recv().await.is_some());
208+
assert!(rx.recv().await.is_none());
209+
}
210+
}

0 commit comments

Comments
 (0)