Skip to content

Commit e9eff28

Browse files
committed
feat(supervisor): wire the OTLP relay into the proxy and gateway session
Start the relay before networking and register its connection server as the proxy's reserved destination, so the first staged workload stream to the relay address is served immediately. The relay lifecycle rides into the gateway session through `SessionRuntimeContext`: the session advertises `otel_export` only when a relay is running, gates forwarding on the gateway's confirmation, and forwards buffered items with a non-blocking send so telemetry can never stall control traffic. Items received while no session has confirmed accumulate in the bounded buffer. `BoundaryAccess::drain_telemetry` sends a one-shot drain request to the session, which stops the receiver and flushes the buffer onto its stream before acking. The supervisor runs it as a completion phase ahead of the main-process exit report, bounded at three seconds so an unreachable gateway cannot delay the report. Span enrichment takes the driver name from the runtime descriptor's fence evidence and the uid from its resolved workload identity, captured before the identity moves into the sandbox context. The image attribute still reads `OPENSHELL_CONTAINER_IMAGE`, which only the Podman driver sets; the OCSF context shares that gap and it is left for a follow-up. Tests cover the drain handshake (no relay, session gone, ack, deadline), capability advertisement and gating, and an end-to-end HTTP round trip through the reserved-destination adapter over an in-memory boundary stream. Refs NVIDIA#2641 Signed-off-by: Roland Huß <[email protected]>
1 parent d9083da commit e9eff28

6 files changed

Lines changed: 461 additions & 16 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-supervisor-process/src/delegated.rs‎

Lines changed: 140 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33

44
//! Supervisor-owned access-plane assembly for a remote sandbox.
55
6-
use std::sync::Arc;
76
use std::sync::atomic::{AtomicBool, Ordering};
7+
use std::sync::{Arc, Mutex, PoisonError};
88
use std::time::Duration;
99

1010
use miette::Result;
@@ -13,10 +13,17 @@ use openshell_isolation_interface::contract::{
1313
};
1414
use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit};
1515

16+
use crate::otlp::RelayLifecycle;
17+
use crate::supervisor_session::{DrainRequest, SessionRuntimeContext, TelemetryRelay};
18+
1619
fn ocsf_ctx() -> &'static openshell_ocsf::EventContext {
1720
openshell_ocsf::ctx::ctx()
1821
}
1922

23+
/// Upper bound on the final OTLP relay drain before the exit is reported, so
24+
/// an unreachable gateway cannot delay the exit report.
25+
pub const OTEL_FINAL_DRAIN_TIMEOUT: Duration = Duration::from_secs(3);
26+
2027
/// Supervisor-owned SSH and gateway-session tasks for a running sandbox.
2128
pub struct BoundaryAccess {
2229
instance_id: String,
@@ -25,6 +32,8 @@ pub struct BoundaryAccess {
2532
session_task: Option<tokio::task::JoinHandle<()>>,
2633
session_readiness: Option<tokio::sync::watch::Receiver<bool>>,
2734
main_session: Option<Arc<crate::main_session::MainSession>>,
35+
/// Sends the one-shot final drain request to the session task.
36+
telemetry_drain: Mutex<Option<tokio::sync::oneshot::Sender<DrainRequest>>>,
2837
}
2938

3039
impl BoundaryAccess {
@@ -34,6 +43,32 @@ impl BoundaryAccess {
3443
&self.instance_id
3544
}
3645

46+
/// Ask the session to stop the OTLP receiver and flush buffered telemetry
47+
/// onto its stream. Returns when the session acks or `deadline` elapses.
48+
/// A boundary without a relay or session returns at once. Only the first
49+
/// call sends a request.
50+
pub async fn drain_telemetry(&self, deadline: Duration) {
51+
let request_tx = self
52+
.telemetry_drain
53+
.lock()
54+
.unwrap_or_else(PoisonError::into_inner)
55+
.take();
56+
let Some(request_tx) = request_tx else {
57+
return;
58+
};
59+
let (done_tx, done_rx) = tokio::sync::oneshot::channel();
60+
if request_tx.send(done_tx).is_err() {
61+
tracing::debug!("OTEL relay drain skipped: session task is gone");
62+
return;
63+
}
64+
if tokio::time::timeout(deadline, done_rx).await.is_err() {
65+
tracing::warn!(
66+
?deadline,
67+
"OTEL relay final drain did not complete within deadline"
68+
);
69+
}
70+
}
71+
3772
/// Observe whether the gateway has accepted the current supervisor
3873
/// session. The value returns to false while the session reconnects.
3974
#[must_use]
@@ -87,9 +122,13 @@ pub async fn start_boundary_access(
87122
port_forward: Arc<dyn BoundaryLoopbackConnector>,
88123
agent: Arc<dyn BoundaryProcess>,
89124
supervisor_session_updates: Option<tokio::sync::watch::Sender<Option<String>>>,
125+
telemetry: Option<RelayLifecycle>,
90126
) -> Result<BoundaryAccess> {
91127
let instance_id = uuid::Uuid::new_v4().to_string();
92128
let terminating = Arc::new(AtomicBool::new(false));
129+
// Without a gateway session the relay has nowhere to forward; dropping
130+
// it here leaves the connection server accepting into a closed buffer,
131+
// which counts drops and never blocks the workload.
93132
let Some(ssh_socket_path) = ssh_socket_path.map(std::path::PathBuf::from) else {
94133
return Ok(BoundaryAccess {
95134
instance_id,
@@ -98,6 +137,7 @@ pub async fn start_boundary_access(
98137
session_task: None,
99138
session_readiness: None,
100139
main_session: None,
140+
telemetry_drain: Mutex::new(None),
101141
});
102142
};
103143

@@ -154,26 +194,32 @@ pub async fn start_boundary_access(
154194
}
155195
}
156196

157-
let (session_task, session_readiness) = match (openshell_endpoint, sandbox_id) {
197+
let (session_task, session_readiness, telemetry_drain) = match (openshell_endpoint, sandbox_id)
198+
{
158199
(Some(endpoint), Some(id)) => {
200+
let (telemetry, drain_tx) = telemetry.map_or((None, None), |relay| {
201+
let (drain_tx, drain_rx) = tokio::sync::oneshot::channel();
202+
(Some(TelemetryRelay { relay, drain_rx }), Some(drain_tx))
203+
});
159204
let (task, mut accepted) = crate::supervisor_session::spawn_with_readiness(
160205
endpoint.to_string(),
161206
id.to_string(),
162207
ssh_socket_path,
163208
port_forward,
164209
None,
165210
terminating.clone(),
166-
crate::supervisor_session::SessionRuntimeContext {
211+
SessionRuntimeContext {
167212
instance_id: instance_id.clone(),
168213
session_id_updates: supervisor_session_updates,
214+
telemetry,
169215
},
170216
);
171217
let accepted_result =
172218
tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready))
173219
.await
174220
.map(|result| result.map(|_| ()));
175221
match accepted_result {
176-
Ok(Ok(())) => (Some(task), Some(accepted)),
222+
Ok(Ok(())) => (Some(task), Some(accepted), drain_tx),
177223
Ok(Err(_)) => {
178224
task.abort();
179225
return Err(miette::miette!(
@@ -188,7 +234,7 @@ pub async fn start_boundary_access(
188234
}
189235
}
190236
}
191-
_ => (None, None),
237+
_ => (None, None, None),
192238
};
193239

194240
Ok(BoundaryAccess {
@@ -198,6 +244,7 @@ pub async fn start_boundary_access(
198244
session_task,
199245
session_readiness,
200246
main_session: Some(main_session),
247+
telemetry_drain: Mutex::new(telemetry_drain),
201248
})
202249
}
203250

@@ -253,6 +300,20 @@ pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instan
253300
mod tests {
254301
use super::*;
255302

303+
fn access_with_drain(
304+
drain: Option<tokio::sync::oneshot::Sender<DrainRequest>>,
305+
) -> BoundaryAccess {
306+
BoundaryAccess {
307+
instance_id: "instance".to_string(),
308+
terminating: Arc::new(AtomicBool::new(false)),
309+
ssh_task: None,
310+
session_task: None,
311+
session_readiness: None,
312+
main_session: None,
313+
telemetry_drain: Mutex::new(drain),
314+
}
315+
}
316+
256317
#[tokio::test]
257318
async fn expected_post_exit_attachment_is_preserved_for_remote_main() {
258319
let main_session = crate::main_session::MainSession::inert();
@@ -263,6 +324,7 @@ mod tests {
263324
session_task: None,
264325
session_readiness: None,
265326
main_session: Some(main_session.clone()),
327+
telemetry_drain: Mutex::new(None),
266328
};
267329

268330
access.publish_main_exit(7, true).await;
@@ -272,4 +334,77 @@ mod tests {
272334
.expect("declared CLI attachment must remain valid after a fast remote main exits");
273335
main_session.end_terminal_attachment();
274336
}
337+
338+
#[tokio::test]
339+
async fn drain_telemetry_returns_at_once_without_a_relay() {
340+
let access = access_with_drain(None);
341+
tokio::time::timeout(
342+
Duration::from_millis(100),
343+
access.drain_telemetry(Duration::from_secs(5)),
344+
)
345+
.await
346+
.expect("no relay means nothing to wait for");
347+
}
348+
349+
#[tokio::test]
350+
async fn drain_telemetry_returns_at_once_when_the_session_is_gone() {
351+
let (drain_tx, drain_rx) = tokio::sync::oneshot::channel::<DrainRequest>();
352+
drop(drain_rx);
353+
let access = access_with_drain(Some(drain_tx));
354+
tokio::time::timeout(
355+
Duration::from_millis(100),
356+
access.drain_telemetry(Duration::from_secs(5)),
357+
)
358+
.await
359+
.expect("a dropped session receiver must not stall the exit path");
360+
}
361+
362+
#[tokio::test]
363+
async fn drain_telemetry_waits_for_the_session_ack_and_sends_once() {
364+
let (drain_tx, drain_rx) = tokio::sync::oneshot::channel::<DrainRequest>();
365+
let access = access_with_drain(Some(drain_tx));
366+
367+
let session = tokio::spawn(async move {
368+
let done_tx = drain_rx.await.expect("drain request arrives");
369+
tokio::time::sleep(Duration::from_millis(50)).await;
370+
let _ = done_tx.send(());
371+
});
372+
373+
let started = std::time::Instant::now();
374+
access.drain_telemetry(Duration::from_secs(5)).await;
375+
assert!(
376+
started.elapsed() >= Duration::from_millis(50),
377+
"drain waits for the session's ack"
378+
);
379+
session.await.unwrap();
380+
381+
// Second call finds the request already sent and returns immediately.
382+
tokio::time::timeout(
383+
Duration::from_millis(100),
384+
access.drain_telemetry(Duration::from_secs(5)),
385+
)
386+
.await
387+
.expect("only the first call sends a request");
388+
}
389+
390+
#[tokio::test]
391+
async fn drain_telemetry_gives_up_at_the_deadline() {
392+
let (drain_tx, drain_rx) = tokio::sync::oneshot::channel::<DrainRequest>();
393+
let access = access_with_drain(Some(drain_tx));
394+
// Hold the request open and never ack.
395+
let held = tokio::spawn(async move {
396+
let done_tx = drain_rx.await.expect("drain request arrives");
397+
tokio::time::sleep(Duration::from_secs(30)).await;
398+
drop(done_tx);
399+
});
400+
401+
let started = std::time::Instant::now();
402+
access.drain_telemetry(Duration::from_millis(100)).await;
403+
let elapsed = started.elapsed();
404+
assert!(
405+
elapsed >= Duration::from_millis(100) && elapsed < Duration::from_secs(2),
406+
"drain returns at the deadline, got {elapsed:?}"
407+
);
408+
held.abort();
409+
}
275410
}

0 commit comments

Comments
 (0)