33
44//! Supervisor-owned access-plane assembly for a remote sandbox.
55
6- use std:: sync:: Arc ;
76use std:: sync:: atomic:: { AtomicBool , Ordering } ;
7+ use std:: sync:: { Arc , Mutex , PoisonError } ;
88use std:: time:: Duration ;
99
1010use miette:: Result ;
@@ -13,10 +13,17 @@ use openshell_isolation_interface::contract::{
1313} ;
1414use openshell_ocsf:: { ActivityId , AppLifecycleBuilder , SeverityId , StatusId , ocsf_emit} ;
1515
16+ use crate :: otlp:: RelayLifecycle ;
17+ use crate :: supervisor_session:: { DrainRequest , SessionRuntimeContext , TelemetryRelay } ;
18+
1619fn ocsf_ctx ( ) -> & ' static openshell_ocsf:: EventContext {
1720 openshell_ocsf:: ctx:: ctx ( )
1821}
1922
23+ /// Upper bound on the final OTLP relay drain before the exit is reported, so
24+ /// an unreachable gateway cannot delay the exit report.
25+ pub const OTEL_FINAL_DRAIN_TIMEOUT : Duration = Duration :: from_secs ( 3 ) ;
26+
2027/// Supervisor-owned SSH and gateway-session tasks for a running sandbox.
2128pub struct BoundaryAccess {
2229 instance_id : String ,
@@ -25,6 +32,8 @@ pub struct BoundaryAccess {
2532 session_task : Option < tokio:: task:: JoinHandle < ( ) > > ,
2633 session_readiness : Option < tokio:: sync:: watch:: Receiver < bool > > ,
2734 main_session : Option < Arc < crate :: main_session:: MainSession > > ,
35+ /// Sends the one-shot final drain request to the session task.
36+ telemetry_drain : Mutex < Option < tokio:: sync:: oneshot:: Sender < DrainRequest > > > ,
2837}
2938
3039impl BoundaryAccess {
@@ -34,6 +43,32 @@ impl BoundaryAccess {
3443 & self . instance_id
3544 }
3645
46+ /// Ask the session to stop the OTLP receiver and flush buffered telemetry
47+ /// onto its stream. Returns when the session acks or `deadline` elapses.
48+ /// A boundary without a relay or session returns at once. Only the first
49+ /// call sends a request.
50+ pub async fn drain_telemetry ( & self , deadline : Duration ) {
51+ let request_tx = self
52+ . telemetry_drain
53+ . lock ( )
54+ . unwrap_or_else ( PoisonError :: into_inner)
55+ . take ( ) ;
56+ let Some ( request_tx) = request_tx else {
57+ return ;
58+ } ;
59+ let ( done_tx, done_rx) = tokio:: sync:: oneshot:: channel ( ) ;
60+ if request_tx. send ( done_tx) . is_err ( ) {
61+ tracing:: debug!( "OTEL relay drain skipped: session task is gone" ) ;
62+ return ;
63+ }
64+ if tokio:: time:: timeout ( deadline, done_rx) . await . is_err ( ) {
65+ tracing:: warn!(
66+ ?deadline,
67+ "OTEL relay final drain did not complete within deadline"
68+ ) ;
69+ }
70+ }
71+
3772 /// Observe whether the gateway has accepted the current supervisor
3873 /// session. The value returns to false while the session reconnects.
3974 #[ must_use]
@@ -87,9 +122,13 @@ pub async fn start_boundary_access(
87122 port_forward : Arc < dyn BoundaryLoopbackConnector > ,
88123 agent : Arc < dyn BoundaryProcess > ,
89124 supervisor_session_updates : Option < tokio:: sync:: watch:: Sender < Option < String > > > ,
125+ telemetry : Option < RelayLifecycle > ,
90126) -> Result < BoundaryAccess > {
91127 let instance_id = uuid:: Uuid :: new_v4 ( ) . to_string ( ) ;
92128 let terminating = Arc :: new ( AtomicBool :: new ( false ) ) ;
129+ // Without a gateway session the relay has nowhere to forward; dropping
130+ // it here leaves the connection server accepting into a closed buffer,
131+ // which counts drops and never blocks the workload.
93132 let Some ( ssh_socket_path) = ssh_socket_path. map ( std:: path:: PathBuf :: from) else {
94133 return Ok ( BoundaryAccess {
95134 instance_id,
@@ -98,6 +137,7 @@ pub async fn start_boundary_access(
98137 session_task : None ,
99138 session_readiness : None ,
100139 main_session : None ,
140+ telemetry_drain : Mutex :: new ( None ) ,
101141 } ) ;
102142 } ;
103143
@@ -154,26 +194,32 @@ pub async fn start_boundary_access(
154194 }
155195 }
156196
157- let ( session_task, session_readiness) = match ( openshell_endpoint, sandbox_id) {
197+ let ( session_task, session_readiness, telemetry_drain) = match ( openshell_endpoint, sandbox_id)
198+ {
158199 ( Some ( endpoint) , Some ( id) ) => {
200+ let ( telemetry, drain_tx) = telemetry. map_or ( ( None , None ) , |relay| {
201+ let ( drain_tx, drain_rx) = tokio:: sync:: oneshot:: channel ( ) ;
202+ ( Some ( TelemetryRelay { relay, drain_rx } ) , Some ( drain_tx) )
203+ } ) ;
159204 let ( task, mut accepted) = crate :: supervisor_session:: spawn_with_readiness (
160205 endpoint. to_string ( ) ,
161206 id. to_string ( ) ,
162207 ssh_socket_path,
163208 port_forward,
164209 None ,
165210 terminating. clone ( ) ,
166- crate :: supervisor_session :: SessionRuntimeContext {
211+ SessionRuntimeContext {
167212 instance_id : instance_id. clone ( ) ,
168213 session_id_updates : supervisor_session_updates,
214+ telemetry,
169215 } ,
170216 ) ;
171217 let accepted_result =
172218 tokio:: time:: timeout ( Duration :: from_secs ( 10 ) , accepted. wait_for ( |ready| * ready) )
173219 . await
174220 . map ( |result| result. map ( |_| ( ) ) ) ;
175221 match accepted_result {
176- Ok ( Ok ( ( ) ) ) => ( Some ( task) , Some ( accepted) ) ,
222+ Ok ( Ok ( ( ) ) ) => ( Some ( task) , Some ( accepted) , drain_tx ) ,
177223 Ok ( Err ( _) ) => {
178224 task. abort ( ) ;
179225 return Err ( miette:: miette!(
@@ -188,7 +234,7 @@ pub async fn start_boundary_access(
188234 }
189235 }
190236 }
191- _ => ( None , None ) ,
237+ _ => ( None , None , None ) ,
192238 } ;
193239
194240 Ok ( BoundaryAccess {
@@ -198,6 +244,7 @@ pub async fn start_boundary_access(
198244 session_task,
199245 session_readiness,
200246 main_session : Some ( main_session) ,
247+ telemetry_drain : Mutex :: new ( telemetry_drain) ,
201248 } )
202249}
203250
@@ -253,6 +300,20 @@ pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instan
253300mod tests {
254301 use super :: * ;
255302
303+ fn access_with_drain (
304+ drain : Option < tokio:: sync:: oneshot:: Sender < DrainRequest > > ,
305+ ) -> BoundaryAccess {
306+ BoundaryAccess {
307+ instance_id : "instance" . to_string ( ) ,
308+ terminating : Arc :: new ( AtomicBool :: new ( false ) ) ,
309+ ssh_task : None ,
310+ session_task : None ,
311+ session_readiness : None ,
312+ main_session : None ,
313+ telemetry_drain : Mutex :: new ( drain) ,
314+ }
315+ }
316+
256317 #[ tokio:: test]
257318 async fn expected_post_exit_attachment_is_preserved_for_remote_main ( ) {
258319 let main_session = crate :: main_session:: MainSession :: inert ( ) ;
@@ -263,6 +324,7 @@ mod tests {
263324 session_task : None ,
264325 session_readiness : None ,
265326 main_session : Some ( main_session. clone ( ) ) ,
327+ telemetry_drain : Mutex :: new ( None ) ,
266328 } ;
267329
268330 access. publish_main_exit ( 7 , true ) . await ;
@@ -272,4 +334,77 @@ mod tests {
272334 . expect ( "declared CLI attachment must remain valid after a fast remote main exits" ) ;
273335 main_session. end_terminal_attachment ( ) ;
274336 }
337+
338+ #[ tokio:: test]
339+ async fn drain_telemetry_returns_at_once_without_a_relay ( ) {
340+ let access = access_with_drain ( None ) ;
341+ tokio:: time:: timeout (
342+ Duration :: from_millis ( 100 ) ,
343+ access. drain_telemetry ( Duration :: from_secs ( 5 ) ) ,
344+ )
345+ . await
346+ . expect ( "no relay means nothing to wait for" ) ;
347+ }
348+
349+ #[ tokio:: test]
350+ async fn drain_telemetry_returns_at_once_when_the_session_is_gone ( ) {
351+ let ( drain_tx, drain_rx) = tokio:: sync:: oneshot:: channel :: < DrainRequest > ( ) ;
352+ drop ( drain_rx) ;
353+ let access = access_with_drain ( Some ( drain_tx) ) ;
354+ tokio:: time:: timeout (
355+ Duration :: from_millis ( 100 ) ,
356+ access. drain_telemetry ( Duration :: from_secs ( 5 ) ) ,
357+ )
358+ . await
359+ . expect ( "a dropped session receiver must not stall the exit path" ) ;
360+ }
361+
362+ #[ tokio:: test]
363+ async fn drain_telemetry_waits_for_the_session_ack_and_sends_once ( ) {
364+ let ( drain_tx, drain_rx) = tokio:: sync:: oneshot:: channel :: < DrainRequest > ( ) ;
365+ let access = access_with_drain ( Some ( drain_tx) ) ;
366+
367+ let session = tokio:: spawn ( async move {
368+ let done_tx = drain_rx. await . expect ( "drain request arrives" ) ;
369+ tokio:: time:: sleep ( Duration :: from_millis ( 50 ) ) . await ;
370+ let _ = done_tx. send ( ( ) ) ;
371+ } ) ;
372+
373+ let started = std:: time:: Instant :: now ( ) ;
374+ access. drain_telemetry ( Duration :: from_secs ( 5 ) ) . await ;
375+ assert ! (
376+ started. elapsed( ) >= Duration :: from_millis( 50 ) ,
377+ "drain waits for the session's ack"
378+ ) ;
379+ session. await . unwrap ( ) ;
380+
381+ // Second call finds the request already sent and returns immediately.
382+ tokio:: time:: timeout (
383+ Duration :: from_millis ( 100 ) ,
384+ access. drain_telemetry ( Duration :: from_secs ( 5 ) ) ,
385+ )
386+ . await
387+ . expect ( "only the first call sends a request" ) ;
388+ }
389+
390+ #[ tokio:: test]
391+ async fn drain_telemetry_gives_up_at_the_deadline ( ) {
392+ let ( drain_tx, drain_rx) = tokio:: sync:: oneshot:: channel :: < DrainRequest > ( ) ;
393+ let access = access_with_drain ( Some ( drain_tx) ) ;
394+ // Hold the request open and never ack.
395+ let held = tokio:: spawn ( async move {
396+ let done_tx = drain_rx. await . expect ( "drain request arrives" ) ;
397+ tokio:: time:: sleep ( Duration :: from_secs ( 30 ) ) . await ;
398+ drop ( done_tx) ;
399+ } ) ;
400+
401+ let started = std:: time:: Instant :: now ( ) ;
402+ access. drain_telemetry ( Duration :: from_millis ( 100 ) ) . await ;
403+ let elapsed = started. elapsed ( ) ;
404+ assert ! (
405+ elapsed >= Duration :: from_millis( 100 ) && elapsed < Duration :: from_secs( 2 ) ,
406+ "drain returns at the deadline, got {elapsed:?}"
407+ ) ;
408+ held. abort ( ) ;
409+ }
275410}
0 commit comments