Skip to content

Commit b3d63c4

Browse files
committed
feat(tracing): show boundary connection time during sandbox startup
Signed-off-by: Shiju <[email protected]>
1 parent 74eacee commit b3d63c4

4 files changed

Lines changed: 266 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-sandbox-backend/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ rustix = { workspace = true, features = ["fs", "process"] }
4444

4545
[dev-dependencies]
4646
jsonwebtoken = { workspace = true }
47+
tracing-subscriber = { workspace = true }
4748

4849
[lints]
4950
workspace = true

‎crates/openshell-sandbox-backend/src/runtime.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,16 @@ impl BoundaryTransport {
156156

157157
/// Open one authenticated physical connection, retrying within the
158158
/// connect window while the remote boundary starts.
159+
#[tracing::instrument(
160+
name = "boundary.connect",
161+
skip_all,
162+
fields(transport = match &self.0 {
163+
TransportSource::Dial => "dial",
164+
TransportSource::Connector(_) => "backend",
165+
TransportSource::RuntimeInitiated(_) => "runtime_initiated",
166+
TransportSource::Routed(_) => "routed",
167+
})
168+
)]
159169
async fn connect(
160170
&self,
161171
descriptor: &SandboxRuntimeDescriptor,
@@ -3036,6 +3046,7 @@ pub mod test_support;
30363046

30373047
#[cfg(test)]
30383048
mod tests {
3049+
mod connection_tracing;
30393050
mod credential_renewal;
30403051
mod delegated;
30413052
mod flow_control;
Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
use super::*;
5+
use std::collections::BTreeMap;
6+
use std::future::Future;
7+
#[cfg(unix)]
8+
use std::future::poll_fn;
9+
use std::sync::Mutex;
10+
use tracing::Instrument as _;
11+
use tracing::instrument::WithSubscriber as _;
12+
use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt as _};
13+
use tracing_subscriber::registry::LookupSpan;
14+
15+
#[derive(Clone, Debug)]
16+
struct ConnectionSpan {
17+
id: tracing::Id,
18+
parent: Option<tracing::Id>,
19+
fields: BTreeMap<&'static str, String>,
20+
closed: bool,
21+
}
22+
23+
#[derive(Clone, Default)]
24+
struct ConnectionSpans(Arc<Mutex<Vec<ConnectionSpan>>>);
25+
26+
impl ConnectionSpans {
27+
fn snapshot(&self) -> Vec<ConnectionSpan> {
28+
self.0.lock().unwrap().clone()
29+
}
30+
}
31+
32+
#[derive(Default)]
33+
struct Fields(BTreeMap<&'static str, String>);
34+
35+
impl tracing::field::Visit for Fields {
36+
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
37+
self.0.insert(field.name(), format!("{value:?}"));
38+
}
39+
40+
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
41+
self.0.insert(field.name(), value.to_string());
42+
}
43+
}
44+
45+
impl<S: tracing::Subscriber + for<'span> LookupSpan<'span>> tracing_subscriber::Layer<S>
46+
for ConnectionSpans
47+
{
48+
fn on_new_span(
49+
&self,
50+
attributes: &tracing::span::Attributes<'_>,
51+
id: &tracing::Id,
52+
context: LayerContext<'_, S>,
53+
) {
54+
let span = context.span(id).unwrap();
55+
if span.name() != "boundary.connect" {
56+
return;
57+
}
58+
let mut fields = Fields::default();
59+
attributes.record(&mut fields);
60+
self.0.lock().unwrap().push(ConnectionSpan {
61+
id: id.clone(),
62+
parent: span.parent().map(|parent| parent.id()),
63+
fields: fields.0,
64+
closed: false,
65+
});
66+
}
67+
68+
fn on_close(&self, id: tracing::Id, _: LayerContext<'_, S>) {
69+
for span in self.0.lock().unwrap().iter_mut() {
70+
if span.id == id {
71+
span.closed = true;
72+
}
73+
}
74+
}
75+
}
76+
77+
#[tokio::test]
78+
async fn connection_span_covers_authenticated_wait_and_cached_reuse() {
79+
for kind in [
80+
"dial",
81+
"backend",
82+
"routed",
83+
#[cfg(unix)]
84+
"runtime_initiated",
85+
] {
86+
let certificate = test_certificate();
87+
let mut descriptor =
88+
tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), certificate.client_tls);
89+
let mut router = None;
90+
let runtime: Pin<Box<dyn Future<Output = BoundaryDuplexStream> + Send>>;
91+
let transport = match kind {
92+
"dial" => {
93+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
94+
descriptor.transport = SandboxTransport::Tcp {
95+
authority: "sandbox.test".to_string(),
96+
addresses: vec![listener.local_addr().unwrap()],
97+
};
98+
runtime = Box::pin(async move {
99+
let stream: BoundaryDuplexStream = Box::new(listener.accept().await.unwrap().0);
100+
stream
101+
});
102+
BoundaryTransport::default()
103+
}
104+
"backend" => {
105+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
106+
let address = listener.local_addr().unwrap();
107+
runtime = Box::pin(async move {
108+
let stream: BoundaryDuplexStream =
109+
Box::new(tokio::net::TcpStream::connect(address).await.unwrap());
110+
stream
111+
});
112+
BoundaryTransport::connector(Arc::new(ReverseTestConnector(listener)))
113+
}
114+
#[cfg(unix)]
115+
"runtime_initiated" => {
116+
let socket_path = PathBuf::from(format!("/tmp/ost-{}.sock", uuid::Uuid::new_v4()));
117+
descriptor.transport = SandboxTransport::RuntimeInitiatedUnix {
118+
socket_path: socket_path.clone(),
119+
};
120+
let transport = BoundaryTransport::for_descriptor(&descriptor).unwrap();
121+
runtime = Box::pin(async move {
122+
let stream: BoundaryDuplexStream =
123+
Box::new(UnixStream::connect(socket_path).await.unwrap());
124+
stream
125+
});
126+
transport
127+
}
128+
"routed" => {
129+
let listener =
130+
crate::runtime_routed::RuntimeRouter::bind("127.0.0.1:0".parse().unwrap())
131+
.unwrap();
132+
let address = listener.local_addr();
133+
let route = crate::runtime_routed::new_route_handle();
134+
descriptor.transport = SandboxTransport::RuntimeInitiatedRouted {
135+
authority: "sandbox.test".to_string(),
136+
route: route.clone(),
137+
};
138+
let registration = listener.register(&route, &descriptor.tls).unwrap();
139+
let transport = BoundaryTransport::routed(&descriptor, registration).unwrap();
140+
router = Some(listener);
141+
runtime = Box::pin(async move {
142+
let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
143+
stream
144+
.write_all(&crate::runtime_routed::encode_route_preface(&route).unwrap())
145+
.await
146+
.unwrap();
147+
let stream: BoundaryDuplexStream = Box::new(stream);
148+
stream
149+
});
150+
transport
151+
}
152+
_ => unreachable!(),
153+
};
154+
let (connected_tx, connected_rx) = tokio::sync::oneshot::channel();
155+
let (authenticate_tx, authenticate_rx) = tokio::sync::oneshot::channel();
156+
let server = tokio::spawn(async move {
157+
let stream = runtime.await;
158+
connected_tx.send(()).unwrap();
159+
authenticate_rx.await.unwrap();
160+
let stream = tokio_rustls::TlsAcceptor::from(certificate.server_config)
161+
.accept(stream)
162+
.await
163+
.unwrap();
164+
serve_test_grpc(Box::new(stream), "a".repeat(32)).await;
165+
});
166+
let mut client = BoundaryClient::new(descriptor, test_bearer(&"a".repeat(32)));
167+
client.transport = transport;
168+
let spans = ConnectionSpans::default();
169+
let subscriber = tracing_subscriber::registry().with(spans.clone());
170+
let exercise = async {
171+
let discovery = tracing::info_span!("supervisor.boundary.discover_policy");
172+
let request = client.discover_policy().instrument(discovery.clone());
173+
tokio::pin!(request);
174+
tokio::select! {
175+
biased;
176+
result = &mut request => panic!("discovery finished before TLS: {result:?}"),
177+
result = connected_rx => result.unwrap(),
178+
}
179+
{
180+
let captured = spans.snapshot();
181+
assert_eq!(captured.len(), 1, "{kind}");
182+
assert_eq!(captured[0].parent, discovery.id(), "{kind}");
183+
assert!(!captured[0].closed, "TLS authentication is still pending");
184+
assert_eq!(
185+
captured[0].fields,
186+
BTreeMap::from([("transport", kind.to_string())])
187+
);
188+
}
189+
authenticate_tx.send(()).unwrap();
190+
assert_eq!(request.await.unwrap(), (None, false));
191+
assert!(spans.snapshot()[0].closed);
192+
193+
assert_eq!(client.discover_policy().await.unwrap(), (None, false));
194+
assert_eq!(spans.snapshot().len(), 1, "cached {kind} connection");
195+
}
196+
.with_subscriber(subscriber);
197+
tokio::time::timeout(Duration::from_secs(5), exercise)
198+
.await
199+
.expect("authenticated discovery completes after the handshake is released");
200+
server.abort();
201+
drop(client);
202+
drop(router);
203+
}
204+
}
205+
206+
#[cfg(unix)]
207+
#[tokio::test]
208+
async fn connection_span_closes_on_error_and_cancellation() {
209+
let certificate = test_certificate();
210+
let mut descriptor =
211+
tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), certificate.client_tls);
212+
descriptor.transport = SandboxTransport::RuntimeInitiatedUnix {
213+
socket_path: PathBuf::from(format!("/tmp/ost-{}.sock", uuid::Uuid::new_v4())),
214+
};
215+
let transport = BoundaryTransport::for_descriptor(&descriptor).unwrap();
216+
let spans = ConnectionSpans::default();
217+
let subscriber = tracing_subscriber::registry().with(spans.clone());
218+
async {
219+
let discovery = tracing::info_span!("supervisor.boundary.discover_policy");
220+
let mut waiting = Box::pin(transport.connect(&descriptor).instrument(discovery.clone()));
221+
poll_fn(|context| {
222+
assert!(waiting.as_mut().poll(context).is_pending());
223+
Poll::Ready(())
224+
})
225+
.await;
226+
assert_eq!(spans.snapshot().len(), 1);
227+
assert!(!spans.snapshot()[0].closed);
228+
drop(waiting);
229+
assert!(spans.snapshot()[0].closed);
230+
231+
let mut wrong_session = descriptor.clone();
232+
wrong_session.tls = test_certificate().client_tls;
233+
assert!(matches!(
234+
transport
235+
.connect(&wrong_session)
236+
.instrument(discovery.clone())
237+
.await,
238+
Err(BackendError::Descriptor(_))
239+
));
240+
let captured = spans.snapshot();
241+
assert_eq!(captured.len(), 2);
242+
for span in &captured {
243+
assert!(span.closed);
244+
assert_eq!(span.parent, discovery.id());
245+
assert_eq!(
246+
span.fields,
247+
BTreeMap::from([("transport", "runtime_initiated".to_string())])
248+
);
249+
}
250+
}
251+
.with_subscriber(subscriber)
252+
.await;
253+
}

0 commit comments

Comments
 (0)