Skip to content

Commit 2d6be3c

Browse files
committed
feat(cli): forward TRACEPARENT/TRACESTATE to gateway for CI trace continuity
The CLI's gateway gRPC channel now passively forwards the W3C trace context (TRACEPARENT/TRACESTATE) from its environment, so a CI pipeline's trace parents the gateway's spans. This needs no OTel collector or provider on the CLI side - short-lived clients on CI runners and jump hosts just pass through what the CI system already exports. Add EnvTraceContextInterceptor to openshell-otel: it reads the trace context once at construction and injects it as gRPC metadata, and is a no-op when the context is unset or invalid. Compose it with EdgeAuthInterceptor via a new GatewayInterceptor, since tonic allows only one interceptor per channel. Closes issue NVIDIA#2639 Signed-off-by: Chris Moore <[email protected]>
1 parent b1cfc91 commit 2d6be3c

6 files changed

Lines changed: 195 additions & 9 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-cli/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ path = "src/main.rs"
1717
[dependencies]
1818
openshell-bootstrap = { path = "../openshell-bootstrap" }
1919
openshell-core = { path = "../openshell-core", default-features = false }
20+
openshell-otel = { path = "../openshell-otel" }
2021
openshell-policy = { path = "../openshell-policy" }
2122
openshell-providers = { path = "../openshell-providers" }
2223
openshell-sdk = { path = "../openshell-sdk" }

‎crates/openshell-cli/src/tls.rs‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ use miette::{IntoDiagnostic, Result, WrapErr};
55
use openshell_core::auth::EdgeAuthInterceptor;
66
use openshell_core::net::set_tcp_nodelay_best_effort;
77
use openshell_core::proto::open_shell_client::OpenShellClient;
8+
use openshell_otel::EnvTraceContextInterceptor;
89
use rustls::{
910
RootCertStore,
1011
client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier},
@@ -25,7 +26,29 @@ use tracing::debug;
2526
use url::{Host, Url};
2627

2728
/// Concrete gRPC client type used by all commands.
28-
pub type GrpcClient = OpenShellClient<InterceptedService<Channel, EdgeAuthInterceptor>>;
29+
pub type GrpcClient = OpenShellClient<InterceptedService<Channel, GatewayInterceptor>>;
30+
31+
/// Interceptor stack applied to every gateway gRPC request.
32+
///
33+
/// Runs authentication header injection first, then passive W3C trace-context
34+
/// forwarding so a CI pipeline's `TRACEPARENT` extends into the gateway's
35+
/// spans. tonic allows a single interceptor per channel, so the two concerns
36+
/// are composed here.
37+
#[derive(Clone)]
38+
pub struct GatewayInterceptor {
39+
auth: EdgeAuthInterceptor,
40+
trace: EnvTraceContextInterceptor,
41+
}
42+
43+
impl tonic::service::Interceptor for GatewayInterceptor {
44+
fn call(
45+
&mut self,
46+
request: tonic::Request<()>,
47+
) -> std::result::Result<tonic::Request<()>, tonic::Status> {
48+
let request = self.auth.call(request)?;
49+
self.trace.call(request)
50+
}
51+
}
2952

3053
#[derive(Clone, Debug, Default)]
3154
pub struct TlsOptions {
@@ -451,17 +474,22 @@ pub async fn build_channel(server: &str, tls: &TlsOptions) -> Result<Channel> {
451474

452475
/// Build a gRPC [`OpenShellClient`].
453476
///
454-
/// When `tls.edge_token` is set, the returned client is wrapped with an
455-
/// interceptor that injects authentication headers on every request.
456-
/// Otherwise, standard mTLS is used (interceptor is a no-op).
477+
/// The returned client carries a [`GatewayInterceptor`] that injects
478+
/// authentication headers (when a token is set) and passively forwards the
479+
/// `TRACEPARENT`/`TRACESTATE` trace context from the environment on every
480+
/// request. With no token and no trace context, the interceptor is a no-op.
457481
pub async fn grpc_client(server: &str, tls: &TlsOptions) -> Result<GrpcClient> {
458482
let channel = build_channel(server, tls).await?;
459483
let interceptor = interceptor_from_tls(tls)?;
460484
Ok(OpenShellClient::with_interceptor(channel, interceptor))
461485
}
462486

463-
fn interceptor_from_tls(tls: &TlsOptions) -> Result<EdgeAuthInterceptor> {
464-
EdgeAuthInterceptor::new(tls.oidc_token.as_deref(), tls.edge_token.as_deref())
487+
fn interceptor_from_tls(tls: &TlsOptions) -> Result<GatewayInterceptor> {
488+
let auth = EdgeAuthInterceptor::new(tls.oidc_token.as_deref(), tls.edge_token.as_deref())?;
489+
Ok(GatewayInterceptor {
490+
auth,
491+
trace: EnvTraceContextInterceptor::from_env(),
492+
})
465493
}
466494

467495
/// Gateway replica that owns the sandbox named in `response`, when it says.

‎crates/openshell-otel/src/lib.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ pub use grpc::{
2121
};
2222
pub use ocsf::{OcsfOtlp, OcsfOtlpLayer};
2323
pub use propagation::{
24-
HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV, TRACESTATE_ENV,
25-
TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment,
26-
set_parent_from_environment, trace_context_environment,
24+
EnvTraceContextInterceptor, HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV,
25+
TRACESTATE_ENV, TraceContextInterceptor, current_trace_context_carrier,
26+
current_trace_context_environment, set_parent_from_environment, trace_context_environment,
2727
};
2828

2929
use opentelemetry::KeyValue;

‎crates/openshell-otel/src/propagation.rs‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
use std::collections::BTreeMap;
77

88
use http::HeaderMap;
9+
use opentelemetry::Context;
910
use opentelemetry::propagation::{Extractor, Injector, TextMapPropagator};
1011
use opentelemetry_sdk::propagation::TraceContextPropagator;
1112
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
@@ -167,9 +168,108 @@ impl tonic::service::Interceptor for TraceContextInterceptor {
167168
}
168169
}
169170

171+
/// Reads W3C trace-context propagation fields from process environment
172+
/// variables (`TRACEPARENT` and `TRACESTATE`).
173+
///
174+
/// CI systems (GitHub Actions, GitLab CI, Jenkins OpenTelemetry plugins)
175+
/// export these variables to propagate the pipeline trace to child processes.
176+
/// Both upper-case and lower-case spellings are accepted; blank values are
177+
/// treated as unset.
178+
#[derive(Debug, Clone, Default)]
179+
struct EnvTraceContext {
180+
traceparent: Option<String>,
181+
tracestate: Option<String>,
182+
}
183+
184+
impl EnvTraceContext {
185+
fn from_env() -> Self {
186+
Self {
187+
traceparent: read_env("TRACEPARENT"),
188+
tracestate: read_env("TRACESTATE"),
189+
}
190+
}
191+
}
192+
193+
/// Read an environment variable by its upper-case name, falling back to the
194+
/// lower-case spelling, trimming whitespace and discarding blank values.
195+
fn read_env(upper: &str) -> Option<String> {
196+
std::env::var(upper)
197+
.ok()
198+
.or_else(|| std::env::var(upper.to_ascii_lowercase()).ok())
199+
.map(|value| value.trim().to_string())
200+
.filter(|value| !value.is_empty())
201+
}
202+
203+
impl Extractor for EnvTraceContext {
204+
fn get(&self, key: &str) -> Option<&str> {
205+
match key {
206+
"traceparent" => self.traceparent.as_deref(),
207+
"tracestate" => self.tracestate.as_deref(),
208+
_ => None,
209+
}
210+
}
211+
212+
fn keys(&self) -> Vec<&str> {
213+
let mut keys = Vec::new();
214+
if self.traceparent.is_some() {
215+
keys.push("traceparent");
216+
}
217+
if self.tracestate.is_some() {
218+
keys.push("tracestate");
219+
}
220+
keys
221+
}
222+
}
223+
224+
/// Passively forwards a W3C trace context captured from the process
225+
/// environment onto outbound tonic requests.
226+
///
227+
/// Unlike [`TraceContextInterceptor`], this does not require an active
228+
/// OpenTelemetry span or an `SdkTracerProvider`; it reads `TRACEPARENT`
229+
/// (and optionally `TRACESTATE`) once at construction and injects them as
230+
/// gRPC metadata. This lets short-lived clients (e.g. the CLI on a CI runner)
231+
/// extend a pipeline trace to the gateway without any collector configuration.
232+
///
233+
/// When `TRACEPARENT` is unset or invalid the captured context carries no
234+
/// valid span, so injection is a no-op.
235+
#[derive(Debug, Clone)]
236+
pub struct EnvTraceContextInterceptor {
237+
context: Context,
238+
}
239+
240+
impl EnvTraceContextInterceptor {
241+
/// Capture the W3C trace context from the process environment.
242+
#[must_use]
243+
pub fn from_env() -> Self {
244+
Self {
245+
context: context_from(&EnvTraceContext::from_env()),
246+
}
247+
}
248+
}
249+
250+
/// Extract a [`Context`] from an [`Extractor`] using a fresh base context, so
251+
/// the result depends only on the extractor and not on any ambient context.
252+
fn context_from(extractor: &impl Extractor) -> Context {
253+
TraceContextPropagator::new().extract_with_context(&Context::new(), extractor)
254+
}
255+
256+
impl tonic::service::Interceptor for EnvTraceContextInterceptor {
257+
fn call(
258+
&mut self,
259+
mut request: tonic::Request<()>,
260+
) -> Result<tonic::Request<()>, tonic::Status> {
261+
TraceContextPropagator::new().inject_context(
262+
&self.context,
263+
&mut MetadataMapInjector::new(request.metadata_mut()),
264+
);
265+
Ok(request)
266+
}
267+
}
268+
170269
#[cfg(test)]
171270
mod tests {
172271
use super::*;
272+
use tonic::service::Interceptor as _;
173273

174274
fn env_test_exporter() -> (
175275
opentelemetry_sdk::trace::SdkTracerProvider,
@@ -320,4 +420,58 @@ mod tests {
320420
Some("value")
321421
);
322422
}
423+
424+
fn interceptor_with(extractor: EnvTraceContext) -> EnvTraceContextInterceptor {
425+
EnvTraceContextInterceptor {
426+
context: context_from(&extractor),
427+
}
428+
}
429+
430+
#[test]
431+
fn env_interceptor_forwards_valid_traceparent() {
432+
let traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
433+
let mut interceptor = interceptor_with(EnvTraceContext {
434+
traceparent: Some(traceparent.to_string()),
435+
tracestate: Some("vendor=value".to_string()),
436+
});
437+
438+
let request = interceptor.call(tonic::Request::new(())).unwrap();
439+
440+
assert_eq!(
441+
request
442+
.metadata()
443+
.get("traceparent")
444+
.and_then(|value| value.to_str().ok()),
445+
Some(traceparent)
446+
);
447+
assert_eq!(
448+
request
449+
.metadata()
450+
.get("tracestate")
451+
.and_then(|value| value.to_str().ok()),
452+
Some("vendor=value")
453+
);
454+
}
455+
456+
#[test]
457+
fn env_interceptor_is_noop_without_traceparent() {
458+
let mut interceptor = interceptor_with(EnvTraceContext::default());
459+
460+
let request = interceptor.call(tonic::Request::new(())).unwrap();
461+
462+
assert!(request.metadata().get("traceparent").is_none());
463+
assert!(request.metadata().get("tracestate").is_none());
464+
}
465+
466+
#[test]
467+
fn env_interceptor_is_noop_for_invalid_traceparent() {
468+
let mut interceptor = interceptor_with(EnvTraceContext {
469+
traceparent: Some("not-a-valid-traceparent".to_string()),
470+
tracestate: None,
471+
});
472+
473+
let request = interceptor.call(tonic::Request::new(())).unwrap();
474+
475+
assert!(request.metadata().get("traceparent").is_none());
476+
}
323477
}

‎docs/observability/traces.mdx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ Give each installation that reports to the same collector its own name, includin
6969

7070
The gateway continues a valid W3C `traceparent` sent by the caller, so a client's trace can include the gateway's spans for its requests. Without one, the gateway starts a new trace.
7171

72+
The CLI forwards `TRACEPARENT` and `TRACESTATE` from its environment on every gateway request. A CI system that exports these variables (GitHub Actions, GitLab CI, Jenkins OpenTelemetry plugins) therefore parents the gateway's spans for the commands it runs, with no collector or provider configuration on the CI runner.
73+
7274
The gateway also passes its trace context to compute drivers, so driver spans appear in the same trace as the gateway request that caused them. Built-in drivers export their spans through the gateway's process. Managed external drivers receive the trace endpoint, gateway name, and compute driver from the gateway. Drivers that you run yourself own their telemetry configuration.
7375

7476
## Next Steps

0 commit comments

Comments
 (0)