Skip to content

Commit 1f7bd68

Browse files
committed
feat(cli): forward TRACEPARENT/TRACESTATE to gateway for CI trace continuity
The CLI's gateway gRPC channel now passively forwards the W3C trace context (TRACEPARENT/TRACESTATE) from its environment, so a CI pipeline's trace parents the gateway's spans. This needs no OTel collector or provider on the CLI side - short-lived clients on CI runners and jump hosts just pass through what the CI system already exports. Add EnvTraceContextInterceptor to openshell-otel: it reads the trace context once at construction and injects it as gRPC metadata, and is a no-op when the context is unset or invalid. Compose it with EdgeAuthInterceptor via a new GatewayInterceptor, since tonic allows only one interceptor per channel. Closes issue NVIDIA#2639 Signed-off-by: Chris Moore <[email protected]>
1 parent 44ef984 commit 1f7bd68

7 files changed

Lines changed: 213 additions & 8 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-cli/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ path = "src/main.rs"
1818
openshell-crypto = { workspace = true }
1919
openshell-bootstrap = { path = "../openshell-bootstrap" }
2020
openshell-core = { path = "../openshell-core", default-features = false }
21+
openshell-otel = { path = "../openshell-otel" }
2122
openshell-policy = { path = "../openshell-policy" }
2223
openshell-providers = { path = "../openshell-providers" }
2324
openshell-sdk = { path = "../openshell-sdk" }

‎crates/openshell-cli/src/tls.rs‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
use miette::{IntoDiagnostic, Result, WrapErr};
5+
use openshell_otel::EnvTraceContextInterceptor;
56
use openshell_sdk::{AuthConfig, ClientConfig, ClientIdentity, OpenShellClient};
67
use rustls::{
78
RootCertStore,
@@ -14,7 +15,31 @@ use tonic::metadata::AsciiMetadataValue;
1415
use tonic::transport::Channel;
1516

1617
/// Concrete gRPC client type used by all commands.
17-
pub type GrpcClient = openshell_sdk::raw::AuthedGrpcClient;
18+
pub type GrpcClient = openshell_sdk::raw::GrpcClient<
19+
tonic::service::interceptor::InterceptedService<Channel, GatewayInterceptor>,
20+
>;
21+
22+
/// Interceptor stack applied to every gateway gRPC request.
23+
///
24+
/// Runs authentication header injection first, then passive W3C trace-context
25+
/// forwarding so a CI pipeline's `TRACEPARENT` extends into the gateway's
26+
/// spans. tonic allows a single interceptor per channel, so the two concerns
27+
/// are composed here.
28+
#[derive(Clone)]
29+
pub struct GatewayInterceptor {
30+
auth: openshell_sdk::EdgeAuthInterceptor,
31+
trace: EnvTraceContextInterceptor,
32+
}
33+
34+
impl tonic::service::Interceptor for GatewayInterceptor {
35+
fn call(
36+
&mut self,
37+
request: tonic::Request<()>,
38+
) -> std::result::Result<tonic::Request<()>, tonic::Status> {
39+
let request = self.auth.call(request)?;
40+
self.trace.call(request)
41+
}
42+
}
1843

1944
#[derive(Clone, Debug, Default)]
2045
pub struct TlsOptions {
@@ -305,17 +330,28 @@ pub async fn build_channel(server: &str, tls: &TlsOptions) -> Result<Channel> {
305330

306331
/// Build the SDK client and return its raw gRPC escape hatch.
307332
///
308-
/// When `tls.edge_token` is set, the returned client is wrapped with an
309-
/// interceptor that injects authentication headers on every request.
310-
/// Otherwise, standard mTLS is used (interceptor is a no-op).
333+
/// The returned client carries a [`GatewayInterceptor`] that injects
334+
/// authentication headers (when a token is set) and passively forwards the
335+
/// `TRACEPARENT`/`TRACESTATE` trace context from the environment on every
336+
/// request. With no token and no trace context, the interceptor is a no-op.
311337
pub async fn grpc_client(server: &str, tls: &TlsOptions) -> Result<GrpcClient> {
312338
let config = sdk_client_config(server, tls)?;
313339
// Keep the SDK's transport state machine out of every command future.
314340
// Most CLI commands compose this helper into larger async operations.
315341
let client = Box::pin(OpenShellClient::connect(config))
316342
.await
317343
.into_diagnostic()?;
318-
Ok(client.raw_grpc())
344+
// tonic allows a single interceptor per channel: compose the SDK's auth
345+
// interceptor (sharing its live bearer slot) with the CLI's passive
346+
// trace-context forwarding.
347+
let interceptor = GatewayInterceptor {
348+
auth: client.auth_interceptor(),
349+
trace: EnvTraceContextInterceptor::from_env(),
350+
};
351+
Ok(openshell_sdk::raw::GrpcClient::with_interceptor(
352+
client.channel(),
353+
interceptor,
354+
))
319355
}
320356

321357
/// Gateway replica that owns the sandbox named in `response`, when it says.

‎crates/openshell-otel/src/lib.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ pub use grpc::{
2121
};
2222
pub use ocsf::{OcsfOtlp, OcsfOtlpLayer};
2323
pub use propagation::{
24-
HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV, TRACESTATE_ENV,
25-
TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment,
26-
set_parent_from_environment, set_parent_from_trace_context, trace_context_environment,
24+
EnvTraceContextInterceptor, HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV,
25+
TRACESTATE_ENV, TraceContextInterceptor, current_trace_context_carrier,
26+
current_trace_context_environment, set_parent_from_environment, set_parent_from_trace_context,
27+
trace_context_environment,
2728
};
2829

2930
use opentelemetry::KeyValue;

‎crates/openshell-otel/src/propagation.rs‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
use std::collections::BTreeMap;
77

88
use http::HeaderMap;
9+
use opentelemetry::Context;
910
use opentelemetry::propagation::{Extractor, Injector, TextMapPropagator};
1011
use opentelemetry_sdk::propagation::TraceContextPropagator;
1112
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
@@ -177,9 +178,108 @@ impl tonic::service::Interceptor for TraceContextInterceptor {
177178
}
178179
}
179180

181+
/// Reads W3C trace-context propagation fields from process environment
182+
/// variables (`TRACEPARENT` and `TRACESTATE`).
183+
///
184+
/// CI systems (GitHub Actions, GitLab CI, Jenkins OpenTelemetry plugins)
185+
/// export these variables to propagate the pipeline trace to child processes.
186+
/// Both upper-case and lower-case spellings are accepted; blank values are
187+
/// treated as unset.
188+
#[derive(Debug, Clone, Default)]
189+
struct EnvTraceContext {
190+
traceparent: Option<String>,
191+
tracestate: Option<String>,
192+
}
193+
194+
impl EnvTraceContext {
195+
fn from_env() -> Self {
196+
Self {
197+
traceparent: read_env("TRACEPARENT"),
198+
tracestate: read_env("TRACESTATE"),
199+
}
200+
}
201+
}
202+
203+
/// Read an environment variable by its upper-case name, falling back to the
204+
/// lower-case spelling, trimming whitespace and discarding blank values.
205+
fn read_env(upper: &str) -> Option<String> {
206+
std::env::var(upper)
207+
.ok()
208+
.or_else(|| std::env::var(upper.to_ascii_lowercase()).ok())
209+
.map(|value| value.trim().to_string())
210+
.filter(|value| !value.is_empty())
211+
}
212+
213+
impl Extractor for EnvTraceContext {
214+
fn get(&self, key: &str) -> Option<&str> {
215+
match key {
216+
"traceparent" => self.traceparent.as_deref(),
217+
"tracestate" => self.tracestate.as_deref(),
218+
_ => None,
219+
}
220+
}
221+
222+
fn keys(&self) -> Vec<&str> {
223+
let mut keys = Vec::new();
224+
if self.traceparent.is_some() {
225+
keys.push("traceparent");
226+
}
227+
if self.tracestate.is_some() {
228+
keys.push("tracestate");
229+
}
230+
keys
231+
}
232+
}
233+
234+
/// Passively forwards a W3C trace context captured from the process
235+
/// environment onto outbound tonic requests.
236+
///
237+
/// Unlike [`TraceContextInterceptor`], this does not require an active
238+
/// OpenTelemetry span or an `SdkTracerProvider`; it reads `TRACEPARENT`
239+
/// (and optionally `TRACESTATE`) once at construction and injects them as
240+
/// gRPC metadata. This lets short-lived clients (e.g. the CLI on a CI runner)
241+
/// extend a pipeline trace to the gateway without any collector configuration.
242+
///
243+
/// When `TRACEPARENT` is unset or invalid the captured context carries no
244+
/// valid span, so injection is a no-op.
245+
#[derive(Debug, Clone)]
246+
pub struct EnvTraceContextInterceptor {
247+
context: Context,
248+
}
249+
250+
impl EnvTraceContextInterceptor {
251+
/// Capture the W3C trace context from the process environment.
252+
#[must_use]
253+
pub fn from_env() -> Self {
254+
Self {
255+
context: context_from(&EnvTraceContext::from_env()),
256+
}
257+
}
258+
}
259+
260+
/// Extract a [`Context`] from an [`Extractor`] using a fresh base context, so
261+
/// the result depends only on the extractor and not on any ambient context.
262+
fn context_from(extractor: &impl Extractor) -> Context {
263+
TraceContextPropagator::new().extract_with_context(&Context::new(), extractor)
264+
}
265+
266+
impl tonic::service::Interceptor for EnvTraceContextInterceptor {
267+
fn call(
268+
&mut self,
269+
mut request: tonic::Request<()>,
270+
) -> Result<tonic::Request<()>, tonic::Status> {
271+
TraceContextPropagator::new().inject_context(
272+
&self.context,
273+
&mut MetadataMapInjector::new(request.metadata_mut()),
274+
);
275+
Ok(request)
276+
}
277+
}
278+
180279
#[cfg(test)]
181280
mod tests {
182281
use super::*;
282+
use tonic::service::Interceptor as _;
183283

184284
fn env_test_exporter() -> (
185285
opentelemetry_sdk::trace::SdkTracerProvider,
@@ -330,4 +430,58 @@ mod tests {
330430
Some("value")
331431
);
332432
}
433+
434+
fn interceptor_with(extractor: EnvTraceContext) -> EnvTraceContextInterceptor {
435+
EnvTraceContextInterceptor {
436+
context: context_from(&extractor),
437+
}
438+
}
439+
440+
#[test]
441+
fn env_interceptor_forwards_valid_traceparent() {
442+
let traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
443+
let mut interceptor = interceptor_with(EnvTraceContext {
444+
traceparent: Some(traceparent.to_string()),
445+
tracestate: Some("vendor=value".to_string()),
446+
});
447+
448+
let request = interceptor.call(tonic::Request::new(())).unwrap();
449+
450+
assert_eq!(
451+
request
452+
.metadata()
453+
.get("traceparent")
454+
.and_then(|value| value.to_str().ok()),
455+
Some(traceparent)
456+
);
457+
assert_eq!(
458+
request
459+
.metadata()
460+
.get("tracestate")
461+
.and_then(|value| value.to_str().ok()),
462+
Some("vendor=value")
463+
);
464+
}
465+
466+
#[test]
467+
fn env_interceptor_is_noop_without_traceparent() {
468+
let mut interceptor = interceptor_with(EnvTraceContext::default());
469+
470+
let request = interceptor.call(tonic::Request::new(())).unwrap();
471+
472+
assert!(request.metadata().get("traceparent").is_none());
473+
assert!(request.metadata().get("tracestate").is_none());
474+
}
475+
476+
#[test]
477+
fn env_interceptor_is_noop_for_invalid_traceparent() {
478+
let mut interceptor = interceptor_with(EnvTraceContext {
479+
traceparent: Some("not-a-valid-traceparent".to_string()),
480+
tracestate: None,
481+
});
482+
483+
let request = interceptor.call(tonic::Request::new(())).unwrap();
484+
485+
assert!(request.metadata().get("traceparent").is_none());
486+
}
333487
}

‎crates/openshell-sdk/src/client.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,16 @@ impl OpenShellClient {
8383
self.channel.clone()
8484
}
8585

86+
/// Clone of the auth interceptor backing [`OpenShellClient::raw_grpc`].
87+
///
88+
/// tonic allows a single interceptor per channel, so callers that need
89+
/// to stack another concern (e.g. trace-context forwarding) on top of
90+
/// authentication can compose around this clone — it shares the same
91+
/// live bearer slot, so OIDC refresh still reaches the composed stack.
92+
pub fn auth_interceptor(&self) -> EdgeAuthInterceptor {
93+
self.interceptor.clone()
94+
}
95+
8696
/// Authenticated gRPC client for the main `OpenShell` service.
8797
///
8898
/// Use this when the curated surface below doesn't expose the RPC or

‎docs/observability/traces.mdx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ Give each installation that reports to the same collector its own name, includin
6969

7070
The gateway continues a valid W3C `traceparent` sent by the caller, so a client's trace can include the gateway's spans for its requests. Without one, the gateway starts a new trace.
7171

72+
The CLI forwards `TRACEPARENT` and `TRACESTATE` from its environment on every gateway request. A CI system that exports these variables (GitHub Actions, GitLab CI, Jenkins OpenTelemetry plugins) therefore parents the gateway's spans for the commands it runs, with no collector or provider configuration on the CI runner.
73+
7274
The gateway also passes its trace context to compute drivers, so driver spans appear in the same trace as the gateway request that caused them. Built-in drivers export their spans through the gateway's process. Managed external drivers receive the trace endpoint, gateway name, and compute driver from the gateway. Drivers that you run yourself own their telemetry configuration.
7375

7476
## Next Steps

0 commit comments

Comments
 (0)