Skip to content

Commit 43a25db

Browse files
committed
feat(gateway): accept relayed OTLP export data from supervisors
Add the gateway half of the supervisor OTLP telemetry relay, rebuilt on top of the RFC 0012 split-boundary architecture (NVIDIA#2942). Protocol: `SupervisorHello.capabilities` and `SessionAccepted.capabilities` negotiate optional supervisor features; `OtelExportData` is a new `SupervisorMessage` payload carrying OTLP trace bytes and OCSF events. Both repeated fields default to empty and the new oneof case is ignored by older peers, so the change is wire-compatible in both directions. Gateway: `confirm_capabilities` grants `otel_export` only when `[openshell.gateway.otlp]` produced a relay exporter. `handle_otel_export` forwards trace data through a dedicated `OtelRelayExporter` that bypasses the gateway's own tracer provider so supervisor-enriched resource attributes survive, and re-emits bounded, well-formed OCSF events on the `ocsf_relay` target. The capability selection and OCSF acceptance checks are pure functions with unit tests; the exporter has decode and URI tests that need no collector. The public RPC schema inventory grows by one message; the fingerprint and architecture/gateway.md are updated accordingly. The supervisor sends an empty capability list until its relay lands in a later commit. Refs NVIDIA#2641 Signed-off-by: Roland Huß <[email protected]>
1 parent 71c3cd9 commit 43a25db

15 files changed

Lines changed: 492 additions & 8 deletions

File tree

‎Cargo.lock‎

Lines changed: 46 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ tracing-appender = "0.2"
7171
opentelemetry = "0.32"
7272
opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
7373
opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
74+
opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace", "with-serde"] }
7475
tracing-opentelemetry = { version = "0.33", default-features = false, features = ["tracing-log"] }
7576

7677
# Metrics

‎crates/openshell-ocsf/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ pub use builders::{
6464

6565
// --- Tracing layers ---
6666
pub use tracing_layers::{
67-
OCSF_TARGET, OcsfJsonlLayer, OcsfShorthandLayer, clear_current_event, clone_current_event,
68-
emit_ocsf_event, emit_ocsf_event_routed, set_current_event,
67+
OCSF_TARGET, OcsfJsonlLayer, OcsfRelayLayer, OcsfRelaySink, OcsfShorthandLayer,
68+
clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed,
69+
set_current_event,
6970
};

‎crates/openshell-ocsf/src/tracing_layers/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@
99
1010
pub(crate) mod event_bridge;
1111
mod jsonl_layer;
12+
mod relay_layer;
1213
mod shorthand_layer;
1314

1415
pub use event_bridge::{
1516
OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed,
1617
set_current_event,
1718
};
1819
pub use jsonl_layer::OcsfJsonlLayer;
20+
pub use relay_layer::{OcsfRelayLayer, OcsfRelaySink};
1921
pub use shorthand_layer::OcsfShorthandLayer;
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Tracing layer that captures OCSF events and forwards them as JSON bytes
5+
//! through a telemetry buffer sender for relay to the gateway.
6+
7+
use std::sync::Arc;
8+
9+
use tracing::Subscriber;
10+
use tracing_subscriber::Layer;
11+
use tracing_subscriber::layer::Context;
12+
13+
use super::event_bridge::{OCSF_TARGET, clone_current_event};
14+
15+
/// Callback trait for delivering serialized OCSF events.
16+
pub trait OcsfRelaySink: Send + Sync + 'static {
17+
fn send(&self, json_bytes: Vec<u8>);
18+
}
19+
20+
/// A tracing layer that captures OCSF events and serializes them to JSON
21+
/// for relay through the telemetry transport.
22+
pub struct OcsfRelayLayer {
23+
sink: Arc<dyn OcsfRelaySink>,
24+
}
25+
26+
impl OcsfRelayLayer {
27+
pub fn new(sink: Arc<dyn OcsfRelaySink>) -> Self {
28+
Self { sink }
29+
}
30+
}
31+
32+
impl<S: Subscriber> Layer<S> for OcsfRelayLayer {
33+
fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
34+
if event.metadata().target() != OCSF_TARGET {
35+
return;
36+
}
37+
38+
let Some(ocsf_event) = clone_current_event() else {
39+
return;
40+
};
41+
42+
if let Ok(json) = serde_json::to_vec(&ocsf_event) {
43+
self.sink.send(json);
44+
}
45+
}
46+
}

‎crates/openshell-otel/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
66
mod driver;
77
mod grpc;
8-
mod propagation;
8+
pub mod propagation;
99

1010
pub use driver::{
1111
BoxGrpcStream, ComputeDriverTracing, DriverTracingConfig, DriverTracingHandle,

‎crates/openshell-otel/src/propagation.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,22 @@ impl Injector for MetadataMapInjector<'_> {
5454
}
5555
}
5656

57+
/// Writes OpenTelemetry propagation fields to HTTP headers.
58+
#[derive(Debug)]
59+
pub struct HeaderMapInjector<'a>(pub &'a mut HeaderMap);
60+
61+
impl Injector for HeaderMapInjector<'_> {
62+
fn set(&mut self, key: &str, value: String) {
63+
let Ok(key) = key.parse::<http::header::HeaderName>() else {
64+
return;
65+
};
66+
let Ok(value) = value.parse() else {
67+
return;
68+
};
69+
self.0.insert(key, value);
70+
}
71+
}
72+
5773
#[derive(Debug)]
5874
struct TraceContextMapInjector<'a>(&'a mut BTreeMap<String, String>);
5975

@@ -75,6 +91,15 @@ pub fn current_trace_context_carrier() -> Option<BTreeMap<String, String>> {
7591
carrier.contains_key("traceparent").then_some(carrier)
7692
}
7793

94+
/// Inject W3C traceparent into HTTP headers if not already present.
95+
pub fn inject_traceparent_if_missing(headers: &mut HeaderMap) {
96+
if headers.contains_key("traceparent") {
97+
return;
98+
}
99+
let context = tracing::Span::current().context();
100+
TraceContextPropagator::new().inject_context(&context, &mut HeaderMapInjector(headers));
101+
}
102+
78103
/// Injects the active W3C trace context into an outbound tonic request.
79104
#[derive(Debug, Clone, Copy)]
80105
pub struct TraceContextInterceptor;

‎crates/openshell-server/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ tracing-appender = { workspace = true }
7575
# OpenTelemetry (OTLP trace export, opt-in via [openshell.gateway.otlp])
7676
opentelemetry = { workspace = true }
7777
opentelemetry_sdk = { workspace = true }
78+
opentelemetry-proto = { workspace = true }
7879
tracing-opentelemetry = { workspace = true }
7980

8081
# Metrics

‎crates/openshell-server/src/lib.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ mod http;
2828
mod middleware;
2929
mod multiplex;
3030
mod ocsf_log;
31+
pub(crate) mod otel_relay;
3132
mod otel_tracing;
3233
mod pagination;
3334
mod persistence;
@@ -356,6 +357,10 @@ pub struct ServerState {
356357
/// Empty when OIDC is not configured — `authorize_workspace()` treats
357358
/// every authenticated user as Platform Admin in that case.
358359
pub admin_role: String,
360+
361+
/// Dedicated OTLP exporter for relayed telemetry from supervisors.
362+
/// `None` when the gateway has no OTLP endpoint configured.
363+
pub otel_relay_exporter: Option<Arc<otel_relay::OtelRelayExporter>>,
359364
}
360365

361366
fn is_benign_tls_handshake_failure(error: &std::io::Error) -> bool {
@@ -450,6 +455,7 @@ impl ServerState {
450455
provider_profile_sources:
451456
provider_profile_sources::ProviderProfileSources::with_default_sources(),
452457
admin_role,
458+
otel_relay_exporter: None,
453459
}
454460
}
455461
}
@@ -698,6 +704,7 @@ pub(crate) async fn run_server(
698704
state.gateway_interceptors = gateway_interceptors;
699705
state.provider_profile_sources = provider_profile_sources;
700706
state.extension_jwt_issuer = extension_jwt_issuer.clone();
707+
state.otel_relay_exporter = otel_relay::try_create_exporter(config_file.as_ref()).await;
701708
state.sandbox_session_jwt_authority = sandbox_session_jwt_authority;
702709
if let Some(issuer) = extension_jwt_issuer {
703710
spawn_gateway_extension_token_refresh(issuer, gateway_extension_credentials);
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Dedicated OTLP exporter for relayed telemetry from supervisors.
5+
//!
6+
//! Uses a separate gRPC client to forward pre-enriched trace data to the
7+
//! configured OTLP collector, bypassing the gateway's own `SdkTracerProvider`
8+
//! which would overwrite resource attributes.
9+
10+
use std::sync::Arc;
11+
12+
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
13+
use prost::Message;
14+
use tonic::transport::Channel;
15+
use tracing::{debug, info};
16+
17+
use opentelemetry_proto::tonic::collector::trace::v1::trace_service_client::TraceServiceClient;
18+
19+
/// Exporter that forwards raw protobuf-encoded trace data to an OTLP collector.
20+
#[derive(Debug, Clone)]
21+
pub struct OtelRelayExporter {
22+
client: TraceServiceClient<Channel>,
23+
}
24+
25+
impl OtelRelayExporter {
26+
/// Connect to the OTLP collector at the given gRPC endpoint.
27+
pub async fn connect(endpoint: &str) -> Result<Self, ConnectError> {
28+
let channel = Channel::from_shared(endpoint.to_string())
29+
.map_err(|e| ConnectError::InvalidUri(e.to_string()))?
30+
.connect()
31+
.await
32+
.map_err(ConnectError::Transport)?;
33+
Ok(Self {
34+
client: TraceServiceClient::new(channel),
35+
})
36+
}
37+
38+
/// Export raw protobuf-encoded `ExportTraceServiceRequest` bytes.
39+
pub async fn export_raw(&self, trace_data: Vec<u8>) -> Result<(), ExportError> {
40+
let request = ExportTraceServiceRequest::decode(trace_data.as_slice())
41+
.map_err(ExportError::Decode)?;
42+
43+
let mut client = self.client.clone();
44+
client
45+
.export(tonic::Request::new(request))
46+
.await
47+
.map_err(ExportError::Grpc)?;
48+
49+
Ok(())
50+
}
51+
}
52+
53+
#[derive(Debug, thiserror::Error)]
54+
pub enum ExportError {
55+
#[error("failed to decode trace data: {0}")]
56+
Decode(prost::DecodeError),
57+
#[error("gRPC export failed: {0}")]
58+
Grpc(tonic::Status),
59+
}
60+
61+
#[derive(Debug, thiserror::Error)]
62+
pub enum ConnectError {
63+
#[error("invalid OTLP endpoint URI: {0}")]
64+
InvalidUri(String),
65+
#[error("transport error: {0}")]
66+
Transport(tonic::transport::Error),
67+
}
68+
69+
/// Create a relay exporter from the gateway's OTLP config, if configured.
70+
pub async fn try_create_exporter(
71+
config_file: Option<&crate::config_file::ConfigFile>,
72+
) -> Option<Arc<OtelRelayExporter>> {
73+
let Some(cf) = config_file else {
74+
debug!("no config file; OTEL relay disabled");
75+
return None;
76+
};
77+
let Some(otlp) = cf.openshell.gateway.otlp.as_ref() else {
78+
debug!("no [openshell.gateway.otlp] section in config; OTEL relay disabled");
79+
return None;
80+
};
81+
match OtelRelayExporter::connect(&otlp.endpoint).await {
82+
Ok(exporter) => {
83+
info!(endpoint = %otlp.endpoint, "OTEL relay exporter connected");
84+
Some(Arc::new(exporter))
85+
}
86+
Err(e) => {
87+
tracing::warn!(
88+
endpoint = %otlp.endpoint,
89+
error = %e,
90+
"failed to connect OTEL relay exporter; relay disabled"
91+
);
92+
None
93+
}
94+
}
95+
}
96+
97+
#[cfg(test)]
98+
mod tests {
99+
use super::*;
100+
101+
#[tokio::test]
102+
async fn connect_rejects_invalid_endpoint_uri() {
103+
let err = OtelRelayExporter::connect("not a valid uri")
104+
.await
105+
.expect_err("an unparsable endpoint must not connect");
106+
assert!(matches!(err, ConnectError::InvalidUri(_)), "{err}");
107+
}
108+
109+
#[tokio::test]
110+
async fn export_raw_rejects_undecodable_trace_bytes() {
111+
// A lazy channel performs no I/O, so the decode failure surfaces
112+
// before any RPC is attempted and no collector is needed.
113+
let channel = Channel::from_static("http://127.0.0.1:1").connect_lazy();
114+
let exporter = OtelRelayExporter {
115+
client: TraceServiceClient::new(channel),
116+
};
117+
let err = exporter
118+
.export_raw(vec![0xff, 0xff, 0xff])
119+
.await
120+
.expect_err("garbage bytes must not decode as ExportTraceServiceRequest");
121+
assert!(matches!(err, ExportError::Decode(_)), "{err}");
122+
}
123+
}

0 commit comments

Comments
 (0)