Skip to content

Commit bd06460

Browse files
committed
feat(gateway): identify gateways in exported traces
Attach the configured gateway installation name and compute driver to the gateway OpenTelemetry resource so operators can filter traces from multiple installations that share a collector. Forward the gateway name and OTLP endpoint to managed external drivers so their distinct service resources carry the same installation identity. Keep service.name stable per process type, omit blank resource values, and leave per-span operation names and request attributes unchanged. Refs #2507 Signed-off-by: Kris Hicks <[email protected]>
1 parent 8217814 commit bd06460

11 files changed

Lines changed: 255 additions & 87 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -713,10 +713,12 @@ between a trace and its log lines. Store and compute-driver spans become
713713
children of the request span. Reconciliation, provider refresh, and
714714
driver-watch loops create their own operation spans because they have no
715715
inbound request to provide a parent. gRPC status is recorded when response
716-
trailers arrive.
716+
trailers arrive. Gateway spans carry resource attributes for the gateway
717+
identity and configured compute driver.
717718

718-
The gateway forwards OTLP configuration and W3C trace context to managed
719-
external drivers. Each driver exports under its own service name.
719+
The gateway forwards OTLP configuration, its configured gateway name, and W3C
720+
trace context to managed external drivers. Each driver exports under its own
721+
service name and carries the gateway name as a resource attribute.
720722

721723
Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP
722724
failure stops the gateway from serving: a malformed endpoint is logged at

‎crates/openshell-driver-podman/src/main.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ struct Args {
3636
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
3737
otlp_endpoint: Option<String>,
3838

39+
#[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
40+
gateway_name: Option<String>,
41+
3942
/// Path to the Podman API Unix socket.
4043
#[arg(long, env = "OPENSHELL_PODMAN_SOCKET")]
4144
podman_socket: Option<PathBuf>,
@@ -159,8 +162,10 @@ struct Args {
159162
#[tokio::main]
160163
async fn main() -> Result<()> {
161164
let args = Args::parse();
162-
let (tracer_provider, setup_error) =
163-
openshell_driver_podman::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
165+
let (tracer_provider, setup_error) = openshell_driver_podman::otel_tracing::provider_for(
166+
args.otlp_endpoint.as_deref(),
167+
args.gateway_name.as_deref(),
168+
);
164169
tracing_subscriber::registry()
165170
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
166171
.with(tracing_subscriber::fmt::layer())
@@ -272,17 +277,20 @@ mod tests {
272277
}
273278

274279
#[test]
275-
fn accepts_gateway_otlp_endpoint() {
280+
fn accepts_gateway_otlp_configuration() {
276281
let args = Args::try_parse_from([
277282
"openshell-driver-podman",
278283
"--otlp-endpoint",
279284
"http://collector.internal:4317",
285+
"--gateway-name",
286+
"production-us-west",
280287
])
281-
.expect("OTLP endpoint should be accepted");
288+
.expect("OTLP configuration should be accepted");
282289

283290
assert_eq!(
284291
args.otlp_endpoint.as_deref(),
285292
Some("http://collector.internal:4317")
286293
);
294+
assert_eq!(args.gateway_name.as_deref(), Some("production-us-west"));
287295
}
288296
}

‎crates/openshell-driver-podman/src/otel_tracing.rs‎

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -89,13 +89,28 @@ pub(crate) fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'stati
8989
}
9090
}
9191

92+
/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
9293
#[must_use]
93-
pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
94-
openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
95-
endpoint,
96-
service_name: ServiceName::Fixed(SERVICE_NAME),
97-
service_version: Some(openshell_core::VERSION),
98-
resource_attributes: Vec::new(),
94+
pub fn provider_for(
95+
endpoint: Option<&str>,
96+
gateway_name: Option<&str>,
97+
) -> (Option<SdkTracerProvider>, Option<SetupError>) {
98+
openshell_otel::provider_for(endpoint.map(|endpoint| {
99+
OtlpTraceConfig {
100+
endpoint,
101+
service_name: ServiceName::Fixed(SERVICE_NAME),
102+
service_version: Some(openshell_core::VERSION),
103+
resource_attributes: gateway_name
104+
.map(str::trim)
105+
.filter(|name| !name.is_empty())
106+
.map(|name| {
107+
vec![opentelemetry::KeyValue::new(
108+
"openshell.gateway.name",
109+
name.to_string(),
110+
)]
111+
})
112+
.unwrap_or_default(),
113+
}
99114
}))
100115
}
101116

@@ -145,6 +160,7 @@ mod tests {
145160
struct Received {
146161
spans: Vec<Span>,
147162
service_names: Vec<String>,
163+
gateway_names: Vec<String>,
148164
}
149165

150166
#[derive(Clone)]
@@ -162,18 +178,22 @@ mod tests {
162178
let mut received = self.received.lock().unwrap();
163179
for resource_span in request.into_inner().resource_spans {
164180
if let Some(resource) = resource_span.resource {
165-
received.service_names.extend(
166-
resource
167-
.attributes
168-
.into_iter()
169-
.filter(|attribute| attribute.key == "service.name")
170-
.filter_map(|attribute| attribute.value)
171-
.filter_map(|value| value.value)
172-
.filter_map(|value| match value {
173-
opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value),
174-
_ => None,
175-
}),
176-
);
181+
for attribute in resource.attributes {
182+
let Some(value) = attribute.value.and_then(|value| value.value) else {
183+
continue;
184+
};
185+
let opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(
186+
value,
187+
) = value
188+
else {
189+
continue;
190+
};
191+
match attribute.key.as_str() {
192+
"service.name" => received.service_names.push(value),
193+
"openshell.gateway.name" => received.gateway_names.push(value),
194+
_ => {}
195+
}
196+
}
177197
}
178198
for scope_span in resource_span.scope_spans {
179199
received.spans.extend(scope_span.spans);
@@ -241,7 +261,7 @@ mod tests {
241261
}
242262

243263
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
244-
async fn podman_driver_spans_reach_otlp_collector_with_distinct_service_name() {
264+
async fn podman_driver_spans_reach_otlp_collector_with_resource_identity() {
245265
let _tracing_lock = super::test_lock().await;
246266
let received = Arc::new(Mutex::new(Received::default()));
247267
let exported = Arc::new(tokio::sync::Notify::new());
@@ -264,7 +284,10 @@ mod tests {
264284
.await
265285
});
266286

267-
let (provider, error) = super::provider_for(Some(&format!("http://{address}")));
287+
let (provider, error) = super::provider_for(
288+
Some(&format!("http://{address}")),
289+
Some("production-us-west"),
290+
);
268291
assert!(error.is_none());
269292
let provider = provider.expect("provider");
270293
let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -295,5 +318,6 @@ mod tests {
295318
.iter()
296319
.any(|name| name == "openshell-driver-podman")
297320
);
321+
assert_eq!(received.gateway_names, ["production-us-west"]);
298322
}
299323
}

‎crates/openshell-driver-vm/src/main.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ struct Args {
9191
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
9292
otlp_endpoint: Option<String>,
9393

94+
#[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
95+
gateway_name: Option<String>,
96+
9497
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
9598
openshell_endpoint: Option<String>,
9699

@@ -186,8 +189,10 @@ async fn main() -> Result<()> {
186189
return Ok(());
187190
}
188191

189-
let (tracer_provider, setup_error) =
190-
openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
192+
let (tracer_provider, setup_error) = openshell_driver_vm::otel_tracing::provider_for(
193+
args.otlp_endpoint.as_deref(),
194+
args.gateway_name.as_deref(),
195+
);
191196
tracing_subscriber::registry()
192197
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
193198
.with(tracing_subscriber::fmt::layer())
@@ -691,11 +696,13 @@ mod tests {
691696
}
692697

693698
#[test]
694-
fn accepts_gateway_otlp_endpoint() {
699+
fn accepts_gateway_otlp_configuration() {
695700
let args = Args::try_parse_from([
696701
"openshell-driver-vm",
697702
"--otlp-endpoint",
698703
"http://127.0.0.1:4317",
704+
"--gateway-name",
705+
"production-us-west",
699706
]);
700707
assert!(
701708
args.is_ok(),

‎crates/openshell-driver-vm/src/otel_tracing.rs‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,28 @@ fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'static str) {
8888
}
8989
}
9090

91-
/// Build a tracer provider for the configured OTLP/gRPC endpoint.
91+
/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
9292
#[must_use]
93-
pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
94-
openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
95-
endpoint,
96-
service_name: ServiceName::Fixed(SERVICE_NAME),
97-
service_version: Some(openshell_core::VERSION),
98-
resource_attributes: Vec::new(),
93+
pub fn provider_for(
94+
endpoint: Option<&str>,
95+
gateway_name: Option<&str>,
96+
) -> (Option<SdkTracerProvider>, Option<SetupError>) {
97+
openshell_otel::provider_for(endpoint.map(|endpoint| {
98+
OtlpTraceConfig {
99+
endpoint,
100+
service_name: ServiceName::Fixed(SERVICE_NAME),
101+
service_version: Some(openshell_core::VERSION),
102+
resource_attributes: gateway_name
103+
.map(str::trim)
104+
.filter(|name| !name.is_empty())
105+
.map(|name| {
106+
vec![opentelemetry::KeyValue::new(
107+
"openshell.gateway.name",
108+
name.to_string(),
109+
)]
110+
})
111+
.unwrap_or_default(),
112+
}
99113
}))
100114
}
101115

@@ -122,6 +136,7 @@ mod tests {
122136
struct Received {
123137
spans: Vec<Span>,
124138
service_names: Vec<String>,
139+
gateway_names: Vec<String>,
125140
}
126141

127142
#[derive(Clone)]
@@ -140,18 +155,19 @@ mod tests {
140155
let mut received = self.received.lock().unwrap();
141156
for resource_span in request.into_inner().resource_spans {
142157
if let Some(resource) = resource_span.resource {
143-
received.service_names.extend(
144-
resource
145-
.attributes
146-
.into_iter()
147-
.filter(|attribute| attribute.key == "service.name")
148-
.filter_map(|attribute| attribute.value)
149-
.filter_map(|value| value.value)
150-
.filter_map(|value| match value {
151-
opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value),
152-
_ => None,
153-
}),
154-
);
158+
for attribute in resource.attributes {
159+
let Some(value) = attribute.value.and_then(|value| value.value) else {
160+
continue;
161+
};
162+
let opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) = value else {
163+
continue;
164+
};
165+
match attribute.key.as_str() {
166+
"service.name" => received.service_names.push(value),
167+
"openshell.gateway.name" => received.gateway_names.push(value),
168+
_ => {}
169+
}
170+
}
155171
}
156172
for scope_span in resource_span.scope_spans {
157173
received.spans.extend(scope_span.spans);
@@ -211,7 +227,7 @@ mod tests {
211227
}
212228

213229
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
214-
async fn vm_driver_spans_reach_otlp_collector_with_distinct_service_name() {
230+
async fn vm_driver_spans_reach_otlp_collector_with_resource_identity() {
215231
let received = Arc::new(Mutex::new(Received::default()));
216232
let exported = Arc::new(tokio::sync::Notify::new());
217233
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -233,7 +249,10 @@ mod tests {
233249
.await
234250
});
235251

236-
let (provider, error) = super::provider_for(Some(&format!("http://{address}")));
252+
let (provider, error) = super::provider_for(
253+
Some(&format!("http://{address}")),
254+
Some("production-us-west"),
255+
);
237256
assert!(error.is_none(), "valid OTLP endpoint should configure");
238257
let provider = provider.expect("provider");
239258
let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -272,5 +291,6 @@ mod tests {
272291
"VM spans should use a distinct service name, got {:?}",
273292
received.service_names
274293
);
294+
assert_eq!(received.gateway_names, ["production-us-west"]);
275295
}
276296
}

‎crates/openshell-server/src/cli.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,12 +509,17 @@ async fn run_from_args(
509509
.config_file
510510
.as_ref()
511511
.and_then(|f| f.openshell.gateway.otlp.as_ref());
512+
let gateway_resource = crate::otel_tracing::GatewayResourceAttributes::new(
513+
Some(prepared.config.name.as_str()),
514+
Some(compute_driver.name()),
515+
);
512516
let (tracing_handle, setup_error) = crate::tracing_setup::install(
513517
EnvFilter::try_from_default_env()
514518
.unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)),
515519
&tracing_log_bus,
516520
otlp_config,
517521
crate::tracing_setup::podman_export_enabled(&compute_driver),
522+
gateway_resource,
518523
);
519524

520525
let has_client_ca = prepared

‎crates/openshell-server/src/compute/vm.rs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,7 @@ pub async fn spawn(
478478
.arg("--expected-peer-pid")
479479
.arg(std::process::id().to_string());
480480
command.arg("--log-level").arg(&config.log_level);
481-
append_otlp_args(&mut command, otlp_config);
481+
append_otlp_args(&mut command, otlp_config, &config.name);
482482
command
483483
.arg("--openshell-endpoint")
484484
.arg(&vm_config.grpc_endpoint);
@@ -521,9 +521,10 @@ pub async fn spawn(
521521
}
522522

523523
#[cfg(unix)]
524-
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>) {
524+
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) {
525525
if let Some(config) = otlp_config {
526526
command.arg("--otlp-endpoint").arg(&config.endpoint);
527+
command.arg("--gateway-name").arg(gateway_name);
527528
}
528529
}
529530

@@ -621,22 +622,31 @@ mod tests {
621622
use tempfile::tempdir;
622623

623624
#[test]
624-
fn vm_driver_command_includes_gateway_otlp_endpoint() {
625+
fn vm_driver_command_includes_gateway_otlp_configuration() {
625626
let mut command = tokio::process::Command::new("openshell-driver-vm");
626627
append_otlp_args(
627628
&mut command,
628629
Some(&OtlpConfig {
629630
endpoint: "http://collector.internal:4317".to_string(),
630631
service_name: Some("custom-gateway".to_string()),
631632
}),
633+
"production-us-west",
632634
);
633635

634636
let args = command
635637
.as_std()
636638
.get_args()
637639
.map(|arg| arg.to_string_lossy().into_owned())
638640
.collect::<Vec<_>>();
639-
assert_eq!(args, ["--otlp-endpoint", "http://collector.internal:4317"]);
641+
assert_eq!(
642+
args,
643+
[
644+
"--otlp-endpoint",
645+
"http://collector.internal:4317",
646+
"--gateway-name",
647+
"production-us-west"
648+
]
649+
);
640650
}
641651

642652
#[tokio::test]

0 commit comments

Comments
 (0)