Skip to content

Commit 2acb2ad

Browse files
committed
feat(gateway): identify gateways in exported traces
Attach the configured gateway installation name and compute driver to the gateway OpenTelemetry resource so operators can filter traces from multiple installations that share a collector. Forward the gateway name and OTLP endpoint to managed external drivers so their distinct service resources carry the same installation identity. Keep service.name stable per process type, omit blank resource values, and leave per-span operation names and request attributes unchanged. Refs #2507 Signed-off-by: Kris Hicks <[email protected]>
1 parent 8217814 commit 2acb2ad

9 files changed

Lines changed: 193 additions & 62 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -713,10 +713,12 @@ between a trace and its log lines. Store and compute-driver spans become
713713
children of the request span. Reconciliation, provider refresh, and
714714
driver-watch loops create their own operation spans because they have no
715715
inbound request to provide a parent. gRPC status is recorded when response
716-
trailers arrive.
716+
trailers arrive. Gateway spans carry resource attributes for the gateway
717+
identity and configured compute driver.
717718

718-
The gateway forwards OTLP configuration and W3C trace context to managed
719-
external drivers. Each driver exports under its own service name.
719+
The gateway forwards OTLP configuration, its configured gateway name, and W3C
720+
trace context to managed external drivers. Each driver exports under its own
721+
service name and carries the gateway name as a resource attribute.
720722

721723
Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP
722724
failure stops the gateway from serving: a malformed endpoint is logged at

‎crates/openshell-driver-vm/src/main.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ struct Args {
9191
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
9292
otlp_endpoint: Option<String>,
9393

94+
#[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
95+
gateway_name: Option<String>,
96+
9497
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
9598
openshell_endpoint: Option<String>,
9699

@@ -186,8 +189,10 @@ async fn main() -> Result<()> {
186189
return Ok(());
187190
}
188191

189-
let (tracer_provider, setup_error) =
190-
openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
192+
let (tracer_provider, setup_error) = openshell_driver_vm::otel_tracing::provider_for(
193+
args.otlp_endpoint.as_deref(),
194+
args.gateway_name.as_deref(),
195+
);
191196
tracing_subscriber::registry()
192197
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
193198
.with(tracing_subscriber::fmt::layer())
@@ -691,11 +696,13 @@ mod tests {
691696
}
692697

693698
#[test]
694-
fn accepts_gateway_otlp_endpoint() {
699+
fn accepts_gateway_otlp_configuration() {
695700
let args = Args::try_parse_from([
696701
"openshell-driver-vm",
697702
"--otlp-endpoint",
698703
"http://127.0.0.1:4317",
704+
"--gateway-name",
705+
"production-us-west",
699706
]);
700707
assert!(
701708
args.is_ok(),

‎crates/openshell-driver-vm/src/otel_tracing.rs‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,28 @@ fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'static str) {
8888
}
8989
}
9090

91-
/// Build a tracer provider for the configured OTLP/gRPC endpoint.
91+
/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
9292
#[must_use]
93-
pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
94-
openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
95-
endpoint,
96-
service_name: ServiceName::Fixed(SERVICE_NAME),
97-
service_version: Some(openshell_core::VERSION),
98-
resource_attributes: Vec::new(),
93+
pub fn provider_for(
94+
endpoint: Option<&str>,
95+
gateway_name: Option<&str>,
96+
) -> (Option<SdkTracerProvider>, Option<SetupError>) {
97+
openshell_otel::provider_for(endpoint.map(|endpoint| {
98+
OtlpTraceConfig {
99+
endpoint,
100+
service_name: ServiceName::Fixed(SERVICE_NAME),
101+
service_version: Some(openshell_core::VERSION),
102+
resource_attributes: gateway_name
103+
.map(str::trim)
104+
.filter(|name| !name.is_empty())
105+
.map(|name| {
106+
vec![opentelemetry::KeyValue::new(
107+
"openshell.gateway.name",
108+
name.to_string(),
109+
)]
110+
})
111+
.unwrap_or_default(),
112+
}
99113
}))
100114
}
101115

@@ -122,6 +136,7 @@ mod tests {
122136
struct Received {
123137
spans: Vec<Span>,
124138
service_names: Vec<String>,
139+
gateway_names: Vec<String>,
125140
}
126141

127142
#[derive(Clone)]
@@ -140,18 +155,19 @@ mod tests {
140155
let mut received = self.received.lock().unwrap();
141156
for resource_span in request.into_inner().resource_spans {
142157
if let Some(resource) = resource_span.resource {
143-
received.service_names.extend(
144-
resource
145-
.attributes
146-
.into_iter()
147-
.filter(|attribute| attribute.key == "service.name")
148-
.filter_map(|attribute| attribute.value)
149-
.filter_map(|value| value.value)
150-
.filter_map(|value| match value {
151-
opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value),
152-
_ => None,
153-
}),
154-
);
158+
for attribute in resource.attributes {
159+
let Some(value) = attribute.value.and_then(|value| value.value) else {
160+
continue;
161+
};
162+
let opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) = value else {
163+
continue;
164+
};
165+
match attribute.key.as_str() {
166+
"service.name" => received.service_names.push(value),
167+
"openshell.gateway.name" => received.gateway_names.push(value),
168+
_ => {}
169+
}
170+
}
155171
}
156172
for scope_span in resource_span.scope_spans {
157173
received.spans.extend(scope_span.spans);
@@ -211,7 +227,7 @@ mod tests {
211227
}
212228

213229
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
214-
async fn vm_driver_spans_reach_otlp_collector_with_distinct_service_name() {
230+
async fn vm_driver_spans_reach_otlp_collector_with_resource_identity() {
215231
let received = Arc::new(Mutex::new(Received::default()));
216232
let exported = Arc::new(tokio::sync::Notify::new());
217233
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -233,7 +249,10 @@ mod tests {
233249
.await
234250
});
235251

236-
let (provider, error) = super::provider_for(Some(&format!("http://{address}")));
252+
let (provider, error) = super::provider_for(
253+
Some(&format!("http://{address}")),
254+
Some("production-us-west"),
255+
);
237256
assert!(error.is_none(), "valid OTLP endpoint should configure");
238257
let provider = provider.expect("provider");
239258
let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -272,5 +291,6 @@ mod tests {
272291
"VM spans should use a distinct service name, got {:?}",
273292
received.service_names
274293
);
294+
assert_eq!(received.gateway_names, ["production-us-west"]);
275295
}
276296
}

‎crates/openshell-server/src/cli.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,12 +509,17 @@ async fn run_from_args(
509509
.config_file
510510
.as_ref()
511511
.and_then(|f| f.openshell.gateway.otlp.as_ref());
512+
let gateway_resource = crate::otel_tracing::GatewayResourceAttributes::new(
513+
Some(prepared.config.name.as_str()),
514+
Some(compute_driver.name()),
515+
);
512516
let (tracing_handle, setup_error) = crate::tracing_setup::install(
513517
EnvFilter::try_from_default_env()
514518
.unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)),
515519
&tracing_log_bus,
516520
otlp_config,
517521
crate::tracing_setup::podman_export_enabled(&compute_driver),
522+
gateway_resource,
518523
);
519524

520525
let has_client_ca = prepared

‎crates/openshell-server/src/compute/vm.rs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,7 @@ pub async fn spawn(
478478
.arg("--expected-peer-pid")
479479
.arg(std::process::id().to_string());
480480
command.arg("--log-level").arg(&config.log_level);
481-
append_otlp_args(&mut command, otlp_config);
481+
append_otlp_args(&mut command, otlp_config, &config.name);
482482
command
483483
.arg("--openshell-endpoint")
484484
.arg(&vm_config.grpc_endpoint);
@@ -521,9 +521,10 @@ pub async fn spawn(
521521
}
522522

523523
#[cfg(unix)]
524-
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>) {
524+
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) {
525525
if let Some(config) = otlp_config {
526526
command.arg("--otlp-endpoint").arg(&config.endpoint);
527+
command.arg("--gateway-name").arg(gateway_name);
527528
}
528529
}
529530

@@ -621,22 +622,31 @@ mod tests {
621622
use tempfile::tempdir;
622623

623624
#[test]
624-
fn vm_driver_command_includes_gateway_otlp_endpoint() {
625+
fn vm_driver_command_includes_gateway_otlp_configuration() {
625626
let mut command = tokio::process::Command::new("openshell-driver-vm");
626627
append_otlp_args(
627628
&mut command,
628629
Some(&OtlpConfig {
629630
endpoint: "http://collector.internal:4317".to_string(),
630631
service_name: Some("custom-gateway".to_string()),
631632
}),
633+
"production-us-west",
632634
);
633635

634636
let args = command
635637
.as_std()
636638
.get_args()
637639
.map(|arg| arg.to_string_lossy().into_owned())
638640
.collect::<Vec<_>>();
639-
assert_eq!(args, ["--otlp-endpoint", "http://collector.internal:4317"]);
641+
assert_eq!(
642+
args,
643+
[
644+
"--otlp-endpoint",
645+
"http://collector.internal:4317",
646+
"--gateway-name",
647+
"production-us-west"
648+
]
649+
);
640650
}
641651

642652
#[tokio::test]

‎crates/openshell-server/src/lib.rs‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -579,13 +579,8 @@ pub(crate) async fn run_server(
579579
let sandbox_index = SandboxIndex::new();
580580
let sandbox_watch_bus = SandboxWatchBus::new();
581581
let supervisor_sessions = Arc::new(supervisor_session::SupervisorSessionRegistry::new());
582-
let driver_startup = compute::driver_config::DriverStartupContext {
583-
file: config_file.as_ref(),
584-
guest_tls: guest_tls.as_ref(),
585-
gateway_port: config.bind_address.port(),
586-
gateway_tls_enabled: config.tls.is_some(),
587-
endpoint_overrides: &config.compute_driver_endpoints,
588-
};
582+
let driver_startup =
583+
compute_driver_startup_context(&config, config_file.as_ref(), guest_tls.as_ref());
589584
let (compute, operator_allowlist) = build_compute_runtime(
590585
&config,
591586
driver_startup,
@@ -1596,14 +1591,28 @@ async fn build_compute_runtime(
15961591
Ok((runtime, operator_allowlist))
15971592
}
15981593

1594+
fn compute_driver_startup_context<'a>(
1595+
config: &'a Config,
1596+
config_file: Option<&'a config_file::ConfigFile>,
1597+
guest_tls: Option<&'a compute::driver_config::GuestTlsPaths>,
1598+
) -> compute::driver_config::DriverStartupContext<'a> {
1599+
compute::driver_config::DriverStartupContext {
1600+
file: config_file,
1601+
guest_tls,
1602+
gateway_port: config.bind_address.port(),
1603+
gateway_tls_enabled: config.tls.is_some(),
1604+
endpoint_overrides: &config.compute_driver_endpoints,
1605+
}
1606+
}
1607+
15991608
#[derive(Debug, Clone)]
16001609
pub(crate) enum ConfiguredComputeDriver {
16011610
Registered(ComputeDriverRegistration),
16021611
Remote { name: String },
16031612
}
16041613

16051614
impl ConfiguredComputeDriver {
1606-
fn name(&self) -> &str {
1615+
pub(crate) fn name(&self) -> &str {
16071616
match self {
16081617
Self::Registered(registration) => &registration.name,
16091618
Self::Remote { name } => name,

0 commit comments

Comments
 (0)