Skip to content

Commit 769e26d

Browse files
committed
feat(gateway): identify gateways in exported traces
Attach the configured gateway installation name and compute driver to the gateway OpenTelemetry resource so operators can filter traces from multiple installations that share a collector. Forward the gateway name and OTLP endpoint to managed external drivers so their distinct service resources carry the same installation identity. Keep service.name stable per process type, omit blank resource values, and leave per-span operation names and request attributes unchanged. Refs #2507 Signed-off-by: Kris Hicks <[email protected]>
1 parent 7ff93da commit 769e26d

8 files changed

Lines changed: 176 additions & 54 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -663,10 +663,12 @@ between a trace and its log lines. Store and compute-driver spans become
663663
children of the request span. Reconciliation, provider refresh, and
664664
driver-watch loops create their own operation spans because they have no
665665
inbound request to provide a parent. gRPC status is recorded when response
666-
trailers arrive.
666+
trailers arrive. Gateway spans carry resource attributes for the gateway
667+
identity and configured compute driver.
667668

668-
The gateway forwards OTLP configuration and W3C trace context to managed
669-
external drivers. Each driver exports under its own service name.
669+
The gateway forwards OTLP configuration, its configured gateway name, and W3C
670+
trace context to managed external drivers. Each driver exports under its own
671+
service name and carries the gateway name as a resource attribute.
670672

671673
Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP
672674
failure stops the gateway from serving: a malformed endpoint is logged at

‎crates/openshell-driver-vm/src/main.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ struct Args {
9191
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
9292
otlp_endpoint: Option<String>,
9393

94+
#[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
95+
gateway_name: Option<String>,
96+
9497
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
9598
openshell_endpoint: Option<String>,
9699

@@ -186,8 +189,10 @@ async fn main() -> Result<()> {
186189
return Ok(());
187190
}
188191

189-
let (tracer_provider, setup_error) =
190-
openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
192+
let (tracer_provider, setup_error) = openshell_driver_vm::otel_tracing::provider_for(
193+
args.otlp_endpoint.as_deref(),
194+
args.gateway_name.as_deref(),
195+
);
191196
tracing_subscriber::registry()
192197
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
193198
.with(tracing_subscriber::fmt::layer())
@@ -691,11 +696,13 @@ mod tests {
691696
}
692697

693698
#[test]
694-
fn accepts_gateway_otlp_endpoint() {
699+
fn accepts_gateway_otlp_configuration() {
695700
let args = Args::try_parse_from([
696701
"openshell-driver-vm",
697702
"--otlp-endpoint",
698703
"http://127.0.0.1:4317",
704+
"--gateway-name",
705+
"production-us-west",
699706
]);
700707
assert!(
701708
args.is_ok(),

‎crates/openshell-driver-vm/src/otel_tracing.rs‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,28 @@ fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'static str) {
8080
}
8181
}
8282

83-
/// Build a tracer provider for the configured OTLP/gRPC endpoint.
83+
/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
8484
#[must_use]
85-
pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
86-
openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
87-
endpoint,
88-
service_name: ServiceName::Fixed(SERVICE_NAME),
89-
service_version: Some(openshell_core::VERSION),
90-
resource_attributes: Vec::new(),
85+
pub fn provider_for(
86+
endpoint: Option<&str>,
87+
gateway_name: Option<&str>,
88+
) -> (Option<SdkTracerProvider>, Option<SetupError>) {
89+
openshell_otel::provider_for(endpoint.map(|endpoint| {
90+
OtlpTraceConfig {
91+
endpoint,
92+
service_name: ServiceName::Fixed(SERVICE_NAME),
93+
service_version: Some(openshell_core::VERSION),
94+
resource_attributes: gateway_name
95+
.map(str::trim)
96+
.filter(|name| !name.is_empty())
97+
.map(|name| {
98+
vec![opentelemetry::KeyValue::new(
99+
"openshell.gateway.name",
100+
name.to_string(),
101+
)]
102+
})
103+
.unwrap_or_default(),
104+
}
91105
}))
92106
}
93107

@@ -114,6 +128,7 @@ mod tests {
114128
struct Received {
115129
spans: Vec<Span>,
116130
service_names: Vec<String>,
131+
gateway_names: Vec<String>,
117132
}
118133

119134
#[derive(Clone)]
@@ -132,18 +147,19 @@ mod tests {
132147
let mut received = self.received.lock().unwrap();
133148
for resource_span in request.into_inner().resource_spans {
134149
if let Some(resource) = resource_span.resource {
135-
received.service_names.extend(
136-
resource
137-
.attributes
138-
.into_iter()
139-
.filter(|attribute| attribute.key == "service.name")
140-
.filter_map(|attribute| attribute.value)
141-
.filter_map(|value| value.value)
142-
.filter_map(|value| match value {
143-
opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value),
144-
_ => None,
145-
}),
146-
);
150+
for attribute in resource.attributes {
151+
let Some(value) = attribute.value.and_then(|value| value.value) else {
152+
continue;
153+
};
154+
let opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) = value else {
155+
continue;
156+
};
157+
match attribute.key.as_str() {
158+
"service.name" => received.service_names.push(value),
159+
"openshell.gateway.name" => received.gateway_names.push(value),
160+
_ => {}
161+
}
162+
}
147163
}
148164
for scope_span in resource_span.scope_spans {
149165
received.spans.extend(scope_span.spans);
@@ -202,7 +218,7 @@ mod tests {
202218
}
203219

204220
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
205-
async fn vm_driver_spans_reach_otlp_collector_with_distinct_service_name() {
221+
async fn vm_driver_spans_reach_otlp_collector_with_resource_identity() {
206222
let received = Arc::new(Mutex::new(Received::default()));
207223
let exported = Arc::new(tokio::sync::Notify::new());
208224
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -224,7 +240,10 @@ mod tests {
224240
.await
225241
});
226242

227-
let (provider, error) = super::provider_for(Some(&format!("http://{address}")));
243+
let (provider, error) = super::provider_for(
244+
Some(&format!("http://{address}")),
245+
Some("production-us-west"),
246+
);
228247
assert!(error.is_none(), "valid OTLP endpoint should configure");
229248
let provider = provider.expect("provider");
230249
let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -263,5 +282,6 @@ mod tests {
263282
"VM spans should use a distinct service name, got {:?}",
264283
received.service_names
265284
);
285+
assert_eq!(received.gateway_names, ["production-us-west"]);
266286
}
267287
}

‎crates/openshell-server/src/cli.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,11 +474,16 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> {
474474
.config_file
475475
.as_ref()
476476
.and_then(|f| f.openshell.gateway.otlp.as_ref());
477+
let gateway_resource = crate::otel_tracing::GatewayResourceAttributes::new(
478+
Some(prepared.config.name.as_str()),
479+
prepared.config.compute_drivers.first().map(String::as_str),
480+
);
477481
let (tracing_handle, setup_error) = crate::tracing_setup::install(
478482
EnvFilter::try_from_default_env()
479483
.unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)),
480484
&tracing_log_bus,
481485
otlp_config,
486+
gateway_resource,
482487
);
483488

484489
let has_client_ca = prepared

‎crates/openshell-server/src/compute/vm.rs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,7 @@ pub async fn spawn(
478478
.arg("--expected-peer-pid")
479479
.arg(std::process::id().to_string());
480480
command.arg("--log-level").arg(&config.log_level);
481-
append_otlp_args(&mut command, otlp_config);
481+
append_otlp_args(&mut command, otlp_config, &config.name);
482482
command
483483
.arg("--openshell-endpoint")
484484
.arg(&vm_config.grpc_endpoint);
@@ -521,9 +521,10 @@ pub async fn spawn(
521521
}
522522

523523
#[cfg(unix)]
524-
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>) {
524+
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) {
525525
if let Some(config) = otlp_config {
526526
command.arg("--otlp-endpoint").arg(&config.endpoint);
527+
command.arg("--gateway-name").arg(gateway_name);
527528
}
528529
}
529530

@@ -621,22 +622,31 @@ mod tests {
621622
use tempfile::tempdir;
622623

623624
#[test]
624-
fn vm_driver_command_includes_gateway_otlp_endpoint() {
625+
fn vm_driver_command_includes_gateway_otlp_configuration() {
625626
let mut command = tokio::process::Command::new("openshell-driver-vm");
626627
append_otlp_args(
627628
&mut command,
628629
Some(&OtlpConfig {
629630
endpoint: "http://collector.internal:4317".to_string(),
630631
service_name: Some("custom-gateway".to_string()),
631632
}),
633+
"production-us-west",
632634
);
633635

634636
let args = command
635637
.as_std()
636638
.get_args()
637639
.map(|arg| arg.to_string_lossy().into_owned())
638640
.collect::<Vec<_>>();
639-
assert_eq!(args, ["--otlp-endpoint", "http://collector.internal:4317"]);
641+
assert_eq!(
642+
args,
643+
[
644+
"--otlp-endpoint",
645+
"http://collector.internal:4317",
646+
"--gateway-name",
647+
"production-us-west"
648+
]
649+
);
640650
}
641651

642652
#[tokio::test]

0 commit comments

Comments
 (0)