Skip to content

Commit fcc0678

Browse files
committed
feat(gateway): identify gateways in exported traces
Attach the configured gateway installation name and compute driver to the gateway OpenTelemetry resource so operators can filter traces from multiple installations that share a collector. Forward the gateway name and OTLP endpoint to managed external drivers so their distinct service resources carry the same installation identity. Keep service.name stable per process type, omit blank resource values, and leave per-span operation names and request attributes unchanged. Refs #2507 Signed-off-by: Kris Hicks <[email protected]>
1 parent 89e022e commit fcc0678

9 files changed

Lines changed: 218 additions & 63 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -696,10 +696,12 @@ between a trace and its log lines. Store and compute-driver spans become
696696
children of the request span. Reconciliation, provider refresh, and
697697
driver-watch loops create their own operation spans because they have no
698698
inbound request to provide a parent. gRPC status is recorded when response
699-
trailers arrive.
699+
trailers arrive. Gateway spans carry resource attributes for the gateway
700+
identity and configured compute driver.
700701

701-
The gateway forwards OTLP configuration and W3C trace context to managed
702-
external drivers. Each driver exports under its own service name.
702+
The gateway forwards OTLP configuration, its configured gateway name, and W3C
703+
trace context to managed external drivers. Each driver exports under its own
704+
service name and carries the gateway name as a resource attribute.
703705

704706
Two invariants shape the failure behavior. Telemetry is diagnostic, so no OTLP
705707
failure stops the gateway from serving: a malformed endpoint is logged at

‎crates/openshell-driver-vm/src/main.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ struct Args {
9191
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
9292
otlp_endpoint: Option<String>,
9393

94+
#[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
95+
gateway_name: Option<String>,
96+
9497
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
9598
openshell_endpoint: Option<String>,
9699

@@ -186,8 +189,10 @@ async fn main() -> Result<()> {
186189
return Ok(());
187190
}
188191

189-
let (tracer_provider, setup_error) =
190-
openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
192+
let (tracer_provider, setup_error) = openshell_driver_vm::otel_tracing::provider_for(
193+
args.otlp_endpoint.as_deref(),
194+
args.gateway_name.as_deref(),
195+
);
191196
tracing_subscriber::registry()
192197
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
193198
.with(tracing_subscriber::fmt::layer())
@@ -691,11 +696,13 @@ mod tests {
691696
}
692697

693698
#[test]
694-
fn accepts_gateway_otlp_endpoint() {
699+
fn accepts_gateway_otlp_configuration() {
695700
let args = Args::try_parse_from([
696701
"openshell-driver-vm",
697702
"--otlp-endpoint",
698703
"http://127.0.0.1:4317",
704+
"--gateway-name",
705+
"production-us-west",
699706
]);
700707
assert!(
701708
args.is_ok(),

‎crates/openshell-driver-vm/src/otel_tracing.rs‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,28 @@ fn compute_driver_rpc_operation(path: &str) -> (&'static str, &'static str) {
8181
}
8282
}
8383

84-
/// Build a tracer provider for the configured OTLP/gRPC endpoint.
84+
/// Build a tracer provider for the configured OTLP/gRPC endpoint and gateway.
8585
#[must_use]
86-
pub fn provider_for(endpoint: Option<&str>) -> (Option<SdkTracerProvider>, Option<SetupError>) {
87-
openshell_otel::provider_for(endpoint.map(|endpoint| OtlpTraceConfig {
88-
endpoint,
89-
service_name: ServiceName::Fixed(SERVICE_NAME),
90-
service_version: Some(openshell_core::VERSION),
91-
resource_attributes: Vec::new(),
86+
pub fn provider_for(
87+
endpoint: Option<&str>,
88+
gateway_name: Option<&str>,
89+
) -> (Option<SdkTracerProvider>, Option<SetupError>) {
90+
openshell_otel::provider_for(endpoint.map(|endpoint| {
91+
OtlpTraceConfig {
92+
endpoint,
93+
service_name: ServiceName::Fixed(SERVICE_NAME),
94+
service_version: Some(openshell_core::VERSION),
95+
resource_attributes: gateway_name
96+
.map(str::trim)
97+
.filter(|name| !name.is_empty())
98+
.map(|name| {
99+
vec![opentelemetry::KeyValue::new(
100+
"openshell.gateway.name",
101+
name.to_string(),
102+
)]
103+
})
104+
.unwrap_or_default(),
105+
}
92106
}))
93107
}
94108

@@ -115,6 +129,7 @@ mod tests {
115129
struct Received {
116130
spans: Vec<Span>,
117131
service_names: Vec<String>,
132+
gateway_names: Vec<String>,
118133
}
119134

120135
#[derive(Clone)]
@@ -133,18 +148,19 @@ mod tests {
133148
let mut received = self.received.lock().unwrap();
134149
for resource_span in request.into_inner().resource_spans {
135150
if let Some(resource) = resource_span.resource {
136-
received.service_names.extend(
137-
resource
138-
.attributes
139-
.into_iter()
140-
.filter(|attribute| attribute.key == "service.name")
141-
.filter_map(|attribute| attribute.value)
142-
.filter_map(|value| value.value)
143-
.filter_map(|value| match value {
144-
opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) => Some(value),
145-
_ => None,
146-
}),
147-
);
151+
for attribute in resource.attributes {
152+
let Some(value) = attribute.value.and_then(|value| value.value) else {
153+
continue;
154+
};
155+
let opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value) = value else {
156+
continue;
157+
};
158+
match attribute.key.as_str() {
159+
"service.name" => received.service_names.push(value),
160+
"openshell.gateway.name" => received.gateway_names.push(value),
161+
_ => {}
162+
}
163+
}
148164
}
149165
for scope_span in resource_span.scope_spans {
150166
received.spans.extend(scope_span.spans);
@@ -204,7 +220,7 @@ mod tests {
204220
}
205221

206222
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
207-
async fn vm_driver_spans_reach_otlp_collector_with_distinct_service_name() {
223+
async fn vm_driver_spans_reach_otlp_collector_with_resource_identity() {
208224
let received = Arc::new(Mutex::new(Received::default()));
209225
let exported = Arc::new(tokio::sync::Notify::new());
210226
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -226,7 +242,10 @@ mod tests {
226242
.await
227243
});
228244

229-
let (provider, error) = super::provider_for(Some(&format!("http://{address}")));
245+
let (provider, error) = super::provider_for(
246+
Some(&format!("http://{address}")),
247+
Some("production-us-west"),
248+
);
230249
assert!(error.is_none(), "valid OTLP endpoint should configure");
231250
let provider = provider.expect("provider");
232251
let subscriber = tracing_subscriber::registry().with(super::layer(&provider));
@@ -265,5 +284,6 @@ mod tests {
265284
"VM spans should use a distinct service name, got {:?}",
266285
received.service_names
267286
);
287+
assert_eq!(received.gateway_names, ["production-us-west"]);
268288
}
269289
}

‎crates/openshell-server/src/cli.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@ use crate::certgen;
1717
use crate::compute::driver_config::GuestTlsPaths;
1818
use crate::config_file::{self, ConfigFile, GatewayFileSection};
1919
use crate::defaults::{self, LocalTlsPaths};
20-
use crate::{ServerStartupConfig, run_server, tracing_bus::TracingLogBus};
20+
use crate::{
21+
ServerStartupConfig, configured_compute_driver_name, run_server, tracing_bus::TracingLogBus,
22+
};
2123

2224
/// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing.
2325
///
@@ -490,11 +492,17 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> {
490492
.config_file
491493
.as_ref()
492494
.and_then(|f| f.openshell.gateway.otlp.as_ref());
495+
let compute_driver_name = configured_compute_driver_name(&prepared)?;
496+
let gateway_resource = crate::otel_tracing::GatewayResourceAttributes::new(
497+
Some(prepared.config.name.as_str()),
498+
Some(compute_driver_name.as_str()),
499+
);
493500
let (tracing_handle, setup_error) = crate::tracing_setup::install(
494501
EnvFilter::try_from_default_env()
495502
.unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)),
496503
&tracing_log_bus,
497504
otlp_config,
505+
gateway_resource,
498506
);
499507

500508
let has_client_ca = prepared

‎crates/openshell-server/src/compute/vm.rs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -478,7 +478,7 @@ pub async fn spawn(
478478
.arg("--expected-peer-pid")
479479
.arg(std::process::id().to_string());
480480
command.arg("--log-level").arg(&config.log_level);
481-
append_otlp_args(&mut command, otlp_config);
481+
append_otlp_args(&mut command, otlp_config, &config.name);
482482
command
483483
.arg("--openshell-endpoint")
484484
.arg(&vm_config.grpc_endpoint);
@@ -521,9 +521,10 @@ pub async fn spawn(
521521
}
522522

523523
#[cfg(unix)]
524-
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>) {
524+
fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) {
525525
if let Some(config) = otlp_config {
526526
command.arg("--otlp-endpoint").arg(&config.endpoint);
527+
command.arg("--gateway-name").arg(gateway_name);
527528
}
528529
}
529530

@@ -621,22 +622,31 @@ mod tests {
621622
use tempfile::tempdir;
622623

623624
#[test]
624-
fn vm_driver_command_includes_gateway_otlp_endpoint() {
625+
fn vm_driver_command_includes_gateway_otlp_configuration() {
625626
let mut command = tokio::process::Command::new("openshell-driver-vm");
626627
append_otlp_args(
627628
&mut command,
628629
Some(&OtlpConfig {
629630
endpoint: "http://collector.internal:4317".to_string(),
630631
service_name: Some("custom-gateway".to_string()),
631632
}),
633+
"production-us-west",
632634
);
633635

634636
let args = command
635637
.as_std()
636638
.get_args()
637639
.map(|arg| arg.to_string_lossy().into_owned())
638640
.collect::<Vec<_>>();
639-
assert_eq!(args, ["--otlp-endpoint", "http://collector.internal:4317"]);
641+
assert_eq!(
642+
args,
643+
[
644+
"--otlp-endpoint",
645+
"http://collector.internal:4317",
646+
"--gateway-name",
647+
"production-us-west"
648+
]
649+
);
640650
}
641651

642652
#[tokio::test]

‎crates/openshell-server/src/lib.rs‎

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -583,13 +583,8 @@ pub(crate) async fn run_server(
583583
let sandbox_index = SandboxIndex::new();
584584
let sandbox_watch_bus = SandboxWatchBus::new();
585585
let supervisor_sessions = Arc::new(supervisor_session::SupervisorSessionRegistry::new());
586-
let driver_startup = compute::driver_config::DriverStartupContext {
587-
file: config_file.as_ref(),
588-
guest_tls: guest_tls.as_ref(),
589-
gateway_port: config.bind_address.port(),
590-
gateway_tls_enabled: config.tls.is_some(),
591-
endpoint_overrides: &config.compute_driver_endpoints,
592-
};
586+
let driver_startup =
587+
compute_driver_startup_context(&config, config_file.as_ref(), guest_tls.as_ref());
593588
let (compute, operator_allowlist) = build_compute_runtime(
594589
&config,
595590
driver_startup,
@@ -1205,6 +1200,30 @@ async fn build_compute_runtime(
12051200
Ok((runtime, operator_allowlist))
12061201
}
12071202

1203+
pub(crate) fn configured_compute_driver_name(startup: &ServerStartupConfig) -> Result<String> {
1204+
let driver_startup = compute_driver_startup_context(
1205+
&startup.config,
1206+
startup.config_file.as_ref(),
1207+
startup.guest_tls.as_ref(),
1208+
);
1209+
configured_compute_driver(&startup.config, driver_startup)
1210+
.map(|driver| driver.name().to_string())
1211+
}
1212+
1213+
fn compute_driver_startup_context<'a>(
1214+
config: &'a Config,
1215+
config_file: Option<&'a config_file::ConfigFile>,
1216+
guest_tls: Option<&'a compute::driver_config::GuestTlsPaths>,
1217+
) -> compute::driver_config::DriverStartupContext<'a> {
1218+
compute::driver_config::DriverStartupContext {
1219+
file: config_file,
1220+
guest_tls,
1221+
gateway_port: config.bind_address.port(),
1222+
gateway_tls_enabled: config.tls.is_some(),
1223+
endpoint_overrides: &config.compute_driver_endpoints,
1224+
}
1225+
}
1226+
12081227
#[derive(Debug, Clone)]
12091228
enum ConfiguredComputeDriver {
12101229
Builtin(ComputeDriverKind),
@@ -1347,7 +1366,7 @@ mod tests {
13471366
BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind,
13481367
GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor,
13491368
allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes,
1350-
configured_compute_driver, is_benign_tls_handshake_failure,
1369+
configured_compute_driver, configured_compute_driver_name, is_benign_tls_handshake_failure,
13511370
kubernetes_sandbox_jwt_expiry_disabled, mint_gateway_extension_credential,
13521371
serve_gateway_listener,
13531372
};
@@ -1860,6 +1879,17 @@ mod tests {
18601879
));
18611880
}
18621881

1882+
#[test]
1883+
fn configured_compute_driver_name_uses_effective_driver_resolution() {
1884+
let startup = crate::ServerStartupConfig {
1885+
config: Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]),
1886+
config_file: None,
1887+
guest_tls: None,
1888+
};
1889+
1890+
assert_eq!(configured_compute_driver_name(&startup).unwrap(), "podman");
1891+
}
1892+
18631893
#[test]
18641894
fn configured_compute_driver_resolves_named_remote() {
18651895
let config = Config::new(None).with_compute_drivers(["kyma"]);

0 commit comments

Comments
 (0)