Skip to content

Commit e94b9ae

Browse files
feat(tracing): add datastore tracing plumbing (#2760)
Configure a dedicated datastore tracing target on the OTLP layer while preserving explicit logging filters and avoiding span overhead when OTLP is disabled. This is in preparation for adding trace spans for datastores used in Buzz ## Update — 2026-07-27 - Export HTTP requests as `INFO` server spans under `buzz_relay`, preserving request parentage for datastore spans. - Configure OTEL span filtering independently with `BUZZ_OTEL_FILTER`, so `RUST_LOG` changes cannot break trace topology. - Verify exported HTTP and datastore spans share a trace ID and have the expected parent/child relationship. Co-authored-by: Amp <[email protected]>
1 parent 925a9a7 commit e94b9ae

5 files changed

Lines changed: 152 additions & 7 deletions

File tree

‎.env.example‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,10 @@ RELAY_URL=ws://localhost:3000
105105
# -----------------------------------------------------------------------------
106106
# Logging / Tracing
107107
# -----------------------------------------------------------------------------
108-
RUST_LOG=buzz_relay=debug,buzz_db=debug,buzz_auth=debug,buzz_pubsub=debug,tower_http=debug
108+
RUST_LOG=buzz_relay=debug,buzz_datastore=info,buzz_db=debug,buzz_auth=debug,buzz_pubsub=debug,tower_http=debug
109+
# Optional OpenTelemetry-only target filter. This is deliberately independent
110+
# from RUST_LOG so log verbosity changes cannot break trace parentage.
111+
# BUZZ_OTEL_FILTER=buzz_relay=info,buzz_datastore=info
109112

110113
# OTLP tracing endpoint (optional — leave unset to disable)
111114
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317

‎crates/buzz-relay/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,4 @@ reqwest = { workspace = true }
9292
tokio-tungstenite = { workspace = true }
9393
futures = "0.3"
9494
flate2 = "1.1.9"
95+
opentelemetry_sdk = { workspace = true, features = ["testing"] }

‎crates/buzz-relay/src/main.rs‎

Lines changed: 61 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ use std::sync::Arc;
44

55
use tracing::{error, info, warn};
66
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
7+
8+
fn log_env_filter(rust_log: Option<&str>) -> EnvFilter {
9+
EnvFilter::new(rust_log.unwrap_or("buzz_relay=info"))
10+
}
711
use uuid::Uuid;
812

913
use buzz_audit::AuditService;
@@ -107,9 +111,17 @@ async fn main() -> anyhow::Result<()> {
107111
};
108112

109113
tracing_subscriber::registry()
110-
.with(fmt::layer().json().flatten_event(true))
111-
.with(EnvFilter::from_default_env().add_directive("buzz_relay=info".parse()?))
112-
.with(otel_layer)
114+
.with(
115+
fmt::layer()
116+
.json()
117+
.flatten_event(true)
118+
.with_filter(log_env_filter(std::env::var("RUST_LOG").ok().as_deref())),
119+
)
120+
.with(otel_layer.map(|layer| {
121+
layer.with_filter(telemetry::otel_env_filter(
122+
std::env::var("BUZZ_OTEL_FILTER").ok().as_deref(),
123+
))
124+
}))
113125
.init();
114126

115127
// Log any exporter-build failure now that the subscriber is installed.
@@ -1060,6 +1072,52 @@ async fn main() -> anyhow::Result<()> {
10601072
Ok(())
10611073
}
10621074

1075+
#[cfg(test)]
1076+
mod env_filter_tests {
1077+
use super::log_env_filter;
1078+
use buzz_relay::telemetry::otel_env_filter;
1079+
use tracing_subscriber::prelude::*;
1080+
1081+
#[test]
1082+
fn unset_enables_datastore_only_for_otel_filter() {
1083+
let logs = tracing_subscriber::registry().with(log_env_filter(None));
1084+
tracing::subscriber::with_default(logs, || {
1085+
assert!(!tracing::enabled!(target: "buzz_datastore", tracing::Level::INFO));
1086+
assert!(tracing::enabled!(target: "buzz_relay", tracing::Level::INFO));
1087+
});
1088+
1089+
let otel = tracing_subscriber::registry().with(otel_env_filter(None));
1090+
tracing::subscriber::with_default(otel, || {
1091+
assert!(tracing::enabled!(target: "buzz_datastore", tracing::Level::INFO));
1092+
});
1093+
}
1094+
1095+
#[test]
1096+
fn explicit_datastore_off_is_preserved_alone() {
1097+
assert_eq!(
1098+
otel_env_filter(Some("buzz_datastore=off")).to_string(),
1099+
"buzz_datastore=off"
1100+
);
1101+
}
1102+
1103+
#[test]
1104+
fn explicit_datastore_debug_is_preserved_alone() {
1105+
assert_eq!(
1106+
otel_env_filter(Some("buzz_datastore=debug")).to_string(),
1107+
"buzz_datastore=debug"
1108+
);
1109+
}
1110+
1111+
#[test]
1112+
fn log_and_otel_filters_are_configured_independently() {
1113+
assert_eq!(log_env_filter(Some("warn")).to_string(), "warn");
1114+
assert_eq!(
1115+
otel_env_filter(Some("buzz_relay=debug")).to_string(),
1116+
"buzz_relay=debug"
1117+
);
1118+
}
1119+
}
1120+
10631121
async fn run_community_revalidator(
10641122
state: Arc<AppState>,
10651123
period: std::time::Duration,

‎crates/buzz-relay/src/router.rs‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ use std::sync::atomic::Ordering;
44
use std::sync::Arc;
55

66
use axum::{
7+
body::Body,
78
extract::{ConnectInfo, FromRequest, State, WebSocketUpgrade},
8-
http::{HeaderMap, StatusCode},
9+
http::{HeaderMap, Request, StatusCode},
910
middleware,
1011
response::{IntoResponse, Json},
1112
routing::{get, post, put},
@@ -16,7 +17,7 @@ use tower::ServiceExt;
1617
use tower_http::cors::{AllowOrigin, CorsLayer};
1718
use tower_http::limit::RequestBodyLimitLayer;
1819
use tower_http::services::ServeDir;
19-
use tower_http::trace::TraceLayer;
20+
use tower_http::trace::{HttpMakeClassifier, TraceLayer};
2021

2122
use crate::api;
2223
use crate::audio;
@@ -187,10 +188,23 @@ pub fn build_router(state: Arc<AppState>) -> Router {
187188

188189
merged
189190
.layer(middleware::from_fn(track_metrics))
190-
.layer(TraceLayer::new_for_http())
191+
.layer(http_trace_layer())
191192
.layer(build_cors_layer(&state.config.cors_origins))
192193
}
193194

195+
fn http_trace_layer() -> TraceLayer<HttpMakeClassifier, fn(&Request<Body>) -> tracing::Span> {
196+
TraceLayer::new_for_http().make_span_with(make_http_span as fn(&Request<Body>) -> tracing::Span)
197+
}
198+
199+
fn make_http_span(request: &Request<Body>) -> tracing::Span {
200+
tracing::info_span!(
201+
target: "buzz_relay",
202+
"http.request",
203+
otel.kind = "server",
204+
http.request.method = %request.method(),
205+
)
206+
}
207+
194208
fn is_admin_spa_path(path: &str) -> bool {
195209
path == "/"
196210
|| path == "/reports"
@@ -435,9 +449,14 @@ fn build_cors_layer(cors_origins: &[String]) -> CorsLayer {
435449
mod tests {
436450
use axum::{routing::get, Router};
437451
use futures_util::SinkExt;
452+
use opentelemetry::trace::TracerProvider as _;
453+
use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
438454
use tokio::net::TcpListener;
439455
use tokio::sync::mpsc;
440456
use tokio_tungstenite::{connect_async, tungstenite::Message};
457+
use tower::ServiceBuilder;
458+
use tracing::Instrument as _;
459+
use tracing_subscriber::prelude::*;
441460

442461
use super::*;
443462

@@ -471,6 +490,60 @@ mod tests {
471490
assert!(!should_serve_spa("/arbitrary", true));
472491
}
473492

493+
#[tokio::test(flavor = "current_thread")]
494+
async fn http_and_datastore_spans_are_exported_in_the_same_trace() {
495+
let exporter = InMemorySpanExporter::default();
496+
let provider = SdkTracerProvider::builder()
497+
.with_simple_exporter(exporter.clone())
498+
.build();
499+
let subscriber = tracing_subscriber::registry().with(
500+
tracing_opentelemetry::layer()
501+
.with_tracer(provider.tracer("test"))
502+
.with_filter(crate::telemetry::otel_env_filter(None)),
503+
);
504+
let _subscriber_guard = tracing::subscriber::set_default(subscriber);
505+
let service = ServiceBuilder::new()
506+
.layer(http_trace_layer())
507+
.service(tower::service_fn(
508+
|_: axum::http::Request<axum::body::Body>| async {
509+
async {}
510+
.instrument(tracing::info_span!(
511+
target: "buzz_datastore",
512+
"SELECT",
513+
otel.kind = "client",
514+
db.system.name = "postgresql",
515+
))
516+
.await;
517+
Ok::<_, std::convert::Infallible>(axum::response::Response::new(
518+
axum::body::Body::empty(),
519+
))
520+
},
521+
));
522+
523+
service
524+
.oneshot(
525+
axum::http::Request::get("/")
526+
.body(axum::body::Body::empty())
527+
.unwrap(),
528+
)
529+
.await
530+
.unwrap();
531+
532+
provider.force_flush().unwrap();
533+
let spans = exporter.get_finished_spans().unwrap();
534+
let http = spans
535+
.iter()
536+
.find(|span| span.name == "http.request")
537+
.unwrap();
538+
let datastore = spans.iter().find(|span| span.name == "SELECT").unwrap();
539+
540+
assert_eq!(
541+
datastore.span_context.trace_id(),
542+
http.span_context.trace_id()
543+
);
544+
assert_eq!(datastore.parent_span_id, http.span_context.span_id());
545+
}
546+
474547
async fn handler_receives_message_with_limit(limit: usize, size: usize) -> bool {
475548
let (received_tx, mut received_rx) = mpsc::unbounded_channel();
476549
let app = Router::new().route(

‎crates/buzz-relay/src/telemetry.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,16 @@
2525
2626
use opentelemetry_otlp::ExporterBuildError;
2727
use opentelemetry_sdk::{resource::EnvResourceDetector, trace::SdkTracerProvider, Resource};
28+
use tracing_subscriber::EnvFilter;
29+
30+
/// Build the filter for spans exported through OpenTelemetry.
31+
///
32+
/// This is intentionally independent from `RUST_LOG`: changing stdout log
33+
/// verbosity must not remove parent spans from exported traces. Set
34+
/// `BUZZ_OTEL_FILTER` to override the default targets.
35+
pub fn otel_env_filter(configured: Option<&str>) -> EnvFilter {
36+
EnvFilter::new(configured.unwrap_or("buzz_relay=info,buzz_datastore=info"))
37+
}
2838

2939
/// Build the OTEL [`Resource`] used by the trace provider.
3040
///

0 commit comments

Comments
 (0)