Skip to content

Commit 7caff12

Browse files
authored
perf(otel): stop exporting spans from steady-state polling (NVIDIA#3915)
Store operation spans and request spans for supervisor-polled RPCs (GetSandboxConfig, ReportProviderReadiness) use DEBUG level, so the default INFO filter no longer exports them. The provider credential refresh worker opens its span only when a state has work. Refs NVIDIA#2698 Signed-off-by: Kris Hicks <[email protected]>
1 parent 21fea95 commit 7caff12

4 files changed

Lines changed: 130 additions & 42 deletions

File tree

‎crates/openshell-server/src/multiplex.rs‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,16 @@ impl MakeRequestId for UuidRequestId {
6363
}
6464
}
6565

66+
/// Paths called on a timer rather than by someone waiting on the result.
67+
const POLLED_PATHS: &[&str] = &[
68+
"/health",
69+
"/healthz",
70+
"/readyz",
71+
"/openshell.v1.OpenShell/GetSandboxConfig",
72+
"/openshell.v1.OpenShell/ReportProviderReadiness",
73+
"/openshell.v1.OpenShell/PeerReportProviderReadiness",
74+
];
75+
6676
/// Build a tracing span for an inbound request, recording the `request_id`
6777
/// header (set by [`UuidRequestId`] or supplied by the client).
6878
fn make_request_span<B>(req: &Request<B>) -> Span {
@@ -79,7 +89,7 @@ fn make_request_span<B>(req: &Request<B>) -> Span {
7989
// the callsite name.
8090
let otel_name = otel_span_name(req.method(), path);
8191

82-
let span = if matches!(path, "/health" | "/healthz" | "/readyz") {
92+
let span = if POLLED_PATHS.contains(&path) {
8393
tracing::debug_span!(
8494
"request",
8595
method = %req.method(),
@@ -2223,6 +2233,34 @@ mod tests {
22232233
);
22242234
}
22252235

2236+
#[test]
2237+
fn polled_paths_get_debug_request_spans() {
2238+
let _traced = crate::otel_tracing::test_exporter::install_traced();
2239+
let level = |path: &str| {
2240+
let req = Request::builder()
2241+
.uri(path)
2242+
.body(Empty::<Bytes>::new())
2243+
.unwrap();
2244+
*make_request_span(&req)
2245+
.metadata()
2246+
.expect("span enabled")
2247+
.level()
2248+
};
2249+
2250+
for path in [
2251+
"/healthz",
2252+
"/openshell.v1.OpenShell/GetSandboxConfig",
2253+
"/openshell.v1.OpenShell/ReportProviderReadiness",
2254+
"/openshell.v1.OpenShell/PeerReportProviderReadiness",
2255+
] {
2256+
assert_eq!(level(path), tracing::Level::DEBUG, "{path}");
2257+
}
2258+
assert_eq!(
2259+
level("/openshell.v1.OpenShell/CreateSandbox"),
2260+
tracing::Level::INFO
2261+
);
2262+
}
2263+
22262264
/// The `TraceLayer` creates the server span, so no gRPC handler needs
22272265
/// `#[instrument]`. The request ID carries into it so a trace can be
22282266
/// correlated with the gateway's logs.

‎crates/openshell-server/src/persistence/mod.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,7 @@ impl Store {
360360
#[allow(clippy::too_many_arguments)]
361361
#[tracing::instrument(
362362
name = "store",
363+
level = "debug",
363364
skip_all,
364365
fields(otel.name = "store.put_if", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace)
365366
)]
@@ -394,6 +395,7 @@ impl Store {
394395
/// anything must use [`Self::put_if`], which is always durable.
395396
#[tracing::instrument(
396397
name = "store",
398+
level = "debug",
397399
skip_all,
398400
fields(otel.name = "store.create_relaxed", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace)
399401
)]
@@ -429,6 +431,7 @@ impl Store {
429431
/// * `Err(Conflict)` - Resource version mismatch
430432
#[tracing::instrument(
431433
name = "store",
434+
level = "debug",
432435
skip_all,
433436
fields(otel.name = "store.delete_if", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id)
434437
)]
@@ -445,6 +448,7 @@ impl Store {
445448
#[allow(clippy::too_many_arguments)]
446449
#[tracing::instrument(
447450
name = "store",
451+
level = "debug",
448452
skip_all,
449453
fields(otel.name = "store.put_scoped", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace, scope = %scope)
450454
)]
@@ -477,6 +481,7 @@ impl Store {
477481
#[allow(clippy::too_many_arguments)]
478482
#[tracing::instrument(
479483
name = "store",
484+
level = "debug",
480485
skip_all,
481486
fields(otel.name = "store.create_scoped", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace, scope = %scope)
482487
)]
@@ -510,6 +515,7 @@ impl Store {
510515
#[allow(clippy::too_many_arguments)]
511516
#[tracing::instrument(
512517
name = "store",
518+
level = "debug",
513519
skip_all,
514520
fields(otel.name = "store.create_if_workspace_count_below", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace, max_count = max_count)
515521
)]
@@ -537,6 +543,7 @@ impl Store {
537543
/// Fetch an object by id.
538544
#[tracing::instrument(
539545
name = "store",
546+
level = "debug",
540547
skip_all,
541548
fields(otel.name = "store.get", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id)
542549
)]
@@ -551,6 +558,7 @@ impl Store {
551558
/// Fetch an object by name within an object type and workspace.
552559
#[tracing::instrument(
553560
name = "store",
561+
level = "debug",
554562
skip_all,
555563
fields(
556564
otel.name = "store.get_by_name", otel.status_code = tracing::field::Empty,
@@ -571,6 +579,7 @@ impl Store {
571579
/// Delete an object by id.
572580
#[tracing::instrument(
573581
name = "store",
582+
level = "debug",
574583
skip_all,
575584
fields(otel.name = "store.delete", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id)
576585
)]
@@ -581,6 +590,7 @@ impl Store {
581590
/// Delete objects of one type by id in bounded, set-based statements.
582591
#[tracing::instrument(
583592
name = "store",
593+
level = "debug",
584594
skip_all,
585595
fields(
586596
otel.name = "store.delete_many",
@@ -597,6 +607,7 @@ impl Store {
597607
/// Count objects of a given type within a workspace.
598608
#[tracing::instrument(
599609
name = "store",
610+
level = "debug",
600611
skip_all,
601612
fields(otel.name = "store.count_in_workspace", otel.status_code = tracing::field::Empty, object_type = %object_type, workspace = %workspace)
602613
)]
@@ -611,6 +622,7 @@ impl Store {
611622
/// Delete all objects of a given type within a workspace.
612623
#[tracing::instrument(
613624
name = "store",
625+
level = "debug",
614626
skip_all,
615627
fields(otel.name = "store.delete_all_in_workspace", otel.status_code = tracing::field::Empty, object_type = %object_type, workspace = %workspace)
616628
)]
@@ -625,6 +637,7 @@ impl Store {
625637
/// Delete all objects of a given type with a matching scope.
626638
#[tracing::instrument(
627639
name = "store",
640+
level = "debug",
628641
skip_all,
629642
fields(otel.name = "store.delete_by_scope", otel.status_code = tracing::field::Empty, object_type = %object_type, scope = %scope)
630643
)]
@@ -635,6 +648,7 @@ impl Store {
635648
/// Delete an object by name within an object type and workspace.
636649
#[tracing::instrument(
637650
name = "store",
651+
level = "debug",
638652
skip_all,
639653
fields(otel.name = "store.delete_by_name", otel.status_code = tracing::field::Empty, object_type = %object_type, workspace = %workspace, object.name = %name)
640654
)]
@@ -650,6 +664,7 @@ impl Store {
650664
/// List objects by type and workspace.
651665
#[tracing::instrument(
652666
name = "store",
667+
level = "debug",
653668
skip_all,
654669
fields(otel.name = "store.list", otel.status_code = tracing::field::Empty, object_type = %object_type, workspace = %workspace)
655670
)]
@@ -666,6 +681,7 @@ impl Store {
666681
/// List objects by type across all workspaces.
667682
#[tracing::instrument(
668683
name = "store",
684+
level = "debug",
669685
skip_all,
670686
fields(otel.name = "store.list_by_type", otel.status_code = tracing::field::Empty, object_type = %object_type)
671687
)]
@@ -681,6 +697,7 @@ impl Store {
681697
/// List workspace objects after a stable cursor, without offset drift.
682698
#[tracing::instrument(
683699
name = "store",
700+
level = "debug",
684701
skip_all,
685702
fields(
686703
otel.name = "store.list_after",
@@ -702,6 +719,7 @@ impl Store {
702719
/// List objects across workspaces after a stable cursor, without offset drift.
703720
#[tracing::instrument(
704721
name = "store",
722+
level = "debug",
705723
skip_all,
706724
fields(
707725
otel.name = "store.list_by_type_after",
@@ -827,6 +845,7 @@ impl Store {
827845
/// UUIDs which are globally unique. Revisit if non-UUID scopes are introduced.
828846
#[tracing::instrument(
829847
name = "store",
848+
level = "debug",
830849
skip_all,
831850
fields(otel.name = "store.list_by_scope", otel.status_code = tracing::field::Empty, object_type = %object_type, scope = %scope)
832851
)]
@@ -844,6 +863,7 @@ impl Store {
844863
/// Label selector format: "key1=value1,key2=value2" (comma-separated equality matches).
845864
#[tracing::instrument(
846865
name = "store",
866+
level = "debug",
847867
skip_all,
848868
fields(
849869
otel.name = "store.list_with_selector", otel.status_code = tracing::field::Empty,
@@ -912,6 +932,7 @@ impl Store {
912932
/// List objects by type across all workspaces with label selector filtering.
913933
#[tracing::instrument(
914934
name = "store",
935+
level = "debug",
915936
skip_all,
916937
fields(otel.name = "store.list_all_with_selector", otel.status_code = tracing::field::Empty, object_type = %object_type, label_selector = %label_selector)
917938
)]
@@ -1310,6 +1331,7 @@ pub fn parse_label_selector(selector: &str) -> PersistenceResult<HashMap<String,
13101331
impl Store {
13111332
#[tracing::instrument(
13121333
name = "store",
1334+
level = "debug",
13131335
skip_all,
13141336
fields(otel.name = "store.put", otel.status_code = tracing::field::Empty, object_type = %object_type, object.id = %id, object.name = %name, workspace = %workspace)
13151337
)]

‎crates/openshell-server/src/provider_refresh.rs‎

Lines changed: 68 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize};
2020
use std::collections::HashMap;
2121
use std::time::Duration;
2222
use tonic::{Code, Status};
23-
use tracing::{info, warn};
23+
use tracing::{Instrument as _, info, warn};
2424

2525
use crate::storage_proto::{
2626
StoredProviderCredentialRefreshStateV2 as StoredProviderCredentialRefreshState,
@@ -1931,24 +1931,13 @@ pub fn spawn_refresh_worker(state: std::sync::Arc<crate::ServerState>, interval:
19311931
});
19321932
}
19331933

1934-
#[tracing::instrument(
1935-
name = "refresh",
1936-
skip_all,
1937-
fields(
1938-
otel.name = "refresh.provider_credentials",
1939-
watched_count = tracing::field::Empty,
1940-
due_count = tracing::field::Empty,
1941-
)
1942-
)]
19431934
async fn run_refresh_worker_tick(
19441935
store: &Store,
19451936
credentials: Option<&crate::credentials::CredentialRuntime>,
19461937
compute: Option<&crate::compute::ComputeRuntime>,
19471938
) -> Result<(), Status> {
19481939
let now_ms = current_time_ms();
1949-
let states = list_all_refresh_states(store).await.inspect_err(|_| {
1950-
crate::otel_tracing::mark_error(&tracing::Span::current());
1951-
})?;
1940+
let states = list_all_refresh_states(store).await?;
19521941
let watched_count = states.len();
19531942
let due_count = states
19541943
.iter()
@@ -1958,13 +1947,44 @@ async fn run_refresh_worker_tick(
19581947
.iter()
19591948
.filter(|state| state.status == "rotation_requested")
19601949
.count();
1961-
let span = tracing::Span::current();
1962-
span.record("watched_count", watched_count);
1963-
span.record("due_count", due_count);
19641950
info!(
19651951
watched_count,
19661952
due_count, rotation_requested_count, "provider credential refresh worker sweep"
19671953
);
1954+
if !states
1955+
.iter()
1956+
.any(|state| refresh_state_has_work(state, now_ms))
1957+
{
1958+
return Ok(());
1959+
}
1960+
let span = tracing::info_span!(
1961+
"refresh",
1962+
otel.name = "refresh.provider_credentials",
1963+
watched_count,
1964+
due_count,
1965+
);
1966+
Box::pin(refresh_states(store, credentials, compute, states, now_ms).instrument(span)).await;
1967+
Ok(())
1968+
}
1969+
1970+
fn refresh_state_has_work(state: &StoredProviderCredentialRefreshState, now_ms: i64) -> bool {
1971+
state
1972+
.metadata
1973+
.as_ref()
1974+
.is_some_and(|metadata| metadata.deletion_time.is_some())
1975+
|| !state.pending_secret_deletions.is_empty()
1976+
|| state.next_refresh_at_ms <= 0
1977+
|| state.next_refresh_at_ms <= now_ms
1978+
|| state.status == "rotation_requested"
1979+
}
1980+
1981+
async fn refresh_states(
1982+
store: &Store,
1983+
credentials: Option<&crate::credentials::CredentialRuntime>,
1984+
compute: Option<&crate::compute::ComputeRuntime>,
1985+
states: Vec<StoredProviderCredentialRefreshState>,
1986+
now_ms: i64,
1987+
) {
19681988
for state in states {
19691989
if state
19701990
.metadata
@@ -2079,7 +2099,6 @@ async fn run_refresh_worker_tick(
20792099
);
20802100
}
20812101
}
2082-
Ok(())
20832102
}
20842103

20852104
#[cfg(test)]
@@ -3560,11 +3579,9 @@ mod tests {
35603579
assert_eq!(credentials.stored_credential_count(), Some(0));
35613580
}
35623581

3563-
/// The worker ticks on a timer with no inbound request, so without a span
3564-
/// of its own its store reads export as anonymous single-span traces.
35653582
#[tokio::test]
35663583
#[ignore = "flaky under concurrent test execution"]
3567-
async fn refresh_worker_ticks_are_roots_and_store_operations_have_parents() {
3584+
async fn refresh_worker_records_a_root_span_only_when_a_state_has_work() {
35683585
use crate::otel_tracing::test_exporter;
35693586

35703587
let store = test_store().await;
@@ -3573,27 +3590,38 @@ mod tests {
35733590
Box::pin(run_refresh_worker_tick(&store, None, None))
35743591
.await
35753592
.unwrap();
3593+
assert!(
3594+
traced
3595+
.spans_named("refresh.provider_credentials")
3596+
.is_empty(),
3597+
"an idle tick records no refresh span"
3598+
);
35763599

3577-
let spans = traced.finished_spans();
3578-
let root = spans
3579-
.iter()
3580-
.find(|s| s.name == "refresh.provider_credentials")
3581-
.unwrap_or_else(|| {
3582-
panic!(
3583-
"the tick records a span of its own, got {:?}",
3584-
spans.iter().map(|s| &s.name).collect::<Vec<_>>()
3585-
)
3586-
});
3587-
3588-
test_exporter::assert_is_root(root);
3589-
let store_span = spans
3590-
.iter()
3591-
.find(|span| {
3592-
span.name.starts_with("store.")
3593-
&& span.span_context.trace_id() == root.span_context.trace_id()
3594-
})
3595-
.expect("the tick records its store operation");
3596-
test_exporter::assert_has_parent(store_span);
3600+
let provider = provider("my-external", "outlook");
3601+
store.put_message(&provider).await.unwrap();
3602+
let state = new_refresh_state(
3603+
&provider,
3604+
"default",
3605+
"MS_GRAPH_ACCESS_TOKEN",
3606+
NewRefreshStateConfig {
3607+
additional_output_keys: HashMap::new(),
3608+
strategy: ProviderCredentialRefreshStrategy::External,
3609+
material: HashMap::new(),
3610+
secret_material_keys: Vec::new(),
3611+
expires_at_ms: 0,
3612+
token_url: String::new(),
3613+
scopes: Vec::new(),
3614+
refresh_before: None,
3615+
max_lifetime: None,
3616+
},
3617+
)
3618+
.unwrap();
3619+
put_refresh_state(&store, &state).await.unwrap();
3620+
3621+
Box::pin(run_refresh_worker_tick(&store, None, None))
3622+
.await
3623+
.unwrap();
3624+
test_exporter::assert_is_root(&traced.span_named("refresh.provider_credentials"));
35973625
}
35983626

35993627
#[test]

0 commit comments

Comments
 (0)