Skip to content

Commit 3ef5380

Browse files
committed
fix(server): align routed request metric labels and outcomes
Count a local relay as successful only when its supervisor claims it, the same event that answers a peer relay on the owner, and rename the outcomes to local_error and remote_error so they say where an attempt failed. Label the peer latency histogram by operation, like the routed request counter, and rename the target label to relay_kind so it does not read as the Prometheus scrape target. Rename RelayCapacity.global to per_replica, describe the latency histogram as peer-only, and restore the note that unavailable spikes are expected during rollouts. Part of #3528 Signed-off-by: Emilien Macchi <[email protected]>
1 parent 00924b7 commit 3ef5380

4 files changed

Lines changed: 315 additions & 140 deletions

File tree

‎crates/openshell-server/src/gateway_metrics.rs‎

Lines changed: 66 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,10 @@ pub const RELAY_CLAIM_DURATION_SECONDS: &str = "openshell_server_relay_claim_dur
3636
pub const PEER_REQUEST_DURATION_SECONDS: &str = "openshell_server_peer_request_duration_seconds";
3737

3838
const LABEL_REASON: &str = "reason";
39-
const LABEL_METHOD: &str = "method";
4039
const LABEL_OPERATION: &str = "operation";
4140
const LABEL_OUTCOME: &str = "outcome";
4241
const LABEL_GRPC_CODE: &str = "grpc_code";
43-
const LABEL_TARGET: &str = "target";
42+
const LABEL_RELAY_KIND: &str = "relay_kind";
4443
const LABEL_ROUTE: &str = "route";
4544

4645
/// Buckets for the new latency histograms, 1 ms to 15 s. The top buckets cover the 10 s relay
@@ -56,12 +55,12 @@ const BUCKETED_HISTOGRAMS: [&str; 2] =
5655

5756
/// Protocol the supervisor is asked to relay. Never label metrics with the target address.
5857
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
59-
pub enum RelayTarget {
58+
pub enum RelayKind {
6059
Ssh,
6160
Tcp,
6261
}
6362

64-
impl RelayTarget {
63+
impl RelayKind {
6564
pub const ALL: [Self; 2] = [Self::Ssh, Self::Tcp];
6665

6766
pub const fn label(self) -> &'static str {
@@ -108,8 +107,8 @@ impl RelayRejection {
108107
}
109108
}
110109

111-
/// Outbound peer RPC. The `method` label takes the same values as the `method` label of
112-
/// `openshell_server_grpc_requests_total` on the owning replica.
110+
/// Routed operation. For a peer request, the owning replica records the matching gRPC method in
111+
/// `openshell_server_grpc_requests_total`, for example `PeerRelay` for `relay`.
113112
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
114113
pub enum PeerRpc {
115114
Relay,
@@ -126,15 +125,6 @@ impl PeerRpc {
126125
Self::GetSandboxProviderStatus,
127126
];
128127

129-
pub const fn label(self) -> &'static str {
130-
match self {
131-
Self::Relay => "PeerRelay",
132-
Self::ReportProviderReadiness => "PeerReportProviderReadiness",
133-
Self::ReportEndpointStatus => "PeerReportEndpointStatus",
134-
Self::GetSandboxProviderStatus => "PeerGetSandboxProviderStatus",
135-
}
136-
}
137-
138128
const fn operation(self) -> &'static str {
139129
match self {
140130
Self::Relay => "relay",
@@ -145,19 +135,23 @@ impl PeerRpc {
145135
}
146136
}
147137

138+
/// Where a routed attempt ended. A relay succeeds only when the supervisor claims it, on either
139+
/// route, so the values mean the same thing for local and peer attempts.
148140
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
149-
enum PeerOutcome {
141+
enum AttemptOutcome {
150142
Success,
151-
ClientError,
152-
RpcError,
143+
/// Failed on this replica, including cancellation by the caller.
144+
LocalError,
145+
/// The owning replica returned an error, or the open peer connection failed.
146+
RemoteError,
153147
}
154148

155-
impl PeerOutcome {
149+
impl AttemptOutcome {
156150
const fn label(self) -> &'static str {
157151
match self {
158152
Self::Success => "success",
159-
Self::ClientError => "client_error",
160-
Self::RpcError => "rpc_error",
153+
Self::LocalError => "local_error",
154+
Self::RemoteError => "remote_error",
161155
}
162156
}
163157
}
@@ -193,7 +187,7 @@ const fn grpc_code_label(code: Code) -> &'static str {
193187
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
194188
pub struct RelayCapacity {
195189
/// Pending relays allowed on one replica.
196-
pub global: usize,
190+
pub per_replica: usize,
197191
/// Pending relays allowed for one sandbox on one replica.
198192
pub per_sandbox: usize,
199193
}
@@ -264,22 +258,22 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
264258
describe_histogram!(
265259
PEER_REQUEST_DURATION_SECONDS,
266260
Unit::Seconds,
267-
"Latency of requests to the owning replica. For PeerRelay, until the owner's supervisor claimed the relay."
261+
"Latency of outbound requests to the owning replica. For relays, until the owner's supervisor claimed the relay."
268262
);
269263

270264
// `increment(0)` registers a series without overwriting a value recorded earlier.
271265
gauge!(SUPERVISOR_SESSIONS).increment(0.0);
272266
gauge!(RELAY_PENDING).increment(0.0);
273-
gauge!(RELAY_PENDING_CAPACITY).set(count_as_f64(relay.global));
267+
gauge!(RELAY_PENDING_CAPACITY).set(count_as_f64(relay.per_replica));
274268
gauge!(RELAY_PENDING_PER_SANDBOX_CAPACITY).set(count_as_f64(relay.per_sandbox));
275-
for target in RelayTarget::ALL {
269+
for kind in RelayKind::ALL {
276270
for route in RelayRoute::ALL {
277271
counter!(
278272
ROUTED_REQUEST_ATTEMPTS_TOTAL,
279273
LABEL_OPERATION => PeerRpc::Relay.operation(),
280274
LABEL_ROUTE => route.label(),
281-
LABEL_TARGET => target.label(),
282-
LABEL_OUTCOME => PeerOutcome::Success.label(),
275+
LABEL_RELAY_KIND => kind.label(),
276+
LABEL_OUTCOME => AttemptOutcome::Success.label(),
283277
LABEL_GRPC_CODE => grpc_code_label(Code::Ok)
284278
)
285279
.increment(0);
@@ -297,8 +291,8 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
297291
ROUTED_REQUEST_ATTEMPTS_TOTAL,
298292
LABEL_OPERATION => rpc.operation(),
299293
LABEL_ROUTE => RelayRoute::Peer.label(),
300-
LABEL_TARGET => "none",
301-
LABEL_OUTCOME => PeerOutcome::Success.label(),
294+
LABEL_RELAY_KIND => "none",
295+
LABEL_OUTCOME => AttemptOutcome::Success.label(),
302296
LABEL_GRPC_CODE => grpc_code_label(Code::Ok)
303297
)
304298
.increment(0);
@@ -357,14 +351,13 @@ pub fn record_relay_claimed(waited: Duration) {
357351
}
358352

359353
/// Counts one local relay setup or outbound peer attempt exactly once, and times peer requests.
360-
/// Local success means enqueued; peer relay success means the supervisor claimed it.
361-
/// Dropping an unfinished timer (the caller's future was cancelled) records
362-
/// `client_error` / `cancelled`.
363-
#[must_use = "finish the timer with client_error() or finish()"]
354+
/// A relay succeeds when the supervisor claims it, on either route. Dropping an unfinished
355+
/// timer (the caller gave up) records `local_error` / `cancelled`.
356+
#[must_use = "finish the timer with local_error() or finish()"]
364357
pub struct RoutedRequestTimer {
365358
rpc: PeerRpc,
366359
route: RelayRoute,
367-
target: Option<RelayTarget>,
360+
relay_kind: Option<RelayKind>,
368361
started: Instant,
369362
recorded: bool,
370363
}
@@ -374,8 +367,8 @@ impl RoutedRequestTimer {
374367
Self {
375368
rpc,
376369
route: RelayRoute::Peer,
377-
target: if rpc == PeerRpc::Relay {
378-
Some(RelayTarget::Ssh)
370+
relay_kind: if rpc == PeerRpc::Relay {
371+
Some(RelayKind::Ssh)
379372
} else {
380373
None
381374
},
@@ -384,32 +377,34 @@ impl RoutedRequestTimer {
384377
}
385378
}
386379

387-
pub fn relay(target: RelayTarget, route: RelayRoute) -> Self {
380+
pub fn relay(kind: RelayKind, route: RelayRoute) -> Self {
388381
Self {
389382
rpc: PeerRpc::Relay,
390383
route,
391-
target: Some(target),
384+
relay_kind: Some(kind),
392385
started: Instant::now(),
393386
recorded: false,
394387
}
395388
}
396389

397-
/// The request failed before it reached the peer: token, channel, headers, or stream setup.
398-
pub fn client_error(&mut self, status: &Status) {
399-
self.record(PeerOutcome::ClientError, status.code());
390+
/// The attempt failed on this replica: local relay setup or claim (including the claim
391+
/// window closing), or a peer request that failed before it reached the owner (token,
392+
/// channel, headers, or stream setup).
393+
pub fn local_error(&mut self, status: &Status) {
394+
self.record(AttemptOutcome::LocalError, status.code());
400395
}
401396

402397
/// Record the raw tonic result of the RPC itself. Call this BEFORE any remap to
403398
/// `Unavailable`, so the owner's code (for example `resource_exhausted`) is kept.
404399
pub fn finish<T>(&mut self, result: &Result<T, Status>) {
405400
match result {
406-
Ok(_) => self.record(PeerOutcome::Success, Code::Ok),
407-
Err(status) if self.route == RelayRoute::Local => self.client_error(status),
408-
Err(status) => self.record(PeerOutcome::RpcError, status.code()),
401+
Ok(_) => self.record(AttemptOutcome::Success, Code::Ok),
402+
Err(status) if self.route == RelayRoute::Local => self.local_error(status),
403+
Err(status) => self.record(AttemptOutcome::RemoteError, status.code()),
409404
}
410405
}
411406

412-
fn record(&mut self, outcome: PeerOutcome, code: Code) {
407+
fn record(&mut self, outcome: AttemptOutcome, code: Code) {
413408
if self.recorded {
414409
return;
415410
}
@@ -418,15 +413,15 @@ impl RoutedRequestTimer {
418413
ROUTED_REQUEST_ATTEMPTS_TOTAL,
419414
LABEL_OPERATION => self.rpc.operation(),
420415
LABEL_ROUTE => self.route.label(),
421-
LABEL_TARGET => self.target.map_or("none", RelayTarget::label),
416+
LABEL_RELAY_KIND => self.relay_kind.map_or("none", RelayKind::label),
422417
LABEL_OUTCOME => outcome.label(),
423418
LABEL_GRPC_CODE => grpc_code_label(code)
424419
)
425420
.increment(1);
426421
if self.route == RelayRoute::Peer {
427422
histogram!(
428423
PEER_REQUEST_DURATION_SECONDS,
429-
LABEL_METHOD => self.rpc.label(),
424+
LABEL_OPERATION => self.rpc.operation(),
430425
LABEL_OUTCOME => outcome.label()
431426
)
432427
.record(self.started.elapsed());
@@ -436,7 +431,7 @@ impl RoutedRequestTimer {
436431

437432
impl Drop for RoutedRequestTimer {
438433
fn drop(&mut self) {
439-
self.record(PeerOutcome::ClientError, Code::Cancelled);
434+
self.record(AttemptOutcome::LocalError, Code::Cancelled);
440435
}
441436
}
442437

@@ -505,7 +500,7 @@ mod tests {
505500
fn describe_and_initialize_exports_capacity_and_zero_series() {
506501
let metrics = MetricsCapture::install();
507502
describe_and_initialize(RelayCapacity {
508-
global: 256,
503+
per_replica: 256,
509504
per_sandbox: 32,
510505
});
511506

@@ -526,10 +521,10 @@ mod tests {
526521
] {
527522
assert_eq!(metrics.value(series), Some(expected), "{series}");
528523
}
529-
for target in ["ssh", "tcp"] {
524+
for kind in ["ssh", "tcp"] {
530525
for route in ["local", "peer"] {
531526
let series = format!(
532-
"openshell_server_routed_request_attempts_total{{operation=\"relay\",route=\"{route}\",target=\"{target}\",outcome=\"success\",grpc_code=\"ok\"}}"
527+
"openshell_server_routed_request_attempts_total{{operation=\"relay\",route=\"{route}\",relay_kind=\"{kind}\",outcome=\"success\",grpc_code=\"ok\"}}"
533528
);
534529
assert_eq!(metrics.value(&series), Some(0), "{series}");
535530
}
@@ -540,7 +535,7 @@ mod tests {
540535
"get_sandbox_provider_status",
541536
] {
542537
let series = format!(
543-
"openshell_server_routed_request_attempts_total{{operation=\"{operation}\",route=\"peer\",target=\"none\",outcome=\"success\",grpc_code=\"ok\"}}"
538+
"openshell_server_routed_request_attempts_total{{operation=\"{operation}\",route=\"peer\",relay_kind=\"none\",outcome=\"success\",grpc_code=\"ok\"}}"
544539
);
545540
assert_eq!(metrics.value(&series), Some(0), "{series}");
546541
}
@@ -554,23 +549,23 @@ mod tests {
554549
#[test]
555550
fn routed_attempts_have_seven_bounded_success_series_and_keep_counts_on_initialize() {
556551
let metrics = MetricsCapture::install();
557-
for target in RelayTarget::ALL {
552+
for kind in RelayKind::ALL {
558553
for route in RelayRoute::ALL {
559-
RoutedRequestTimer::relay(target, route).finish(&Ok::<(), Status>(()));
554+
RoutedRequestTimer::relay(kind, route).finish(&Ok::<(), Status>(()));
560555
}
561556
}
562-
RoutedRequestTimer::relay(RelayTarget::Tcp, RelayRoute::Peer).finish(&Ok::<(), Status>(()));
557+
RoutedRequestTimer::relay(RelayKind::Tcp, RelayRoute::Peer).finish(&Ok::<(), Status>(()));
563558
describe_and_initialize(RelayCapacity {
564-
global: 256,
559+
per_replica: 256,
565560
per_sandbox: 32,
566561
});
567562

568-
for target in ["ssh", "tcp"] {
563+
for kind in ["ssh", "tcp"] {
569564
for route in ["local", "peer"] {
570565
let series = format!(
571-
"openshell_server_routed_request_attempts_total{{operation=\"relay\",route=\"{route}\",target=\"{target}\",outcome=\"success\",grpc_code=\"ok\"}}"
566+
"openshell_server_routed_request_attempts_total{{operation=\"relay\",route=\"{route}\",relay_kind=\"{kind}\",outcome=\"success\",grpc_code=\"ok\"}}"
572567
);
573-
let expected = if target == "tcp" && route == "peer" {
568+
let expected = if kind == "tcp" && route == "peer" {
574569
2
575570
} else {
576571
1
@@ -598,7 +593,7 @@ mod tests {
598593
histogram!(RELAY_CLAIM_DURATION_SECONDS).record(sample);
599594
histogram!(
600595
PEER_REQUEST_DURATION_SECONDS,
601-
LABEL_METHOD => "PeerRelay",
596+
LABEL_OPERATION => "relay",
602597
LABEL_OUTCOME => "success"
603598
)
604599
.record(sample);
@@ -682,23 +677,23 @@ mod tests {
682677
drop(relay);
683678
assert_eq!(
684679
metrics.value(
685-
"openshell_server_routed_request_attempts_total{operation=\"relay\",route=\"peer\",target=\"ssh\",outcome=\"rpc_error\",grpc_code=\"resource_exhausted\"}"
680+
"openshell_server_routed_request_attempts_total{operation=\"relay\",route=\"peer\",relay_kind=\"ssh\",outcome=\"remote_error\",grpc_code=\"resource_exhausted\"}"
686681
),
687682
Some(1)
688683
);
689684
assert_eq!(
690685
metrics.value(
691-
"openshell_server_peer_request_duration_seconds_count{method=\"PeerRelay\",outcome=\"rpc_error\"}"
686+
"openshell_server_peer_request_duration_seconds_count{operation=\"relay\",outcome=\"remote_error\"}"
692687
),
693688
Some(1)
694689
);
695690

696691
let mut endpoint = RoutedRequestTimer::start(PeerRpc::ReportEndpointStatus);
697-
endpoint.client_error(&Status::unavailable("x"));
692+
endpoint.local_error(&Status::unavailable("x"));
698693
drop(endpoint);
699694
assert_eq!(
700695
metrics.value(
701-
"openshell_server_routed_request_attempts_total{operation=\"report_endpoint_status\",route=\"peer\",target=\"none\",outcome=\"client_error\",grpc_code=\"unavailable\"}"
696+
"openshell_server_routed_request_attempts_total{operation=\"report_endpoint_status\",route=\"peer\",relay_kind=\"none\",outcome=\"local_error\",grpc_code=\"unavailable\"}"
702697
),
703698
Some(1)
704699
);
@@ -708,7 +703,7 @@ mod tests {
708703
drop(provider_status);
709704
assert_eq!(
710705
metrics.value(
711-
"openshell_server_routed_request_attempts_total{operation=\"get_sandbox_provider_status\",route=\"peer\",target=\"none\",outcome=\"success\",grpc_code=\"ok\"}"
706+
"openshell_server_routed_request_attempts_total{operation=\"get_sandbox_provider_status\",route=\"peer\",relay_kind=\"none\",outcome=\"success\",grpc_code=\"ok\"}"
712707
),
713708
Some(1)
714709
);
@@ -720,7 +715,7 @@ mod tests {
720715
drop(RoutedRequestTimer::start(PeerRpc::ReportProviderReadiness));
721716
assert_eq!(
722717
metrics.value(
723-
"openshell_server_routed_request_attempts_total{operation=\"report_provider_readiness\",route=\"peer\",target=\"none\",outcome=\"client_error\",grpc_code=\"cancelled\"}"
718+
"openshell_server_routed_request_attempts_total{operation=\"report_provider_readiness\",route=\"peer\",relay_kind=\"none\",outcome=\"local_error\",grpc_code=\"cancelled\"}"
724719
),
725720
Some(1)
726721
);
@@ -731,25 +726,25 @@ mod tests {
731726
let metrics = MetricsCapture::install();
732727
let mut timer = RoutedRequestTimer::start(PeerRpc::Relay);
733728
timer.finish(&Ok::<(), Status>(()));
734-
timer.client_error(&Status::unavailable("x"));
729+
timer.local_error(&Status::unavailable("x"));
735730
drop(timer);
736731
assert_eq!(
737732
metrics.value(
738-
"openshell_server_routed_request_attempts_total{operation=\"relay\",route=\"peer\",target=\"ssh\",outcome=\"success\",grpc_code=\"ok\"}"
733+
"openshell_server_routed_request_attempts_total{operation=\"relay\",route=\"peer\",relay_kind=\"ssh\",outcome=\"success\",grpc_code=\"ok\"}"
739734
),
740735
Some(1)
741736
);
742-
assert!(!metrics.render().contains("outcome=\"client_error\""));
737+
assert!(!metrics.render().contains("outcome=\"local_error\""));
743738
}
744739

745740
#[test]
746741
fn local_relay_failure_records_status_without_peer_latency() {
747742
let metrics = MetricsCapture::install();
748-
let mut timer = RoutedRequestTimer::relay(RelayTarget::Tcp, RelayRoute::Local);
743+
let mut timer = RoutedRequestTimer::relay(RelayKind::Tcp, RelayRoute::Local);
749744
timer.finish(&Err::<(), _>(Status::resource_exhausted("capacity")));
750745
drop(timer);
751746
assert_eq!(
752-
metrics.value("openshell_server_routed_request_attempts_total{operation=\"relay\",route=\"local\",target=\"tcp\",outcome=\"client_error\",grpc_code=\"resource_exhausted\"}"),
747+
metrics.value("openshell_server_routed_request_attempts_total{operation=\"relay\",route=\"local\",relay_kind=\"tcp\",outcome=\"local_error\",grpc_code=\"resource_exhausted\"}"),
753748
Some(1)
754749
);
755750
assert!(!metrics.render().contains(PEER_REQUEST_DURATION_SECONDS));

0 commit comments

Comments
 (0)