@@ -36,11 +36,10 @@ pub const RELAY_CLAIM_DURATION_SECONDS: &str = "openshell_server_relay_claim_dur
3636pub const PEER_REQUEST_DURATION_SECONDS : & str = "openshell_server_peer_request_duration_seconds" ;
3737
3838const LABEL_REASON : & str = "reason" ;
39- const LABEL_METHOD : & str = "method" ;
4039const LABEL_OPERATION : & str = "operation" ;
4140const LABEL_OUTCOME : & str = "outcome" ;
4241const LABEL_GRPC_CODE : & str = "grpc_code" ;
43- const LABEL_TARGET : & str = "target " ;
42+ const LABEL_RELAY_KIND : & str = "relay_kind " ;
4443const LABEL_ROUTE : & str = "route" ;
4544
4645/// Buckets for the new latency histograms, 1 ms to 15 s. The top buckets cover the 10 s relay
@@ -56,12 +55,12 @@ const BUCKETED_HISTOGRAMS: [&str; 2] =
5655
5756/// Protocol the supervisor is asked to relay. Never label metrics with the target address.
5857#[ derive( Clone , Copy , Debug , PartialEq , Eq ) ]
59- pub enum RelayTarget {
58+ pub enum RelayKind {
6059 Ssh ,
6160 Tcp ,
6261}
6362
64- impl RelayTarget {
63+ impl RelayKind {
6564 pub const ALL : [ Self ; 2 ] = [ Self :: Ssh , Self :: Tcp ] ;
6665
6766 pub const fn label ( self ) -> & ' static str {
@@ -108,8 +107,8 @@ impl RelayRejection {
108107 }
109108}
110109
111- /// Outbound peer RPC. The `method` label takes the same values as the `method` label of
112- /// `openshell_server_grpc_requests_total` on the owning replica .
110+ /// Routed operation. For a peer request, the owning replica records the matching gRPC method in
111+ /// `openshell_server_grpc_requests_total`, for example `PeerRelay` for `relay` .
113112#[ derive( Clone , Copy , Debug , PartialEq , Eq ) ]
114113pub enum PeerRpc {
115114 Relay ,
@@ -126,15 +125,6 @@ impl PeerRpc {
126125 Self :: GetSandboxProviderStatus ,
127126 ] ;
128127
129- pub const fn label ( self ) -> & ' static str {
130- match self {
131- Self :: Relay => "PeerRelay" ,
132- Self :: ReportProviderReadiness => "PeerReportProviderReadiness" ,
133- Self :: ReportEndpointStatus => "PeerReportEndpointStatus" ,
134- Self :: GetSandboxProviderStatus => "PeerGetSandboxProviderStatus" ,
135- }
136- }
137-
138128 const fn operation ( self ) -> & ' static str {
139129 match self {
140130 Self :: Relay => "relay" ,
@@ -145,19 +135,23 @@ impl PeerRpc {
145135 }
146136}
147137
138+ /// Where a routed attempt ended. A relay succeeds only when the supervisor claims it, on either
139+ /// route, so the values mean the same thing for local and peer attempts.
148140#[ derive( Clone , Copy , Debug , PartialEq , Eq ) ]
149- enum PeerOutcome {
141+ enum AttemptOutcome {
150142 Success ,
151- ClientError ,
152- RpcError ,
143+ /// Failed on this replica, including cancellation by the caller.
144+ LocalError ,
145+ /// The owning replica returned an error, or the open peer connection failed.
146+ RemoteError ,
153147}
154148
155- impl PeerOutcome {
149+ impl AttemptOutcome {
156150 const fn label ( self ) -> & ' static str {
157151 match self {
158152 Self :: Success => "success" ,
159- Self :: ClientError => "client_error " ,
160- Self :: RpcError => "rpc_error " ,
153+ Self :: LocalError => "local_error " ,
154+ Self :: RemoteError => "remote_error " ,
161155 }
162156 }
163157}
@@ -193,7 +187,7 @@ const fn grpc_code_label(code: Code) -> &'static str {
193187#[ derive( Clone , Copy , Debug , PartialEq , Eq ) ]
194188pub struct RelayCapacity {
195189 /// Pending relays allowed on one replica.
196- pub global : usize ,
190+ pub per_replica : usize ,
197191 /// Pending relays allowed for one sandbox on one replica.
198192 pub per_sandbox : usize ,
199193}
@@ -264,22 +258,22 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
264258 describe_histogram ! (
265259 PEER_REQUEST_DURATION_SECONDS ,
266260 Unit :: Seconds ,
267- "Latency of requests to the owning replica. For PeerRelay , until the owner's supervisor claimed the relay."
261+ "Latency of outbound requests to the owning replica. For relays , until the owner's supervisor claimed the relay."
268262 ) ;
269263
270264 // `increment(0)` registers a series without overwriting a value recorded earlier.
271265 gauge ! ( SUPERVISOR_SESSIONS ) . increment ( 0.0 ) ;
272266 gauge ! ( RELAY_PENDING ) . increment ( 0.0 ) ;
273- gauge ! ( RELAY_PENDING_CAPACITY ) . set ( count_as_f64 ( relay. global ) ) ;
267+ gauge ! ( RELAY_PENDING_CAPACITY ) . set ( count_as_f64 ( relay. per_replica ) ) ;
274268 gauge ! ( RELAY_PENDING_PER_SANDBOX_CAPACITY ) . set ( count_as_f64 ( relay. per_sandbox ) ) ;
275- for target in RelayTarget :: ALL {
269+ for kind in RelayKind :: ALL {
276270 for route in RelayRoute :: ALL {
277271 counter ! (
278272 ROUTED_REQUEST_ATTEMPTS_TOTAL ,
279273 LABEL_OPERATION => PeerRpc :: Relay . operation( ) ,
280274 LABEL_ROUTE => route. label( ) ,
281- LABEL_TARGET => target . label( ) ,
282- LABEL_OUTCOME => PeerOutcome :: Success . label( ) ,
275+ LABEL_RELAY_KIND => kind . label( ) ,
276+ LABEL_OUTCOME => AttemptOutcome :: Success . label( ) ,
283277 LABEL_GRPC_CODE => grpc_code_label( Code :: Ok )
284278 )
285279 . increment ( 0 ) ;
@@ -297,8 +291,8 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
297291 ROUTED_REQUEST_ATTEMPTS_TOTAL ,
298292 LABEL_OPERATION => rpc. operation( ) ,
299293 LABEL_ROUTE => RelayRoute :: Peer . label( ) ,
300- LABEL_TARGET => "none" ,
301- LABEL_OUTCOME => PeerOutcome :: Success . label( ) ,
294+ LABEL_RELAY_KIND => "none" ,
295+ LABEL_OUTCOME => AttemptOutcome :: Success . label( ) ,
302296 LABEL_GRPC_CODE => grpc_code_label( Code :: Ok )
303297 )
304298 . increment ( 0 ) ;
@@ -357,14 +351,13 @@ pub fn record_relay_claimed(waited: Duration) {
357351}
358352
359353/// Counts one local relay setup or outbound peer attempt exactly once, and times peer requests.
360- /// Local success means enqueued; peer relay success means the supervisor claimed it.
361- /// Dropping an unfinished timer (the caller's future was cancelled) records
362- /// `client_error` / `cancelled`.
363- #[ must_use = "finish the timer with client_error() or finish()" ]
354+ /// A relay succeeds when the supervisor claims it, on either route. Dropping an unfinished
355+ /// timer (the caller gave up) records `local_error` / `cancelled`.
356+ #[ must_use = "finish the timer with local_error() or finish()" ]
364357pub struct RoutedRequestTimer {
365358 rpc : PeerRpc ,
366359 route : RelayRoute ,
367- target : Option < RelayTarget > ,
360+ relay_kind : Option < RelayKind > ,
368361 started : Instant ,
369362 recorded : bool ,
370363}
@@ -374,8 +367,8 @@ impl RoutedRequestTimer {
374367 Self {
375368 rpc,
376369 route : RelayRoute :: Peer ,
377- target : if rpc == PeerRpc :: Relay {
378- Some ( RelayTarget :: Ssh )
370+ relay_kind : if rpc == PeerRpc :: Relay {
371+ Some ( RelayKind :: Ssh )
379372 } else {
380373 None
381374 } ,
@@ -384,32 +377,34 @@ impl RoutedRequestTimer {
384377 }
385378 }
386379
387- pub fn relay ( target : RelayTarget , route : RelayRoute ) -> Self {
380+ pub fn relay ( kind : RelayKind , route : RelayRoute ) -> Self {
388381 Self {
389382 rpc : PeerRpc :: Relay ,
390383 route,
391- target : Some ( target ) ,
384+ relay_kind : Some ( kind ) ,
392385 started : Instant :: now ( ) ,
393386 recorded : false ,
394387 }
395388 }
396389
397- /// The request failed before it reached the peer: token, channel, headers, or stream setup.
398- pub fn client_error ( & mut self , status : & Status ) {
399- self . record ( PeerOutcome :: ClientError , status. code ( ) ) ;
390+ /// The attempt failed on this replica: local relay setup or claim (including the claim
391+ /// window closing), or a peer request that failed before it reached the owner (token,
392+ /// channel, headers, or stream setup).
393+ pub fn local_error ( & mut self , status : & Status ) {
394+ self . record ( AttemptOutcome :: LocalError , status. code ( ) ) ;
400395 }
401396
402397 /// Record the raw tonic result of the RPC itself. Call this BEFORE any remap to
403398 /// `Unavailable`, so the owner's code (for example `resource_exhausted`) is kept.
404399 pub fn finish < T > ( & mut self , result : & Result < T , Status > ) {
405400 match result {
406- Ok ( _) => self . record ( PeerOutcome :: Success , Code :: Ok ) ,
407- Err ( status) if self . route == RelayRoute :: Local => self . client_error ( status) ,
408- Err ( status) => self . record ( PeerOutcome :: RpcError , status. code ( ) ) ,
401+ Ok ( _) => self . record ( AttemptOutcome :: Success , Code :: Ok ) ,
402+ Err ( status) if self . route == RelayRoute :: Local => self . local_error ( status) ,
403+ Err ( status) => self . record ( AttemptOutcome :: RemoteError , status. code ( ) ) ,
409404 }
410405 }
411406
412- fn record ( & mut self , outcome : PeerOutcome , code : Code ) {
407+ fn record ( & mut self , outcome : AttemptOutcome , code : Code ) {
413408 if self . recorded {
414409 return ;
415410 }
@@ -418,15 +413,15 @@ impl RoutedRequestTimer {
418413 ROUTED_REQUEST_ATTEMPTS_TOTAL ,
419414 LABEL_OPERATION => self . rpc. operation( ) ,
420415 LABEL_ROUTE => self . route. label( ) ,
421- LABEL_TARGET => self . target . map_or( "none" , RelayTarget :: label) ,
416+ LABEL_RELAY_KIND => self . relay_kind . map_or( "none" , RelayKind :: label) ,
422417 LABEL_OUTCOME => outcome. label( ) ,
423418 LABEL_GRPC_CODE => grpc_code_label( code)
424419 )
425420 . increment ( 1 ) ;
426421 if self . route == RelayRoute :: Peer {
427422 histogram ! (
428423 PEER_REQUEST_DURATION_SECONDS ,
429- LABEL_METHOD => self . rpc. label ( ) ,
424+ LABEL_OPERATION => self . rpc. operation ( ) ,
430425 LABEL_OUTCOME => outcome. label( )
431426 )
432427 . record ( self . started . elapsed ( ) ) ;
@@ -436,7 +431,7 @@ impl RoutedRequestTimer {
436431
437432impl Drop for RoutedRequestTimer {
438433 fn drop ( & mut self ) {
439- self . record ( PeerOutcome :: ClientError , Code :: Cancelled ) ;
434+ self . record ( AttemptOutcome :: LocalError , Code :: Cancelled ) ;
440435 }
441436}
442437
@@ -505,7 +500,7 @@ mod tests {
505500 fn describe_and_initialize_exports_capacity_and_zero_series ( ) {
506501 let metrics = MetricsCapture :: install ( ) ;
507502 describe_and_initialize ( RelayCapacity {
508- global : 256 ,
503+ per_replica : 256 ,
509504 per_sandbox : 32 ,
510505 } ) ;
511506
@@ -526,10 +521,10 @@ mod tests {
526521 ] {
527522 assert_eq ! ( metrics. value( series) , Some ( expected) , "{series}" ) ;
528523 }
529- for target in [ "ssh" , "tcp" ] {
524+ for kind in [ "ssh" , "tcp" ] {
530525 for route in [ "local" , "peer" ] {
531526 let series = format ! (
532- "openshell_server_routed_request_attempts_total{{operation=\" relay\" ,route=\" {route}\" ,target =\" {target }\" ,outcome=\" success\" ,grpc_code=\" ok\" }}"
527+ "openshell_server_routed_request_attempts_total{{operation=\" relay\" ,route=\" {route}\" ,relay_kind =\" {kind }\" ,outcome=\" success\" ,grpc_code=\" ok\" }}"
533528 ) ;
534529 assert_eq ! ( metrics. value( & series) , Some ( 0 ) , "{series}" ) ;
535530 }
@@ -540,7 +535,7 @@ mod tests {
540535 "get_sandbox_provider_status" ,
541536 ] {
542537 let series = format ! (
543- "openshell_server_routed_request_attempts_total{{operation=\" {operation}\" ,route=\" peer\" ,target =\" none\" ,outcome=\" success\" ,grpc_code=\" ok\" }}"
538+ "openshell_server_routed_request_attempts_total{{operation=\" {operation}\" ,route=\" peer\" ,relay_kind =\" none\" ,outcome=\" success\" ,grpc_code=\" ok\" }}"
544539 ) ;
545540 assert_eq ! ( metrics. value( & series) , Some ( 0 ) , "{series}" ) ;
546541 }
@@ -554,23 +549,23 @@ mod tests {
554549 #[ test]
555550 fn routed_attempts_have_seven_bounded_success_series_and_keep_counts_on_initialize ( ) {
556551 let metrics = MetricsCapture :: install ( ) ;
557- for target in RelayTarget :: ALL {
552+ for kind in RelayKind :: ALL {
558553 for route in RelayRoute :: ALL {
559- RoutedRequestTimer :: relay ( target , route) . finish ( & Ok :: < ( ) , Status > ( ( ) ) ) ;
554+ RoutedRequestTimer :: relay ( kind , route) . finish ( & Ok :: < ( ) , Status > ( ( ) ) ) ;
560555 }
561556 }
562- RoutedRequestTimer :: relay ( RelayTarget :: Tcp , RelayRoute :: Peer ) . finish ( & Ok :: < ( ) , Status > ( ( ) ) ) ;
557+ RoutedRequestTimer :: relay ( RelayKind :: Tcp , RelayRoute :: Peer ) . finish ( & Ok :: < ( ) , Status > ( ( ) ) ) ;
563558 describe_and_initialize ( RelayCapacity {
564- global : 256 ,
559+ per_replica : 256 ,
565560 per_sandbox : 32 ,
566561 } ) ;
567562
568- for target in [ "ssh" , "tcp" ] {
563+ for kind in [ "ssh" , "tcp" ] {
569564 for route in [ "local" , "peer" ] {
570565 let series = format ! (
571- "openshell_server_routed_request_attempts_total{{operation=\" relay\" ,route=\" {route}\" ,target =\" {target }\" ,outcome=\" success\" ,grpc_code=\" ok\" }}"
566+ "openshell_server_routed_request_attempts_total{{operation=\" relay\" ,route=\" {route}\" ,relay_kind =\" {kind }\" ,outcome=\" success\" ,grpc_code=\" ok\" }}"
572567 ) ;
573- let expected = if target == "tcp" && route == "peer" {
568+ let expected = if kind == "tcp" && route == "peer" {
574569 2
575570 } else {
576571 1
@@ -598,7 +593,7 @@ mod tests {
598593 histogram ! ( RELAY_CLAIM_DURATION_SECONDS ) . record ( sample) ;
599594 histogram ! (
600595 PEER_REQUEST_DURATION_SECONDS ,
601- LABEL_METHOD => "PeerRelay " ,
596+ LABEL_OPERATION => "relay " ,
602597 LABEL_OUTCOME => "success"
603598 )
604599 . record ( sample) ;
@@ -682,23 +677,23 @@ mod tests {
682677 drop ( relay) ;
683678 assert_eq ! (
684679 metrics. value(
685- "openshell_server_routed_request_attempts_total{operation=\" relay\" ,route=\" peer\" ,target =\" ssh\" ,outcome=\" rpc_error \" ,grpc_code=\" resource_exhausted\" }"
680+ "openshell_server_routed_request_attempts_total{operation=\" relay\" ,route=\" peer\" ,relay_kind =\" ssh\" ,outcome=\" remote_error \" ,grpc_code=\" resource_exhausted\" }"
686681 ) ,
687682 Some ( 1 )
688683 ) ;
689684 assert_eq ! (
690685 metrics. value(
691- "openshell_server_peer_request_duration_seconds_count{method =\" PeerRelay \" ,outcome=\" rpc_error \" }"
686+ "openshell_server_peer_request_duration_seconds_count{operation =\" relay \" ,outcome=\" remote_error \" }"
692687 ) ,
693688 Some ( 1 )
694689 ) ;
695690
696691 let mut endpoint = RoutedRequestTimer :: start ( PeerRpc :: ReportEndpointStatus ) ;
697- endpoint. client_error ( & Status :: unavailable ( "x" ) ) ;
692+ endpoint. local_error ( & Status :: unavailable ( "x" ) ) ;
698693 drop ( endpoint) ;
699694 assert_eq ! (
700695 metrics. value(
701- "openshell_server_routed_request_attempts_total{operation=\" report_endpoint_status\" ,route=\" peer\" ,target =\" none\" ,outcome=\" client_error \" ,grpc_code=\" unavailable\" }"
696+ "openshell_server_routed_request_attempts_total{operation=\" report_endpoint_status\" ,route=\" peer\" ,relay_kind =\" none\" ,outcome=\" local_error \" ,grpc_code=\" unavailable\" }"
702697 ) ,
703698 Some ( 1 )
704699 ) ;
@@ -708,7 +703,7 @@ mod tests {
708703 drop ( provider_status) ;
709704 assert_eq ! (
710705 metrics. value(
711- "openshell_server_routed_request_attempts_total{operation=\" get_sandbox_provider_status\" ,route=\" peer\" ,target =\" none\" ,outcome=\" success\" ,grpc_code=\" ok\" }"
706+ "openshell_server_routed_request_attempts_total{operation=\" get_sandbox_provider_status\" ,route=\" peer\" ,relay_kind =\" none\" ,outcome=\" success\" ,grpc_code=\" ok\" }"
712707 ) ,
713708 Some ( 1 )
714709 ) ;
@@ -720,7 +715,7 @@ mod tests {
720715 drop ( RoutedRequestTimer :: start ( PeerRpc :: ReportProviderReadiness ) ) ;
721716 assert_eq ! (
722717 metrics. value(
723- "openshell_server_routed_request_attempts_total{operation=\" report_provider_readiness\" ,route=\" peer\" ,target =\" none\" ,outcome=\" client_error \" ,grpc_code=\" cancelled\" }"
718+ "openshell_server_routed_request_attempts_total{operation=\" report_provider_readiness\" ,route=\" peer\" ,relay_kind =\" none\" ,outcome=\" local_error \" ,grpc_code=\" cancelled\" }"
724719 ) ,
725720 Some ( 1 )
726721 ) ;
@@ -731,25 +726,25 @@ mod tests {
731726 let metrics = MetricsCapture :: install ( ) ;
732727 let mut timer = RoutedRequestTimer :: start ( PeerRpc :: Relay ) ;
733728 timer. finish ( & Ok :: < ( ) , Status > ( ( ) ) ) ;
734- timer. client_error ( & Status :: unavailable ( "x" ) ) ;
729+ timer. local_error ( & Status :: unavailable ( "x" ) ) ;
735730 drop ( timer) ;
736731 assert_eq ! (
737732 metrics. value(
738- "openshell_server_routed_request_attempts_total{operation=\" relay\" ,route=\" peer\" ,target =\" ssh\" ,outcome=\" success\" ,grpc_code=\" ok\" }"
733+ "openshell_server_routed_request_attempts_total{operation=\" relay\" ,route=\" peer\" ,relay_kind =\" ssh\" ,outcome=\" success\" ,grpc_code=\" ok\" }"
739734 ) ,
740735 Some ( 1 )
741736 ) ;
742- assert ! ( !metrics. render( ) . contains( "outcome=\" client_error \" " ) ) ;
737+ assert ! ( !metrics. render( ) . contains( "outcome=\" local_error \" " ) ) ;
743738 }
744739
745740 #[ test]
746741 fn local_relay_failure_records_status_without_peer_latency ( ) {
747742 let metrics = MetricsCapture :: install ( ) ;
748- let mut timer = RoutedRequestTimer :: relay ( RelayTarget :: Tcp , RelayRoute :: Local ) ;
743+ let mut timer = RoutedRequestTimer :: relay ( RelayKind :: Tcp , RelayRoute :: Local ) ;
749744 timer. finish ( & Err :: < ( ) , _ > ( Status :: resource_exhausted ( "capacity" ) ) ) ;
750745 drop ( timer) ;
751746 assert_eq ! (
752- metrics. value( "openshell_server_routed_request_attempts_total{operation=\" relay\" ,route=\" local\" ,target =\" tcp\" ,outcome=\" client_error \" ,grpc_code=\" resource_exhausted\" }" ) ,
747+ metrics. value( "openshell_server_routed_request_attempts_total{operation=\" relay\" ,route=\" local\" ,relay_kind =\" tcp\" ,outcome=\" local_error \" ,grpc_code=\" resource_exhausted\" }" ) ,
753748 Some ( 1 )
754749 ) ;
755750 assert ! ( !metrics. render( ) . contains( PEER_REQUEST_DURATION_SECONDS ) ) ;
0 commit comments