@@ -25,11 +25,16 @@ use tonic::{Code, Status};
2525pub const SUPERVISOR_SESSIONS : & str = "openshell_server_supervisor_sessions" ;
2626pub const RELAY_PENDING : & str = "openshell_server_relay_pending" ;
2727pub const RELAY_PENDING_CAPACITY : & str = "openshell_server_relay_pending_capacity" ;
28+ pub const MUTATION_LOCK_CONNECTIONS_IN_USE : & str =
29+ "openshell_server_mutation_lock_connections_in_use" ;
30+ pub const MUTATION_LOCK_CONNECTIONS_CAPACITY : & str =
31+ "openshell_server_mutation_lock_connections_capacity" ;
2832// Counters
2933pub const RELAY_REJECTED_TOTAL : & str = "openshell_server_relay_rejected_total" ;
3034pub const RELAY_EXPIRED_TOTAL : & str = "openshell_server_relay_expired_total" ;
3135pub const ROUTED_REQUEST_ATTEMPTS_TOTAL : & str = "openshell_server_routed_request_attempts_total" ;
3236pub const MUTATION_LOCK_TIMEOUTS_TOTAL : & str = "openshell_server_mutation_lock_timeouts_total" ;
37+ pub const MUTATION_LOCK_ERRORS_TOTAL : & str = "openshell_server_mutation_lock_errors_total" ;
3338// Histograms (explicit buckets, see BUCKETED_HISTOGRAMS)
3439pub const RELAY_CLAIM_DURATION_SECONDS : & str = "openshell_server_relay_claim_duration_seconds" ;
3540pub const PEER_REQUEST_DURATION_SECONDS : & str = "openshell_server_peer_request_duration_seconds" ;
@@ -226,16 +231,21 @@ pub fn configure_exporter(builder: PrometheusBuilder) -> Result<PrometheusBuilde
226231}
227232
228233/// Install the process-wide recorder, then describe and zero-initialize the catalog. Call
229- /// once, from `run_server`.
230- pub fn install_global_recorder ( relay : RelayCapacity ) -> Result < PrometheusHandle , BuildError > {
234+ /// once, from `run_server`. `mutation_lock_connections` is the `PostgreSQL` lock pool size,
235+ /// `None` on a backend without a lock pool.
236+ pub fn install_global_recorder (
237+ relay : RelayCapacity ,
238+ mutation_lock_connections : Option < u32 > ,
239+ ) -> Result < PrometheusHandle , BuildError > {
231240 let handle = configure_exporter ( PrometheusBuilder :: new ( ) ) ?. install_recorder ( ) ?;
232- describe_and_initialize ( relay) ;
241+ describe_and_initialize ( relay, mutation_lock_connections ) ;
233242 Ok ( handle)
234243}
235244
236- /// Emit HELP metadata, create every fixed-label series at 0, and publish the relay caps. An
237- /// idle replica then exports 0 instead of "no data", which HPA and `rate()` need.
238- pub fn describe_and_initialize ( relay : RelayCapacity ) {
245+ /// Emit HELP metadata, create every fixed-label series at 0, and publish the relay caps and,
246+ /// with a lock pool, its size. An idle replica then exports 0 instead of "no data", which HPA
247+ /// and `rate()` need.
248+ pub fn describe_and_initialize ( relay : RelayCapacity , mutation_lock_connections : Option < u32 > ) {
239249 describe_gauge ! (
240250 SUPERVISOR_SESSIONS ,
241251 Unit :: Count ,
@@ -271,6 +281,11 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
271281 Unit :: Count ,
272282 "Mutation lock acquisitions that timed out."
273283 ) ;
284+ describe_counter ! (
285+ MUTATION_LOCK_ERRORS_TOTAL ,
286+ Unit :: Count ,
287+ "Mutation lock acquisitions that failed without timing out, such as a lock connection PostgreSQL did not open."
288+ ) ;
274289 describe_histogram ! (
275290 RELAY_CLAIM_DURATION_SECONDS ,
276291 Unit :: Seconds ,
@@ -310,6 +325,21 @@ pub fn describe_and_initialize(relay: RelayCapacity) {
310325 counter ! ( RELAY_EXPIRED_TOTAL ) . increment ( 0 ) ;
311326 for scope in LockScope :: ALL {
312327 counter ! ( MUTATION_LOCK_TIMEOUTS_TOTAL , LABEL_SCOPE => scope. label( ) ) . increment ( 0 ) ;
328+ counter ! ( MUTATION_LOCK_ERRORS_TOTAL , LABEL_SCOPE => scope. label( ) ) . increment ( 0 ) ;
329+ }
330+ if let Some ( capacity) = mutation_lock_connections {
331+ describe_gauge ! (
332+ MUTATION_LOCK_CONNECTIONS_IN_USE ,
333+ Unit :: Count ,
334+ "PostgreSQL lock-pool connections checked out on this replica: held by mutation guards, by acquisitions waiting for an advisory lock, or being returned to the pool."
335+ ) ;
336+ describe_gauge ! (
337+ MUTATION_LOCK_CONNECTIONS_CAPACITY ,
338+ Unit :: Count ,
339+ "Maximum PostgreSQL lock-pool connections on one gateway replica."
340+ ) ;
341+ gauge ! ( MUTATION_LOCK_CONNECTIONS_IN_USE ) . increment ( 0.0 ) ;
342+ gauge ! ( MUTATION_LOCK_CONNECTIONS_CAPACITY ) . set ( f64:: from ( capacity) ) ;
313343 }
314344 for rpc in PeerRpc :: ALL {
315345 if rpc == PeerRpc :: Relay {
@@ -350,6 +380,11 @@ impl GaugeSlot {
350380 Self :: acquire ( RELAY_PENDING )
351381 }
352382
383+ /// Share of `openshell_server_mutation_lock_connections_in_use`.
384+ pub fn mutation_lock_connection ( ) -> Self {
385+ Self :: acquire ( MUTATION_LOCK_CONNECTIONS_IN_USE )
386+ }
387+
353388 fn acquire ( name : & ' static str ) -> Self {
354389 let gauge = gauge ! ( name) ;
355390 gauge. increment ( 1.0 ) ;
@@ -386,12 +421,19 @@ pub fn record_lock_wait(scope: LockScope, waited: Duration) {
386421
387422/// A guard acquisition that timed out: a local wait, a full lock pool, too little time left to
388423/// open a lock connection, or Postgres `lock_timeout` (SQLSTATE 55P03). A lock connection that
389- /// Postgres does not open with at least `LOCK_CONNECTION_MIN_BUDGET` left is not counted. RPC
390- /// callers return the timeout as `Status::unavailable`.
424+ /// Postgres does not open with at least `LOCK_CONNECTION_MIN_BUDGET` left counts in
425+ /// [`record_lock_error`] instead. RPC callers return the timeout as `Status::unavailable`.
391426pub fn record_lock_timeout ( scope : LockScope ) {
392427 counter ! ( MUTATION_LOCK_TIMEOUTS_TOTAL , LABEL_SCOPE => scope. label( ) ) . increment ( 1 ) ;
393428}
394429
430+ /// A guard acquisition that failed without timing out: Postgres did not open a lock connection,
431+ /// or a lock statement failed. RPC callers return it as `Status::internal`. A cancelled
432+ /// acquisition is not counted.
433+ pub fn record_lock_error ( scope : LockScope ) {
434+ counter ! ( MUTATION_LOCK_ERRORS_TOTAL , LABEL_SCOPE => scope. label( ) ) . increment ( 1 ) ;
435+ }
436+
395437/// Counts one local relay setup or outbound peer attempt exactly once, and times peer requests.
396438/// A relay succeeds when the supervisor claims it, on either route. Dropping an unfinished
397439/// timer (the caller gave up) records `local_error` / `cancelled`.
@@ -541,10 +583,12 @@ mod tests {
541583 #[ test]
542584 fn describe_and_initialize_exports_capacity_and_zero_series ( ) {
543585 let metrics = MetricsCapture :: install ( ) ;
544- describe_and_initialize ( RelayCapacity { per_replica : 256 } ) ;
586+ describe_and_initialize ( RelayCapacity { per_replica : 256 } , Some ( 4 ) ) ;
545587
546588 for ( series, expected) in [
547589 ( "openshell_server_relay_pending_capacity" , 256 ) ,
590+ ( "openshell_server_mutation_lock_connections_capacity" , 4 ) ,
591+ ( "openshell_server_mutation_lock_connections_in_use" , 0 ) ,
548592 ( "openshell_server_supervisor_sessions" , 0 ) ,
549593 ( "openshell_server_relay_pending" , 0 ) ,
550594 (
@@ -568,6 +612,18 @@ mod tests {
568612 "openshell_server_mutation_lock_timeouts_total{scope=\" sandbox\" }" ,
569613 0 ,
570614 ) ,
615+ (
616+ "openshell_server_mutation_lock_errors_total{scope=\" global\" }" ,
617+ 0 ,
618+ ) ,
619+ (
620+ "openshell_server_mutation_lock_errors_total{scope=\" workspace\" }" ,
621+ 0 ,
622+ ) ,
623+ (
624+ "openshell_server_mutation_lock_errors_total{scope=\" sandbox\" }" ,
625+ 0 ,
626+ ) ,
571627 ] {
572628 assert_eq ! ( metrics. value( series) , Some ( expected) , "{series}" ) ;
573629 }
@@ -596,6 +652,24 @@ mod tests {
596652 ) ;
597653 }
598654
655+ #[ test]
656+ fn describe_and_initialize_without_a_lock_pool_omits_lock_connection_gauges ( ) {
657+ let metrics = MetricsCapture :: install ( ) ;
658+ describe_and_initialize ( RelayCapacity { per_replica : 256 } , None ) ;
659+
660+ assert_eq ! ( metrics. value( MUTATION_LOCK_CONNECTIONS_IN_USE ) , None ) ;
661+ assert_eq ! ( metrics. value( MUTATION_LOCK_CONNECTIONS_CAPACITY ) , None ) ;
662+ assert ! (
663+ !metrics
664+ . render( )
665+ . contains( "openshell_server_mutation_lock_connections" )
666+ ) ;
667+ assert_eq ! (
668+ metrics. value( "openshell_server_mutation_lock_errors_total{scope=\" sandbox\" }" ) ,
669+ Some ( 0 )
670+ ) ;
671+ }
672+
599673 #[ test]
600674 fn routed_attempts_have_seven_bounded_success_series_and_keep_counts_on_initialize ( ) {
601675 let metrics = MetricsCapture :: install ( ) ;
@@ -605,7 +679,7 @@ mod tests {
605679 }
606680 }
607681 RoutedRequestTimer :: relay ( RelayKind :: Tcp , RelayRoute :: Peer ) . finish ( & Ok :: < ( ) , Status > ( ( ) ) ) ;
608- describe_and_initialize ( RelayCapacity { per_replica : 256 } ) ;
682+ describe_and_initialize ( RelayCapacity { per_replica : 256 } , None ) ;
609683
610684 for kind in [ "ssh" , "tcp" ] {
611685 for route in [ "local" , "peer" ] {
@@ -705,6 +779,14 @@ mod tests {
705779 assert_eq ! ( metrics. value( SUPERVISOR_SESSIONS ) , Some ( 1 ) ) ;
706780 drop ( session) ;
707781 assert_eq ! ( metrics. value( SUPERVISOR_SESSIONS ) , Some ( 0 ) ) ;
782+
783+ let first = GaugeSlot :: mutation_lock_connection ( ) ;
784+ let second = GaugeSlot :: mutation_lock_connection ( ) ;
785+ assert_eq ! ( metrics. value( MUTATION_LOCK_CONNECTIONS_IN_USE ) , Some ( 2 ) ) ;
786+ drop ( first) ;
787+ assert_eq ! ( metrics. value( MUTATION_LOCK_CONNECTIONS_IN_USE ) , Some ( 1 ) ) ;
788+ drop ( second) ;
789+ assert_eq ! ( metrics. value( MUTATION_LOCK_CONNECTIONS_IN_USE ) , Some ( 0 ) ) ;
708790 }
709791
710792 #[ test]
0 commit comments