@@ -1161,48 +1161,38 @@ mod tests {
11611161 }
11621162}
11631163
1164- /// Fixtures for the absence-aware partition audit alerts in
1165- /// `docs/partition-catalog-monitoring.md`, evaluated against the production
1166- /// exporter configuration and the production audit metric emission .
1164+ /// Fixtures for the partition audit alerts in
1165+ /// `docs/partition-catalog-monitoring.md`: the production exporter configuration
1166+ /// and audit metric emission export the series those rules read .
11671167#[ cfg( test) ]
11681168mod partition_alert_tests {
1169- use std:: collections:: BTreeSet ;
1169+ use std:: collections:: BTreeMap ;
11701170 use std:: time:: Duration ;
11711171
11721172 const RUNS : & str = "buzz_partition_audit_runs_total" ;
11731173 const LAST_SUCCESS : & str = "buzz_partition_audit_last_success_timestamp_seconds" ;
11741174 const MANAGED_TABLES : [ & str ; 2 ] = [ "delivery_log" , "events" ] ;
11751175
1176- /// `table` label values of every sample of `metric` in a scrape.
1177- fn tables ( scrape : & str , metric : & str ) -> BTreeSet < String > {
1176+ /// Sample value by `table` label for every sample of `metric` in a scrape.
1177+ /// Counters with several `outcome` labels keep the last sample per table.
1178+ fn samples ( scrape : & str , metric : & str ) -> BTreeMap < String , f64 > {
11781179 scrape
11791180 . lines ( )
11801181 . filter_map ( |line| line. strip_prefix ( metric) ?. strip_prefix ( '{' ) )
11811182 . filter_map ( |labels| {
11821183 let ( _, rest) = labels. split_once ( r#"table=""# ) ?;
1183- rest. split_once ( '"' ) . map ( |( table, _) | table. to_string ( ) )
1184+ let ( table, rest) = rest. split_once ( '"' ) ?;
1185+ let ( _, value) = rest. rsplit_once ( ' ' ) ?;
1186+ Some ( ( table. to_string ( ) , value. parse ( ) . expect ( "numeric sample" ) ) )
11841187 } )
11851188 . collect ( )
11861189 }
11871190
1188- /// Evaluate the documented absence rule for one scrape target:
1189- /// `max without (outcome) (buzz_partition_audit_runs_total)
1190- /// unless buzz_partition_audit_last_success_timestamp_seconds`.
1191- fn missing_success_series ( scrape : & str ) -> BTreeSet < String > {
1192- & tables ( scrape, RUNS ) - & tables ( scrape, LAST_SUCCESS )
1193- }
1194-
1195- fn last_success_value ( scrape : & str , table : & str ) -> Option < f64 > {
1196- scrape
1197- . lines ( )
1198- . find ( |line| {
1199- line. starts_with ( LAST_SUCCESS ) && line. contains ( & format ! ( r#"table="{table}""# ) )
1200- } )
1201- . and_then ( |line| line. rsplit_once ( ' ' ) )
1202- . map ( |( _, value) | value. parse ( ) . expect ( "numeric sample" ) )
1191+ fn tables ( scrape : & str , metric : & str ) -> Vec < String > {
1192+ samples ( scrape, metric) . into_keys ( ) . collect ( )
12031193 }
12041194
1205- fn managed_tables ( ) -> BTreeSet < String > {
1195+ fn managed_tables ( ) -> Vec < String > {
12061196 MANAGED_TABLES
12071197 . iter ( )
12081198 . map ( |table| table. to_string ( ) )
@@ -1225,7 +1215,7 @@ mod partition_alert_tests {
12251215 }
12261216
12271217 #[ test]
1228- fn never_successful_audits_leave_expected_series_without_a_success_timestamp ( ) {
1218+ fn never_successful_audits_export_a_zero_success_timestamp ( ) {
12291219 let ( recorder, handle) = super :: readiness_test_recorder_with_idle_timeout ( 1 ) ;
12301220 metrics:: with_local_recorder ( & recorder, || {
12311221 current_thread_runtime ( ) . block_on ( async {
@@ -1235,58 +1225,62 @@ mod partition_alert_tests {
12351225
12361226 let scrape = handle. render ( ) ;
12371227 assert_eq ! ( tables( & scrape, RUNS ) , managed_tables( ) , "{scrape}" ) ;
1238- assert ! ( tables( & scrape, LAST_SUCCESS ) . is_empty( ) , "{scrape}" ) ;
1239- assert_eq ! ( missing_success_series( & scrape) , managed_tables( ) ) ;
1228+ let zero = MANAGED_TABLES
1229+ . iter ( )
1230+ . map ( |table| ( table. to_string ( ) , 0.0 ) )
1231+ . collect :: < BTreeMap < _ , _ > > ( ) ;
1232+ assert_eq ! ( samples( & scrape, LAST_SUCCESS ) , zero, "{scrape}" ) ;
12401233 }
12411234
12421235 mod postgres_tests {
12431236 use super :: * ;
12441237
12451238 #[ test]
12461239 #[ ignore = "requires Postgres" ]
1247- fn evicted_success_timestamps_leave_expected_series_behind ( ) {
1240+ fn failed_audits_keep_the_last_success_past_the_idle_timeout ( ) {
12481241 let idle_timeout = Duration :: from_secs ( 1 ) ;
12491242 let ( recorder, handle) =
12501243 crate :: metrics:: readiness_test_recorder_with_idle_timeout ( idle_timeout. as_secs ( ) ) ;
12511244 let runtime = current_thread_runtime ( ) ;
1252- let failing = metrics:: with_local_recorder ( & recorder, || {
1245+ let ( working , failing) = metrics:: with_local_recorder ( & recorder, || {
12531246 runtime. block_on ( async {
12541247 let pool = sqlx:: PgPool :: connect ( & crate :: test_support:: database_url ( ) )
12551248 . await
12561249 . expect ( "connect test database" ) ;
1257- buzz_db:: Db :: from_pool ( pool)
1258- . audit_partitions ( 3 )
1259- . await
1260- . expect ( "successful audit" ) ;
1261- unreachable_db ( )
1250+ let working = buzz_db:: Db :: from_pool ( pool) ;
1251+ working. audit_partitions ( 3 ) . await . expect ( "successful audit" ) ;
1252+ ( working, unreachable_db ( ) )
12621253 } )
12631254 } ) ;
12641255
12651256 let scrape = handle. render ( ) ;
1266- assert ! ( missing_success_series ( & scrape) . is_empty ( ) , "{scrape}" ) ;
1267- let first = last_success_value ( & scrape , "events" ) . expect ( "success timestamp" ) ;
1257+ let first = samples ( & scrape, LAST_SUCCESS ) ;
1258+ assert_eq ! ( first. keys ( ) . cloned ( ) . collect :: < Vec <_>> ( ) , managed_tables ( ) ) ;
12681259
1269- let audit_failure = || {
1260+ let audit = |db : & buzz_db :: Db | {
12701261 metrics:: with_local_recorder ( & recorder, || {
1271- runtime. block_on ( async {
1272- assert ! ( failing. audit_partitions( 3 ) . await . is_err( ) ) ;
1273- } ) ;
1274- } ) ;
1262+ runtime. block_on ( async { db. audit_partitions ( 3 ) . await . is_ok ( ) } )
1263+ } )
12751264 } ;
1276- // A failed audit never makes the last success look fresh.
1277- audit_failure ( ) ;
1278- let scrape = handle. render ( ) ;
1279- assert_eq ! ( last_success_value( & scrape, "events" ) , Some ( first) ) ;
1280- assert ! ( missing_success_series( & scrape) . is_empty( ) , "{scrape}" ) ;
1265+ // A failed audit never makes the last success look fresh, and
1266+ // sustained failures keep it exported past the idle timeout, so its
1267+ // age keeps rising instead of the series disappearing.
1268+ assert ! ( !audit( & failing) ) ;
1269+ assert_eq ! ( samples( & handle. render( ) , LAST_SUCCESS ) , first) ;
1270+ std:: thread:: sleep ( idle_timeout + Duration :: from_millis ( 250 ) ) ;
1271+ assert ! ( !audit( & failing) ) ;
1272+ assert_eq ! ( samples( & handle. render( ) , LAST_SUCCESS ) , first) ;
12811273
1282- // Sustained failures let the exporter evict the success timestamp;
1283- // the per-table run counters still identify what to expect .
1274+ // When audits stop running altogether the timestamp is evicted,
1275+ // but the run counters remain for the not-running rule .
12841276 std:: thread:: sleep ( idle_timeout + Duration :: from_millis ( 250 ) ) ;
1285- audit_failure ( ) ;
12861277 let scrape = handle. render ( ) ;
12871278 assert ! ( tables( & scrape, LAST_SUCCESS ) . is_empty( ) , "{scrape}" ) ;
12881279 assert_eq ! ( tables( & scrape, RUNS ) , managed_tables( ) , "{scrape}" ) ;
1289- assert_eq ! ( missing_success_series( & scrape) , managed_tables( ) ) ;
1280+
1281+ // The next success exports a fresh timestamp again.
1282+ assert ! ( audit( & working) ) ;
1283+ assert_eq ! ( tables( & handle. render( ) , LAST_SUCCESS ) , managed_tables( ) ) ;
12901284 }
12911285 }
12921286}
0 commit comments