@@ -531,9 +531,23 @@ impl DependencyDiagnostics {
531531 . then ( || self . sample_completion_epoch_seconds . load ( Ordering :: Acquire ) )
532532 }
533533
534+ /// Re-emits the stored epoch, then checks it is still the stored one.
535+ ///
536+ /// A sample can complete and publish a newer epoch while this write is in
537+ /// flight, and the metrics facade exposes only a bare `set`, so nothing
538+ /// downstream rejects the older value once it lands last. Verifying after
539+ /// the write — rather than before it — is what closes that window: a
540+ /// republish that lost the race re-emits the newer epoch instead of
541+ /// leaving the exported series moved backwards. Another pass costs another
542+ /// completed sample, so this ends as soon as no completion is racing it.
534543 fn republish_dependency_sample_completion ( & self ) {
535- if let Some ( epoch_seconds) = self . latest_sample_completion_epoch_seconds ( ) {
544+ let mut published = None ;
545+ while let Some ( epoch_seconds) = self . latest_sample_completion_epoch_seconds ( ) {
546+ if published == Some ( epoch_seconds) {
547+ break ;
548+ }
536549 publish_dependency_sample_completion_metric ( epoch_seconds) ;
550+ published = Some ( epoch_seconds) ;
537551 }
538552 }
539553}
@@ -1244,6 +1258,229 @@ mod tests {
12441258 } ) ;
12451259 }
12461260
1261+ /// How long a park step waits before it is a failure rather than a hang.
1262+ const PARK_SIGNAL_TIMEOUT : Duration = Duration :: from_secs ( 10 ) ;
1263+
1264+ /// A completion-epoch gauge write that can be stopped at the recorder
1265+ /// boundary — the last instruction of a publish, after the writer has
1266+ /// already read the epoch it is publishing.
1267+ ///
1268+ /// Arming names one epoch value and fires once, so the sampler's own write
1269+ /// passes straight through and only the republish under test parks.
1270+ struct PublishPark {
1271+ armed_for : Mutex < Option < u64 > > ,
1272+ parked_tx : std:: sync:: mpsc:: SyncSender < ( ) > ,
1273+ parked_rx : Mutex < std:: sync:: mpsc:: Receiver < ( ) > > ,
1274+ release_tx : std:: sync:: mpsc:: SyncSender < ( ) > ,
1275+ release_rx : Mutex < std:: sync:: mpsc:: Receiver < ( ) > > ,
1276+ }
1277+
1278+ impl PublishPark {
1279+ fn new ( ) -> Self {
1280+ let ( parked_tx, parked_rx) = std:: sync:: mpsc:: sync_channel ( 1 ) ;
1281+ let ( release_tx, release_rx) = std:: sync:: mpsc:: sync_channel ( 1 ) ;
1282+ Self {
1283+ armed_for : Mutex :: new ( None ) ,
1284+ parked_tx,
1285+ parked_rx : Mutex :: new ( parked_rx) ,
1286+ release_tx,
1287+ release_rx : Mutex :: new ( release_rx) ,
1288+ }
1289+ }
1290+
1291+ fn arm ( & self , epoch_seconds : u64 ) {
1292+ * self . armed_for . lock ( ) . expect ( "park state" ) = Some ( epoch_seconds) ;
1293+ }
1294+
1295+ fn park_if_armed ( & self , epoch_seconds : u64 ) {
1296+ {
1297+ let mut armed = self . armed_for . lock ( ) . expect ( "park state" ) ;
1298+ if * armed != Some ( epoch_seconds) {
1299+ return ;
1300+ }
1301+ * armed = None ;
1302+ }
1303+ self . parked_tx . send ( ( ) ) . expect ( "announce the parked write" ) ;
1304+ self . release_rx
1305+ . lock ( )
1306+ . expect ( "release channel" )
1307+ . recv_timeout ( PARK_SIGNAL_TIMEOUT )
1308+ . expect ( "the parked write must be released" ) ;
1309+ }
1310+
1311+ fn wait_until_parked ( & self ) {
1312+ self . parked_rx
1313+ . lock ( )
1314+ . expect ( "park channel" )
1315+ . recv_timeout ( PARK_SIGNAL_TIMEOUT )
1316+ . expect ( "the armed write must reach the recorder" ) ;
1317+ }
1318+
1319+ fn release ( & self ) {
1320+ self . release_tx . send ( ( ) ) . expect ( "release the parked write" ) ;
1321+ }
1322+ }
1323+
1324+ /// Delegates to the real Prometheus recorder, with the completion-epoch
1325+ /// gauge routed through [`PublishPark`].
1326+ struct ParkingRecorder {
1327+ inner : metrics_exporter_prometheus:: PrometheusRecorder ,
1328+ park : Arc < PublishPark > ,
1329+ }
1330+
1331+ struct ParkingGauge {
1332+ inner : metrics:: Gauge ,
1333+ park : Arc < PublishPark > ,
1334+ }
1335+
1336+ impl metrics:: GaugeFn for ParkingGauge {
1337+ fn increment ( & self , value : f64 ) {
1338+ self . inner . increment ( value) ;
1339+ }
1340+
1341+ fn decrement ( & self , value : f64 ) {
1342+ self . inner . decrement ( value) ;
1343+ }
1344+
1345+ fn set ( & self , value : f64 ) {
1346+ self . park . park_if_armed ( value as u64 ) ;
1347+ self . inner . set ( value) ;
1348+ }
1349+ }
1350+
1351+ impl metrics:: Recorder for ParkingRecorder {
1352+ fn describe_counter (
1353+ & self ,
1354+ key : metrics:: KeyName ,
1355+ unit : Option < metrics:: Unit > ,
1356+ description : metrics:: SharedString ,
1357+ ) {
1358+ self . inner . describe_counter ( key, unit, description) ;
1359+ }
1360+
1361+ fn describe_gauge (
1362+ & self ,
1363+ key : metrics:: KeyName ,
1364+ unit : Option < metrics:: Unit > ,
1365+ description : metrics:: SharedString ,
1366+ ) {
1367+ self . inner . describe_gauge ( key, unit, description) ;
1368+ }
1369+
1370+ fn describe_histogram (
1371+ & self ,
1372+ key : metrics:: KeyName ,
1373+ unit : Option < metrics:: Unit > ,
1374+ description : metrics:: SharedString ,
1375+ ) {
1376+ self . inner . describe_histogram ( key, unit, description) ;
1377+ }
1378+
1379+ fn register_counter (
1380+ & self ,
1381+ key : & metrics:: Key ,
1382+ metadata : & metrics:: Metadata < ' _ > ,
1383+ ) -> metrics:: Counter {
1384+ self . inner . register_counter ( key, metadata)
1385+ }
1386+
1387+ fn register_gauge (
1388+ & self ,
1389+ key : & metrics:: Key ,
1390+ metadata : & metrics:: Metadata < ' _ > ,
1391+ ) -> metrics:: Gauge {
1392+ let gauge = self . inner . register_gauge ( key, metadata) ;
1393+ if key. name ( ) == "buzz_readiness_dependency_sample_completed_timestamp_seconds" {
1394+ metrics:: Gauge :: from_arc ( Arc :: new ( ParkingGauge {
1395+ inner : gauge,
1396+ park : Arc :: clone ( & self . park ) ,
1397+ } ) )
1398+ } else {
1399+ gauge
1400+ }
1401+ }
1402+
1403+ fn register_histogram (
1404+ & self ,
1405+ key : & metrics:: Key ,
1406+ metadata : & metrics:: Metadata < ' _ > ,
1407+ ) -> metrics:: Histogram {
1408+ self . inner . register_histogram ( key, metadata)
1409+ }
1410+ }
1411+
1412+ /// Two tasks write this gauge — the sampler, which owns the epoch, and the
1413+ /// idle-refresh publisher, which may only re-emit it — and the metrics
1414+ /// facade offers a bare `set` with no compare-and-set to lean on. So a
1415+ /// republish that read the stored epoch before a sample completed can still
1416+ /// be inside its own write when the newer epoch lands, and plain last-write
1417+ /// -wins would leave the exported series moved backwards until the next
1418+ /// republish tick.
1419+ ///
1420+ /// This parks the republish at the recorder, the last point in its write,
1421+ /// completes a newer sample behind it — which must not be blocked by the
1422+ /// parked republish — and only then releases it. The scrape must report the
1423+ /// newer completion. Times are injected so the two epochs are exact and the
1424+ /// interleaving does not depend on the wall clock.
1425+ #[ test]
1426+ fn a_republish_racing_a_completion_cannot_move_the_exported_epoch_backwards ( ) {
1427+ const EARLIER_EPOCH_SECONDS : u64 = 1_700_000_000 ;
1428+ const LATER_EPOCH_SECONDS : u64 = 1_700_000_030 ;
1429+
1430+ let ( prometheus, handle) = crate :: metrics:: readiness_test_recorder ( ) ;
1431+ let park = Arc :: new ( PublishPark :: new ( ) ) ;
1432+ let recorder = Arc :: new ( ParkingRecorder {
1433+ inner : prometheus,
1434+ park : Arc :: clone ( & park) ,
1435+ } ) ;
1436+ let diagnostics = Arc :: new ( ready_diagnostics ( ) ) ;
1437+
1438+ metrics:: with_local_recorder ( recorder. as_ref ( ) , || {
1439+ diagnostics. record_dependency_sample_completion (
1440+ SystemTime :: UNIX_EPOCH + Duration :: from_secs ( EARLIER_EPOCH_SECONDS ) ,
1441+ ) ;
1442+ } ) ;
1443+ assert_eq ! (
1444+ sample_completion_metric_value_from_scrape( & handle. render( ) ) ,
1445+ Some ( EARLIER_EPOCH_SECONDS ) ,
1446+ "the first completion owns the epoch"
1447+ ) ;
1448+
1449+ park. arm ( EARLIER_EPOCH_SECONDS ) ;
1450+ let republisher = std:: thread:: spawn ( {
1451+ let recorder = Arc :: clone ( & recorder) ;
1452+ let diagnostics = Arc :: clone ( & diagnostics) ;
1453+ move || {
1454+ metrics:: with_local_recorder ( recorder. as_ref ( ) , || {
1455+ diagnostics. republish_dependency_sample_completion ( ) ;
1456+ } ) ;
1457+ }
1458+ } ) ;
1459+
1460+ park. wait_until_parked ( ) ;
1461+
1462+ metrics:: with_local_recorder ( recorder. as_ref ( ) , || {
1463+ diagnostics. record_dependency_sample_completion (
1464+ SystemTime :: UNIX_EPOCH + Duration :: from_secs ( LATER_EPOCH_SECONDS ) ,
1465+ ) ;
1466+ } ) ;
1467+ assert_eq ! (
1468+ sample_completion_metric_value_from_scrape( & handle. render( ) ) ,
1469+ Some ( LATER_EPOCH_SECONDS ) ,
1470+ "a completed sample must publish its epoch while a republish is still in flight"
1471+ ) ;
1472+
1473+ park. release ( ) ;
1474+ republisher. join ( ) . expect ( "republish thread" ) ;
1475+
1476+ assert_eq ! (
1477+ sample_completion_metric_value_from_scrape( & handle. render( ) ) ,
1478+ Some ( LATER_EPOCH_SECONDS ) ,
1479+ "a republish that lost the race must not re-export the epoch it read \
1480+ before the newer sample completed"
1481+ ) ;
1482+ }
1483+
12471484 #[ test]
12481485 fn readiness_reason_labels_are_the_closed_lifecycle_set ( ) {
12491486 assert_eq ! (
0 commit comments