Skip to content

Commit ed3d7ae

Browse files
fix(relay): stop a racing republish from regressing the completion epoch
The completion-epoch gauge has two writers: the dependency sampler, which owns the epoch, and the idle-refresh publisher, which may only re-emit it. The publisher read the stored epoch and wrote it as two separate steps, so a sample completing in between left the older epoch as the last write — the exported series moved backwards for up to one publisher interval, which is exactly the window freshness alerting reads. The metrics facade offers a bare `set` with no compare-and-set, so the ordering has to come from the writers. Serializing them under one lock would put the sampler's authoritative write behind an idle refresh; instead the republisher now verifies after its write that the epoch it wrote is still the stored one, and re-emits the newer one if a completion landed while that write was in flight. Another pass costs another completed sample, so it ends as soon as nothing is racing it. The regression parks a republish inside the recorder — the last point of its write, after it has read the epoch — completes a newer sample behind it, then releases it and asserts the scrape still reports the newer completion. Against the unfixed publisher it reports the older epoch. Co-Authored-By: Claude Opus 5 <[email protected]> Signed-off-by: tornquist <[email protected]>
1 parent 962bdca commit ed3d7ae

2 files changed

Lines changed: 243 additions & 3 deletions

File tree

‎crates/buzz-relay/src/readiness.rs‎

Lines changed: 238 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,9 +531,23 @@ impl DependencyDiagnostics {
531531
.then(|| self.sample_completion_epoch_seconds.load(Ordering::Acquire))
532532
}
533533

534+
/// Re-emits the stored epoch, then checks it is still the stored one.
535+
///
536+
/// A sample can complete and publish a newer epoch while this write is in
537+
/// flight, and the metrics facade exposes only a bare `set`, so nothing
538+
/// downstream rejects the older value once it lands last. Verifying after
539+
/// the write — rather than before it — is what closes that window: a
540+
/// republish that lost the race re-emits the newer epoch instead of
541+
/// leaving the exported series moved backwards. Another pass costs another
542+
/// completed sample, so this ends as soon as no completion is racing it.
534543
fn republish_dependency_sample_completion(&self) {
535-
if let Some(epoch_seconds) = self.latest_sample_completion_epoch_seconds() {
544+
let mut published = None;
545+
while let Some(epoch_seconds) = self.latest_sample_completion_epoch_seconds() {
546+
if published == Some(epoch_seconds) {
547+
break;
548+
}
536549
publish_dependency_sample_completion_metric(epoch_seconds);
550+
published = Some(epoch_seconds);
537551
}
538552
}
539553
}
@@ -1244,6 +1258,229 @@ mod tests {
12441258
});
12451259
}
12461260

1261+
/// How long a park step waits before it is a failure rather than a hang.
1262+
const PARK_SIGNAL_TIMEOUT: Duration = Duration::from_secs(10);
1263+
1264+
/// A completion-epoch gauge write that can be stopped at the recorder
1265+
/// boundary — the last instruction of a publish, after the writer has
1266+
/// already read the epoch it is publishing.
1267+
///
1268+
/// Arming names one epoch value and fires once, so the sampler's own write
1269+
/// passes straight through and only the republish under test parks.
1270+
struct PublishPark {
1271+
armed_for: Mutex<Option<u64>>,
1272+
parked_tx: std::sync::mpsc::SyncSender<()>,
1273+
parked_rx: Mutex<std::sync::mpsc::Receiver<()>>,
1274+
release_tx: std::sync::mpsc::SyncSender<()>,
1275+
release_rx: Mutex<std::sync::mpsc::Receiver<()>>,
1276+
}
1277+
1278+
impl PublishPark {
1279+
fn new() -> Self {
1280+
let (parked_tx, parked_rx) = std::sync::mpsc::sync_channel(1);
1281+
let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1);
1282+
Self {
1283+
armed_for: Mutex::new(None),
1284+
parked_tx,
1285+
parked_rx: Mutex::new(parked_rx),
1286+
release_tx,
1287+
release_rx: Mutex::new(release_rx),
1288+
}
1289+
}
1290+
1291+
fn arm(&self, epoch_seconds: u64) {
1292+
*self.armed_for.lock().expect("park state") = Some(epoch_seconds);
1293+
}
1294+
1295+
fn park_if_armed(&self, epoch_seconds: u64) {
1296+
{
1297+
let mut armed = self.armed_for.lock().expect("park state");
1298+
if *armed != Some(epoch_seconds) {
1299+
return;
1300+
}
1301+
*armed = None;
1302+
}
1303+
self.parked_tx.send(()).expect("announce the parked write");
1304+
self.release_rx
1305+
.lock()
1306+
.expect("release channel")
1307+
.recv_timeout(PARK_SIGNAL_TIMEOUT)
1308+
.expect("the parked write must be released");
1309+
}
1310+
1311+
fn wait_until_parked(&self) {
1312+
self.parked_rx
1313+
.lock()
1314+
.expect("park channel")
1315+
.recv_timeout(PARK_SIGNAL_TIMEOUT)
1316+
.expect("the armed write must reach the recorder");
1317+
}
1318+
1319+
fn release(&self) {
1320+
self.release_tx.send(()).expect("release the parked write");
1321+
}
1322+
}
1323+
1324+
/// Delegates to the real Prometheus recorder, with the completion-epoch
1325+
/// gauge routed through [`PublishPark`].
1326+
struct ParkingRecorder {
1327+
inner: metrics_exporter_prometheus::PrometheusRecorder,
1328+
park: Arc<PublishPark>,
1329+
}
1330+
1331+
struct ParkingGauge {
1332+
inner: metrics::Gauge,
1333+
park: Arc<PublishPark>,
1334+
}
1335+
1336+
impl metrics::GaugeFn for ParkingGauge {
1337+
fn increment(&self, value: f64) {
1338+
self.inner.increment(value);
1339+
}
1340+
1341+
fn decrement(&self, value: f64) {
1342+
self.inner.decrement(value);
1343+
}
1344+
1345+
fn set(&self, value: f64) {
1346+
self.park.park_if_armed(value as u64);
1347+
self.inner.set(value);
1348+
}
1349+
}
1350+
1351+
impl metrics::Recorder for ParkingRecorder {
1352+
fn describe_counter(
1353+
&self,
1354+
key: metrics::KeyName,
1355+
unit: Option<metrics::Unit>,
1356+
description: metrics::SharedString,
1357+
) {
1358+
self.inner.describe_counter(key, unit, description);
1359+
}
1360+
1361+
fn describe_gauge(
1362+
&self,
1363+
key: metrics::KeyName,
1364+
unit: Option<metrics::Unit>,
1365+
description: metrics::SharedString,
1366+
) {
1367+
self.inner.describe_gauge(key, unit, description);
1368+
}
1369+
1370+
fn describe_histogram(
1371+
&self,
1372+
key: metrics::KeyName,
1373+
unit: Option<metrics::Unit>,
1374+
description: metrics::SharedString,
1375+
) {
1376+
self.inner.describe_histogram(key, unit, description);
1377+
}
1378+
1379+
fn register_counter(
1380+
&self,
1381+
key: &metrics::Key,
1382+
metadata: &metrics::Metadata<'_>,
1383+
) -> metrics::Counter {
1384+
self.inner.register_counter(key, metadata)
1385+
}
1386+
1387+
fn register_gauge(
1388+
&self,
1389+
key: &metrics::Key,
1390+
metadata: &metrics::Metadata<'_>,
1391+
) -> metrics::Gauge {
1392+
let gauge = self.inner.register_gauge(key, metadata);
1393+
if key.name() == "buzz_readiness_dependency_sample_completed_timestamp_seconds" {
1394+
metrics::Gauge::from_arc(Arc::new(ParkingGauge {
1395+
inner: gauge,
1396+
park: Arc::clone(&self.park),
1397+
}))
1398+
} else {
1399+
gauge
1400+
}
1401+
}
1402+
1403+
fn register_histogram(
1404+
&self,
1405+
key: &metrics::Key,
1406+
metadata: &metrics::Metadata<'_>,
1407+
) -> metrics::Histogram {
1408+
self.inner.register_histogram(key, metadata)
1409+
}
1410+
}
1411+
1412+
/// Two tasks write this gauge — the sampler, which owns the epoch, and the
1413+
/// idle-refresh publisher, which may only re-emit it — and the metrics
1414+
/// facade offers a bare `set` with no compare-and-set to lean on. So a
1415+
/// republish that read the stored epoch before a sample completed can still
1416+
/// be inside its own write when the newer epoch lands, and plain last-write
1417+
/// -wins would leave the exported series moved backwards until the next
1418+
/// republish tick.
1419+
///
1420+
/// This parks the republish at the recorder, the last point in its write,
1421+
/// completes a newer sample behind it — which must not be blocked by the
1422+
/// parked republish — and only then releases it. The scrape must report the
1423+
/// newer completion. Times are injected so the two epochs are exact and the
1424+
/// interleaving does not depend on the wall clock.
1425+
#[test]
1426+
fn a_republish_racing_a_completion_cannot_move_the_exported_epoch_backwards() {
1427+
const EARLIER_EPOCH_SECONDS: u64 = 1_700_000_000;
1428+
const LATER_EPOCH_SECONDS: u64 = 1_700_000_030;
1429+
1430+
let (prometheus, handle) = crate::metrics::readiness_test_recorder();
1431+
let park = Arc::new(PublishPark::new());
1432+
let recorder = Arc::new(ParkingRecorder {
1433+
inner: prometheus,
1434+
park: Arc::clone(&park),
1435+
});
1436+
let diagnostics = Arc::new(ready_diagnostics());
1437+
1438+
metrics::with_local_recorder(recorder.as_ref(), || {
1439+
diagnostics.record_dependency_sample_completion(
1440+
SystemTime::UNIX_EPOCH + Duration::from_secs(EARLIER_EPOCH_SECONDS),
1441+
);
1442+
});
1443+
assert_eq!(
1444+
sample_completion_metric_value_from_scrape(&handle.render()),
1445+
Some(EARLIER_EPOCH_SECONDS),
1446+
"the first completion owns the epoch"
1447+
);
1448+
1449+
park.arm(EARLIER_EPOCH_SECONDS);
1450+
let republisher = std::thread::spawn({
1451+
let recorder = Arc::clone(&recorder);
1452+
let diagnostics = Arc::clone(&diagnostics);
1453+
move || {
1454+
metrics::with_local_recorder(recorder.as_ref(), || {
1455+
diagnostics.republish_dependency_sample_completion();
1456+
});
1457+
}
1458+
});
1459+
1460+
park.wait_until_parked();
1461+
1462+
metrics::with_local_recorder(recorder.as_ref(), || {
1463+
diagnostics.record_dependency_sample_completion(
1464+
SystemTime::UNIX_EPOCH + Duration::from_secs(LATER_EPOCH_SECONDS),
1465+
);
1466+
});
1467+
assert_eq!(
1468+
sample_completion_metric_value_from_scrape(&handle.render()),
1469+
Some(LATER_EPOCH_SECONDS),
1470+
"a completed sample must publish its epoch while a republish is still in flight"
1471+
);
1472+
1473+
park.release();
1474+
republisher.join().expect("republish thread");
1475+
1476+
assert_eq!(
1477+
sample_completion_metric_value_from_scrape(&handle.render()),
1478+
Some(LATER_EPOCH_SECONDS),
1479+
"a republish that lost the race must not re-export the epoch it read \
1480+
before the newer sample completed"
1481+
);
1482+
}
1483+
12471484
#[test]
12481485
fn readiness_reason_labels_are_the_closed_lifecycle_set() {
12491486
assert_eq!(

‎deploy/charts/buzz/README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,11 @@ the cached report completed**, in Unix seconds. The relay sampler is the only
187187
owner allowed to advance that epoch, and it writes it immediately after the
188188
cache is replaced. A separate bounded publisher re-emits the stored epoch often
189189
enough to survive local gauge idle-timeout; republishing never advances the
190-
timestamp. So the value stands still when sampling stops, and the time elapsed
191-
since it was written is whatever the reader computes at read time.
190+
timestamp. Nor does it ever move the series backwards: a republish that raced a
191+
completing sample re-emits the newer epoch it finds, so the exported value never
192+
regresses to a completion the pod has already passed. So the value stands still
193+
when sampling stops, and the time elapsed since it was written is whatever the
194+
reader computes at read time.
192195

193196
Following the `buzz_storage_sweep_age_seconds` convention, the series is not
194197
emitted until the first sample completes: its absence means "not yet sampled",

0 commit comments

Comments
 (0)