Skip to content

Commit f1674a2

Browse files
bradseilercodex
andcommitted
fix(relay): retain mesh receives across housekeeping ticks
Complete the preserved 66663e19 remediation with a deterministic production-consumer regression, cancellation-safety documentation, and deadline diagnostics. Original recovered refs and their attribution remain unchanged. Co-authored-by: Codex <[email protected]> Signed-off-by: Brad Seiler <[email protected]>
1 parent 4f5d8f3 commit f1674a2

4 files changed

Lines changed: 133 additions & 34 deletions

File tree

‎crates/buzz-relay/src/api/mesh_demo.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,9 @@ mod tests {
336336
},
337337
)
338338
.await;
339-
assert_eq!(resp.status(), StatusCode::OK);
339+
let status = resp.status();
340340
let body = body_json(resp).await;
341+
assert_eq!(status, StatusCode::OK, "forwarded response: {body}");
341342
assert_eq!(body["outcome"], "forwarded");
342343
assert_eq!(body["echoed_payload"], "mesh echo evidence");
343344
owner_task.abort();

‎crates/buzz-relay/src/mesh_boot.rs‎

Lines changed: 95 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -421,24 +421,39 @@ pub(crate) async fn run_demo_echo(
421421
tracing::info!(%session_id, %peer, "mesh demo echo: session open");
422422
let mut drain_tick = tokio::time::interval(std::time::Duration::from_millis(100));
423423
loop {
424-
let frame = tokio::select! {
425-
_ = drain_tick.tick() => {
426-
if shutting_down.load(Ordering::Relaxed) {
427-
if let Some(community_id) = stream.community_id() {
428-
if let Err(e) = stream.send_goodbye(community_id, GoodbyeReason::Draining).await {
429-
tracing::warn!(%session_id, "mesh demo echo: draining goodbye failed: {e}");
430-
} else {
431-
tracing::info!(%session_id, "mesh demo echo: sent draining goodbye");
424+
let frame = {
425+
// A receive can consume bytes before waiting on Redis validation.
426+
// Keep it alive across housekeeping ticks; only shutdown may cancel it.
427+
let recv = stream.recv_validated(&directory);
428+
tokio::pin!(recv);
429+
loop {
430+
tokio::select! {
431+
_ = drain_tick.tick() => {
432+
if shutting_down.load(Ordering::Relaxed) {
433+
break None;
432434
}
433-
} else {
434-
let _ = stream.finish();
435-
tracing::info!(%session_id, "mesh demo echo: drain before community latch — closing");
436435
}
437-
return;
436+
frame = &mut recv => break Some(frame),
438437
}
439-
continue;
440438
}
441-
frame = stream.recv_validated(&directory) => frame,
439+
};
440+
// The pending receive and its mutable stream borrow have ended before
441+
// the drain response or next echo uses the stream.
442+
let Some(frame) = frame else {
443+
if let Some(community_id) = stream.community_id() {
444+
if let Err(e) = stream
445+
.send_goodbye(community_id, GoodbyeReason::Draining)
446+
.await
447+
{
448+
tracing::warn!(%session_id, "mesh demo echo: draining goodbye failed: {e}");
449+
} else {
450+
tracing::info!(%session_id, "mesh demo echo: sent draining goodbye");
451+
}
452+
} else {
453+
let _ = stream.finish();
454+
tracing::info!(%session_id, "mesh demo echo: drain before community latch — closing");
455+
}
456+
return;
442457
};
443458
match frame {
444459
Ok(Some(ReliableFrame::Data(payload))) => {
@@ -696,6 +711,72 @@ mod tests {
696711
MeshStream::new(Box::new(StubSend), Box::new(StubRecv))
697712
}
698713

714+
struct PausedRecv {
715+
started: Option<tokio::sync::oneshot::Sender<()>>,
716+
resume: Option<tokio::sync::oneshot::Receiver<()>>,
717+
calls: Arc<std::sync::atomic::AtomicUsize>,
718+
}
719+
720+
impl StreamRecvHalf for PausedRecv {
721+
fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> {
722+
self.calls.fetch_add(1, Ordering::Relaxed);
723+
let started = self.started.take();
724+
let resume = self.resume.take();
725+
Box::pin(async move {
726+
if let Some(started) = started {
727+
let _ = started.send(());
728+
}
729+
if let Some(resume) = resume {
730+
resume.await.expect("test resumes the in-flight receive");
731+
}
732+
Ok(None)
733+
})
734+
}
735+
}
736+
737+
/// Housekeeping must retain a receive that already consumed transport state.
738+
/// Returning to recv_frame after a tick loses that state and closes the stream.
739+
#[tokio::test(start_paused = true)]
740+
async fn demo_echo_retains_pending_receive_across_housekeeping_ticks() {
741+
let pool = deadpool_redis::Config::from_url("redis://127.0.0.1:1")
742+
.create_pool(Some(deadpool_redis::Runtime::Tokio1))
743+
.expect("test pool");
744+
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
745+
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
746+
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
747+
let fenced = FencedHeader {
748+
session_id: uuid::Uuid::new_v4(),
749+
generation: 1,
750+
owner_runtime_id: rid(1),
751+
};
752+
let stream = MeshStream::new(
753+
Box::new(StubSend),
754+
Box::new(PausedRecv {
755+
started: Some(started_tx),
756+
resume: Some(resume_rx),
757+
calls: Arc::clone(&calls),
758+
}),
759+
);
760+
let worker = tokio::spawn(run_demo_echo(
761+
SessionDirectory::new(pool),
762+
ReliableInbound {
763+
fenced,
764+
from: rid(2),
765+
stream: crate::tunnel::reliable::ReliableMeshStream::new_inbound(fenced, stream),
766+
},
767+
Arc::new(AtomicBool::new(false)),
768+
));
769+
started_rx
770+
.await
771+
.expect("production consumer started receiving");
772+
tokio::time::advance(std::time::Duration::from_millis(300)).await;
773+
tokio::task::yield_now().await;
774+
assert_eq!(calls.load(Ordering::Relaxed), 1, "tick restarted a receive");
775+
assert!(!worker.is_finished(), "tick abandoned the pending receive");
776+
resume_tx.send(()).expect("receive remains alive");
777+
worker.await.expect("consumer completed after stream EOF");
778+
}
779+
699780
fn rid(byte: u8) -> RuntimeId {
700781
RuntimeId([byte; 32])
701782
}

‎crates/buzz-relay/src/tunnel/reliable.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,9 @@ impl ReliableMeshStream {
329329
/// pinned fenced tuple and the Redis directory. This is the reliable-stream
330330
/// equivalent of Dawn's hot-path media floor, but authoritative: stale or
331331
/// mismatched frames fail the session rather than being dropped silently.
332+
/// This receive is not cancellation-safe: it can consume frame bytes before
333+
/// awaiting directory validation. Preserve an in-flight receive across
334+
/// housekeeping ticks; cancellation is only safe when terminating the stream.
332335
pub async fn recv_validated(
333336
&mut self,
334337
directory: &SessionDirectory,

‎crates/buzz-relay/tests/boot_lifecycle.rs‎

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -168,20 +168,38 @@ fn wait_for_relay_metrics(process: &mut RelayProcess, port: u16) -> String {
168168
/// [`METRICS_SCRAPE_DEADLINE`]. A relay that exits first is a failure, not a
169169
/// timeout, so the panic names the real cause.
170170
fn wait_for_scraped_metric(process: &mut RelayProcess, port: u16, needle: &str) -> String {
171+
wait_for_scraped_metrics(process, port, &[needle])
172+
}
173+
174+
/// Waits for one scrape to contain every requested sample, bounded by
175+
/// [`METRICS_SCRAPE_DEADLINE`]. A scrape can overlap the sampler's sequential
176+
/// metric writes, so related series may appear in different responses.
177+
fn wait_for_scraped_metrics(process: &mut RelayProcess, port: u16, needles: &[&str]) -> String {
178+
let requested = needles.join(", ");
179+
let mut last_scrape = String::new();
180+
let mut last_error = None;
171181
let deadline = Instant::now() + METRICS_SCRAPE_DEADLINE;
172182
loop {
173183
assert!(
174184
process.try_wait().is_none(),
175-
"relay exited before exporting {needle}"
185+
"relay exited before exporting {requested}"
176186
);
177-
if let Ok(response) = scrape_metrics(port) {
178-
if response.contains(needle) {
179-
return response;
187+
match scrape_metrics(port) {
188+
Ok(response) => {
189+
if needles
190+
.iter()
191+
.all(|needle| response.lines().any(|line| line.starts_with(needle)))
192+
{
193+
return response;
194+
}
195+
last_scrape = response;
196+
last_error = None;
180197
}
198+
Err(error) => last_error = Some(error.to_string()),
181199
}
182200
assert!(
183201
Instant::now() < deadline,
184-
"relay did not export {needle} within {METRICS_SCRAPE_DEADLINE:?}"
202+
"relay did not export {requested} within {METRICS_SCRAPE_DEADLINE:?}; last error: {last_error:?}; last scrape: {last_scrape}"
185203
);
186204
thread::sleep(Duration::from_millis(20));
187205
}
@@ -814,10 +832,10 @@ mod postgres_tests {
814832
/// fires immediately, so the wait is bounded by
815833
/// [`METRICS_SCRAPE_DEADLINE`] and never a fixed sleep.
816834
///
817-
/// The completion-timestamp gauge is the needle because `sample` writes it
818-
/// last, after the cache already serves the report: observing it proves the
819-
/// whole cycle ran, with no window where the counters have landed but the
820-
/// timestamp has not.
835+
/// The sampler writes counters and histograms before the completion gauge,
836+
/// but a concurrent `/metrics` scrape is not an atomic snapshot. Wait for
837+
/// the actual timestamp, counter, and duration samples together so a scrape
838+
/// that overlaps those sequential writes cannot report a false failure.
821839
#[test]
822840
#[ignore = "requires PostgreSQL"]
823841
fn startup_owns_the_dependency_sampler() {
@@ -839,10 +857,14 @@ mod postgres_tests {
839857
("REDIS_URL", &redis_url),
840858
("BUZZ_GIT_CONFORMANCE_PROBE", "false"),
841859
]);
842-
let scrape = wait_for_scraped_metric(
860+
let scrape = wait_for_scraped_metrics(
843861
&mut process,
844862
metrics_port,
845-
"buzz_readiness_dependency_sample_completed_timestamp_seconds",
863+
&[
864+
"buzz_readiness_dependency_sample_completed_timestamp_seconds ",
865+
"buzz_readiness_dependency_checks_total{",
866+
"buzz_readiness_check_duration_seconds_count{check=\"overall\"}",
867+
],
846868
);
847869
let output = process.terminate();
848870
let logs = format!(
@@ -851,14 +873,6 @@ mod postgres_tests {
851873
String::from_utf8_lossy(&output.stderr)
852874
);
853875

854-
assert!(
855-
scrape.contains("buzz_readiness_dependency_checks_total{"),
856-
"the same cycle must publish its per-dependency outcomes: {scrape}"
857-
);
858-
assert!(
859-
scrape.contains("buzz_readiness_check_duration_seconds"),
860-
"a completed evaluation must publish its latency too: {scrape}"
861-
);
862876
assert!(
863877
!scrape.contains("buzz_readiness_checks_total{"),
864878
"no readiness probe was sent, so the sampler alone produced this: {scrape}"

0 commit comments

Comments
 (0)