Skip to content

Commit a02372a

Browse files
committed
fix(mesh): retain echo receive across housekeeping ticks
Keep the pending reliable receive and Redis validation alive until a frame arrives or shutdown actually drains the stream. Drop its mutable borrow before the existing Goodbye/finish path. Cover consumed-frame pool contention, drain before and after community latch, EOF, sequential echoes, peer Goodbye, and stale lease rejection. Signed-off-by: Logan Johnson <[email protected]>
1 parent bc006f6 commit a02372a

2 files changed

Lines changed: 343 additions & 14 deletions

File tree

‎crates/buzz-relay/src/mesh_boot.rs‎

Lines changed: 33 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -315,24 +315,40 @@ pub(crate) async fn run_demo_echo(
315315
tracing::info!(%session_id, %peer, "mesh demo echo: session open");
316316
let mut drain_tick = tokio::time::interval(std::time::Duration::from_millis(100));
317317
loop {
318-
let frame = tokio::select! {
319-
_ = drain_tick.tick() => {
320-
if shutting_down.load(Ordering::Relaxed) {
321-
if let Some(community_id) = stream.community_id() {
322-
if let Err(e) = stream.send_goodbye(community_id, GoodbyeReason::Draining).await {
323-
tracing::warn!(%session_id, "mesh demo echo: draining goodbye failed: {e}");
324-
} else {
325-
tracing::info!(%session_id, "mesh demo echo: sent draining goodbye");
318+
let frame = {
319+
// Receiving is not cancellation-safe: it may have consumed bytes or
320+
// a whole frame before awaiting Redis validation. Retain this future
321+
// across housekeeping ticks; only terminal drain may discard it.
322+
let recv = stream.recv_validated(&directory);
323+
tokio::pin!(recv);
324+
loop {
325+
tokio::select! {
326+
_ = drain_tick.tick() => {
327+
if shutting_down.load(Ordering::Relaxed) {
328+
break None;
326329
}
327-
} else {
328-
let _ = stream.finish();
329-
tracing::info!(%session_id, "mesh demo echo: drain before community latch — closing");
330330
}
331-
return;
331+
frame = &mut recv => break Some(frame),
332332
}
333-
continue;
334333
}
335-
frame = stream.recv_validated(&directory) => frame,
334+
};
335+
// The receive future (and its mutable stream borrow) is gone before
336+
// either the drain response or the next echo uses the stream.
337+
let Some(frame) = frame else {
338+
if let Some(community_id) = stream.community_id() {
339+
if let Err(e) = stream
340+
.send_goodbye(community_id, GoodbyeReason::Draining)
341+
.await
342+
{
343+
tracing::warn!(%session_id, "mesh demo echo: draining goodbye failed: {e}");
344+
} else {
345+
tracing::info!(%session_id, "mesh demo echo: sent draining goodbye");
346+
}
347+
} else {
348+
let _ = stream.finish();
349+
tracing::info!(%session_id, "mesh demo echo: drain before community latch — closing");
350+
}
351+
return;
336352
};
337353
match frame {
338354
Ok(Some(ReliableFrame::Data(payload))) => {
@@ -520,6 +536,9 @@ pub async fn boot_mesh(
520536
}))
521537
}
522538

539+
#[cfg(test)]
540+
mod echo_tests;
541+
523542
#[cfg(test)]
524543
mod tests {
525544
use super::*;
Lines changed: 310 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,310 @@
1+
//! Consumer-boundary tests: real reliable framing/Redis validation, controlled
2+
//! transport halves. Redis tests are explicit (never silently skipped):
3+
//! REDIS_URL=... cargo test -p buzz-relay mesh_boot::echo_tests -- --include-ignored
4+
5+
use super::*;
6+
use crate::tunnel::{
7+
directory::{AcquireResult, SessionLease},
8+
reliable::ReliableMeshStream,
9+
};
10+
use buzz_relay_mesh::{
11+
BoxFuture, FencedHeader, MeshError, MeshStreamFrame, StreamRecvHalf, StreamSendHalf,
12+
};
13+
use futures_util::poll;
14+
use std::{future::Future, pin::Pin, sync::atomic::AtomicUsize, time::Duration};
15+
use tokio::sync::mpsc;
16+
17+
#[derive(Default)]
18+
struct Observed {
19+
receives: AtomicUsize,
20+
consumed: AtomicUsize,
21+
finishes: AtomicUsize,
22+
}
23+
24+
struct SendHalf(mpsc::UnboundedSender<MeshStreamFrame>, Arc<Observed>);
25+
impl StreamSendHalf for SendHalf {
26+
fn send_frame(&mut self, frame: MeshStreamFrame) -> BoxFuture<'_, Result<(), MeshError>> {
27+
Box::pin(async move {
28+
self.0.send(frame).unwrap();
29+
Ok(())
30+
})
31+
}
32+
fn finish(&mut self) -> Result<(), MeshError> {
33+
self.1.finishes.fetch_add(1, Ordering::SeqCst);
34+
Ok(())
35+
}
36+
}
37+
38+
struct RecvHalf(mpsc::UnboundedReceiver<MeshStreamFrame>, Arc<Observed>);
39+
impl StreamRecvHalf for RecvHalf {
40+
fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> {
41+
self.1.receives.fetch_add(1, Ordering::SeqCst);
42+
Box::pin(async move {
43+
let frame = self.0.recv().await;
44+
if frame.is_some() {
45+
self.1.consumed.fetch_add(1, Ordering::SeqCst);
46+
}
47+
Ok(frame)
48+
})
49+
}
50+
}
51+
52+
fn streams(fenced: FencedHeader) -> (ReliableInbound, ReliableMeshStream, Arc<Observed>) {
53+
let (input_tx, input_rx) = mpsc::unbounded_channel();
54+
let (output_tx, output_rx) = mpsc::unbounded_channel();
55+
let owner = Arc::new(Observed::default());
56+
let peer = Arc::new(Observed::default());
57+
let inbound = ReliableInbound {
58+
fenced,
59+
from: RuntimeId([18; 32]),
60+
stream: ReliableMeshStream::new_inbound(
61+
fenced,
62+
MeshStream::new(
63+
Box::new(SendHalf(output_tx, owner.clone())),
64+
Box::new(RecvHalf(input_rx, owner.clone())),
65+
),
66+
),
67+
};
68+
let peer_stream = ReliableMeshStream::new_inbound(
69+
fenced,
70+
MeshStream::new(
71+
Box::new(SendHalf(input_tx, peer.clone())),
72+
Box::new(RecvHalf(output_rx, peer)),
73+
),
74+
);
75+
(inbound, peer_stream, owner)
76+
}
77+
78+
fn pool(url: String) -> deadpool_redis::Pool {
79+
let mut config = deadpool_redis::Config::from_url(url);
80+
config.pool = Some(deadpool_redis::PoolConfig::new(1));
81+
config
82+
.create_pool(Some(deadpool_redis::Runtime::Tokio1))
83+
.unwrap()
84+
}
85+
86+
struct Fixture {
87+
pool: deadpool_redis::Pool,
88+
directory: SessionDirectory,
89+
lease: SessionLease,
90+
}
91+
impl Fixture {
92+
async fn new() -> Self {
93+
let pool = pool(std::env::var("REDIS_URL").expect("explicit test Redis required"));
94+
let directory = SessionDirectory::with_lease_ttl(pool.clone(), Duration::from_secs(5));
95+
let community = buzz_core::CommunityId::from_uuid(uuid::Uuid::new_v4());
96+
let lease = match directory
97+
.acquire(
98+
community,
99+
uuid::Uuid::new_v4(),
100+
RuntimeId([17; 32]),
101+
Profile::ReliableStream,
102+
)
103+
.await
104+
.unwrap()
105+
{
106+
AcquireResult::Acquired(lease) => lease,
107+
_ => panic!("UUID collision"),
108+
};
109+
Self {
110+
pool,
111+
directory,
112+
lease,
113+
}
114+
}
115+
116+
async fn cleanup(self) {
117+
self.directory.release(&self.lease).await.unwrap();
118+
let mut conn = self.pool.get().await.unwrap();
119+
let community = self.lease.community_id;
120+
let session = self.lease.session_id;
121+
let _: () = redis::cmd("DEL")
122+
.arg(format!("buzz:{community}:tunnel:{session}:generation"))
123+
.query_async(&mut *conn)
124+
.await
125+
.unwrap();
126+
}
127+
}
128+
129+
// Drive the actual consumer ourselves: a held private pool slot guarantees
130+
// validation is pending, and each poll after 110ms must service a 100ms tick.
131+
// No spawned-task scheduling assumption decides whether the frame was consumed
132+
// or the consumer crossed a tick. The Redis server/other pools are never blocked.
133+
async fn housekeeping(consumer: &mut Pin<&mut impl Future<Output = ()>>) {
134+
tokio::time::sleep(Duration::from_millis(110)).await;
135+
assert!(poll!(consumer.as_mut()).is_pending());
136+
}
137+
138+
#[tokio::test]
139+
#[ignore = "requires explicit REDIS_URL"]
140+
async fn consumed_frame_survives_housekeeping_during_validation() {
141+
let f = Fixture::new().await;
142+
let (inbound, mut peer, seen) = streams(f.lease.fenced_header());
143+
let held = f.pool.get().await.unwrap();
144+
let shutdown = Arc::new(AtomicBool::new(false));
145+
let consumer = run_demo_echo(f.directory.clone(), inbound, shutdown);
146+
tokio::pin!(consumer);
147+
assert!(poll!(&mut consumer).is_pending());
148+
peer.send_bytes(f.lease.community_id, b"first")
149+
.await
150+
.unwrap();
151+
assert!(poll!(&mut consumer).is_pending());
152+
assert_eq!(seen.consumed.load(Ordering::SeqCst), 1);
153+
for _ in 0..3 {
154+
housekeeping(&mut consumer).await;
155+
}
156+
assert_eq!(
157+
seen.receives.load(Ordering::SeqCst),
158+
1,
159+
"pending receive was recreated"
160+
);
161+
drop(held);
162+
tokio::time::timeout(Duration::from_secs(2), async {
163+
let receive = async {
164+
assert!(matches!(peer.recv_validated(&f.directory).await.unwrap(),
165+
Some(ReliableFrame::Data(bytes)) if bytes == b"first"));
166+
peer.send_bytes(f.lease.community_id, b"second")
167+
.await
168+
.unwrap();
169+
assert!(matches!(peer.recv_validated(&f.directory).await.unwrap(),
170+
Some(ReliableFrame::Data(bytes)) if bytes == b"second"));
171+
peer.send_goodbye(f.lease.community_id, GoodbyeReason::Draining)
172+
.await
173+
.unwrap();
174+
};
175+
tokio::join!(&mut consumer, receive);
176+
})
177+
.await
178+
.expect("echoes and peer Goodbye must terminate");
179+
assert_eq!(seen.consumed.load(Ordering::SeqCst), 3);
180+
assert!(
181+
peer.recv_validated(&f.directory).await.unwrap().is_none(),
182+
"no duplicate echo"
183+
);
184+
f.cleanup().await;
185+
}
186+
187+
async fn drain_during_validation(latched: bool) {
188+
let f = Fixture::new().await;
189+
let (inbound, mut peer, seen) = streams(f.lease.fenced_header());
190+
let shutdown = Arc::new(AtomicBool::new(false));
191+
let consumer = run_demo_echo(f.directory.clone(), inbound, shutdown.clone());
192+
tokio::pin!(consumer);
193+
if latched {
194+
peer.send_bytes(f.lease.community_id, b"latch")
195+
.await
196+
.unwrap();
197+
tokio::time::timeout(Duration::from_secs(2), async {
198+
tokio::select! {
199+
_ = &mut consumer => panic!("premature termination"),
200+
frame = peer.recv_validated(&f.directory) => assert!(matches!(frame.unwrap(),
201+
Some(ReliableFrame::Data(bytes)) if bytes == b"latch")),
202+
}
203+
})
204+
.await
205+
.unwrap();
206+
}
207+
let held = f.pool.get().await.unwrap();
208+
peer.send_bytes(f.lease.community_id, b"pending")
209+
.await
210+
.unwrap();
211+
assert!(poll!(&mut consumer).is_pending());
212+
let count = if latched { 2 } else { 1 };
213+
assert_eq!(seen.consumed.load(Ordering::SeqCst), count);
214+
housekeeping(&mut consumer).await;
215+
shutdown.store(true, Ordering::Relaxed);
216+
tokio::time::timeout(Duration::from_secs(1), &mut consumer)
217+
.await
218+
.unwrap();
219+
// Shutdown cannot wait for validation, and must not start another receive.
220+
assert_eq!(seen.receives.load(Ordering::SeqCst), count);
221+
assert_eq!(seen.finishes.load(Ordering::SeqCst), 1);
222+
drop(held);
223+
if latched {
224+
assert!(matches!(
225+
peer.recv_validated(&f.directory).await.unwrap(),
226+
Some(ReliableFrame::Goodbye(GoodbyeReason::Draining))
227+
));
228+
}
229+
assert!(
230+
peer.recv_validated(&f.directory).await.unwrap().is_none(),
231+
"unvalidated Data must not echo"
232+
);
233+
f.cleanup().await;
234+
}
235+
236+
#[tokio::test]
237+
#[ignore = "requires explicit REDIS_URL"]
238+
async fn drain_before_latch_cancels_validation_and_finishes() {
239+
drain_during_validation(false).await;
240+
}
241+
242+
#[tokio::test]
243+
#[ignore = "requires explicit REDIS_URL"]
244+
async fn drain_after_latch_cancels_validation_and_sends_goodbye() {
245+
drain_during_validation(true).await;
246+
}
247+
248+
#[tokio::test]
249+
#[ignore = "requires explicit REDIS_URL"]
250+
async fn retained_receive_still_rejects_released_fence() {
251+
let f = Fixture::new().await;
252+
let (inbound, mut peer, seen) = streams(f.lease.fenced_header());
253+
let mut held = f.pool.get().await.unwrap();
254+
let consumer = run_demo_echo(
255+
f.directory.clone(),
256+
inbound,
257+
Arc::new(AtomicBool::new(false)),
258+
);
259+
tokio::pin!(consumer);
260+
peer.send_bytes(f.lease.community_id, b"stale")
261+
.await
262+
.unwrap();
263+
assert!(poll!(&mut consumer).is_pending());
264+
housekeeping(&mut consumer).await;
265+
// Remove only this test's lease while validation is still waiting for the
266+
// pool slot. Keep its generation floor, just like a normal release.
267+
let _: () = redis::cmd("DEL")
268+
.arg(format!(
269+
"buzz:{}:tunnel:{}:lease",
270+
f.lease.community_id, f.lease.session_id
271+
))
272+
.query_async(&mut *held)
273+
.await
274+
.unwrap();
275+
drop(held);
276+
tokio::time::timeout(Duration::from_secs(2), &mut consumer)
277+
.await
278+
.unwrap();
279+
assert_eq!(seen.consumed.load(Ordering::SeqCst), 1);
280+
assert!(peer.recv_validated(&f.directory).await.unwrap().is_none());
281+
f.cleanup().await;
282+
}
283+
284+
#[tokio::test]
285+
async fn drain_idle_before_latch_and_eof_terminate_without_redis() {
286+
for draining in [true, false] {
287+
let directory = SessionDirectory::new(pool("redis://127.0.0.1:1".into()));
288+
let fenced = FencedHeader {
289+
session_id: uuid::Uuid::new_v4(),
290+
generation: 1,
291+
owner_runtime_id: RuntimeId([17; 32]),
292+
};
293+
let (inbound, peer, seen) = streams(fenced);
294+
let peer = if draining {
295+
Some(peer)
296+
} else {
297+
drop(peer);
298+
None
299+
};
300+
tokio::time::timeout(
301+
Duration::from_secs(1),
302+
run_demo_echo(directory, inbound, Arc::new(AtomicBool::new(draining))),
303+
)
304+
.await
305+
.unwrap();
306+
assert_eq!(seen.finishes.load(Ordering::SeqCst), usize::from(draining));
307+
assert!(seen.receives.load(Ordering::SeqCst) <= 1);
308+
drop(peer);
309+
}
310+
}

0 commit comments

Comments
 (0)