|
| 1 | +//! Consumer-boundary tests: real reliable framing/Redis validation, controlled |
| 2 | +//! transport halves. Redis tests are explicit (never silently skipped): |
| 3 | +//! REDIS_URL=... cargo test -p buzz-relay mesh_boot::echo_tests -- --include-ignored |
| 4 | +
|
| 5 | +use super::*; |
| 6 | +use crate::tunnel::{ |
| 7 | + directory::{AcquireResult, SessionLease}, |
| 8 | + reliable::ReliableMeshStream, |
| 9 | +}; |
| 10 | +use buzz_relay_mesh::{ |
| 11 | + BoxFuture, FencedHeader, MeshError, MeshStreamFrame, StreamRecvHalf, StreamSendHalf, |
| 12 | +}; |
| 13 | +use futures_util::poll; |
| 14 | +use std::{future::Future, pin::Pin, sync::atomic::AtomicUsize, time::Duration}; |
| 15 | +use tokio::sync::mpsc; |
| 16 | + |
| 17 | +#[derive(Default)] |
| 18 | +struct Observed { |
| 19 | + receives: AtomicUsize, |
| 20 | + consumed: AtomicUsize, |
| 21 | + finishes: AtomicUsize, |
| 22 | +} |
| 23 | + |
| 24 | +struct SendHalf(mpsc::UnboundedSender<MeshStreamFrame>, Arc<Observed>); |
| 25 | +impl StreamSendHalf for SendHalf { |
| 26 | + fn send_frame(&mut self, frame: MeshStreamFrame) -> BoxFuture<'_, Result<(), MeshError>> { |
| 27 | + Box::pin(async move { |
| 28 | + self.0.send(frame).unwrap(); |
| 29 | + Ok(()) |
| 30 | + }) |
| 31 | + } |
| 32 | + fn finish(&mut self) -> Result<(), MeshError> { |
| 33 | + self.1.finishes.fetch_add(1, Ordering::SeqCst); |
| 34 | + Ok(()) |
| 35 | + } |
| 36 | +} |
| 37 | + |
| 38 | +struct RecvHalf(mpsc::UnboundedReceiver<MeshStreamFrame>, Arc<Observed>); |
| 39 | +impl StreamRecvHalf for RecvHalf { |
| 40 | + fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> { |
| 41 | + self.1.receives.fetch_add(1, Ordering::SeqCst); |
| 42 | + Box::pin(async move { |
| 43 | + let frame = self.0.recv().await; |
| 44 | + if frame.is_some() { |
| 45 | + self.1.consumed.fetch_add(1, Ordering::SeqCst); |
| 46 | + } |
| 47 | + Ok(frame) |
| 48 | + }) |
| 49 | + } |
| 50 | +} |
| 51 | + |
| 52 | +fn streams(fenced: FencedHeader) -> (ReliableInbound, ReliableMeshStream, Arc<Observed>) { |
| 53 | + let (input_tx, input_rx) = mpsc::unbounded_channel(); |
| 54 | + let (output_tx, output_rx) = mpsc::unbounded_channel(); |
| 55 | + let owner = Arc::new(Observed::default()); |
| 56 | + let peer = Arc::new(Observed::default()); |
| 57 | + let inbound = ReliableInbound { |
| 58 | + fenced, |
| 59 | + from: RuntimeId([18; 32]), |
| 60 | + stream: ReliableMeshStream::new_inbound( |
| 61 | + fenced, |
| 62 | + MeshStream::new( |
| 63 | + Box::new(SendHalf(output_tx, owner.clone())), |
| 64 | + Box::new(RecvHalf(input_rx, owner.clone())), |
| 65 | + ), |
| 66 | + ), |
| 67 | + }; |
| 68 | + let peer_stream = ReliableMeshStream::new_inbound( |
| 69 | + fenced, |
| 70 | + MeshStream::new( |
| 71 | + Box::new(SendHalf(input_tx, peer.clone())), |
| 72 | + Box::new(RecvHalf(output_rx, peer)), |
| 73 | + ), |
| 74 | + ); |
| 75 | + (inbound, peer_stream, owner) |
| 76 | +} |
| 77 | + |
| 78 | +fn pool(url: String) -> deadpool_redis::Pool { |
| 79 | + let mut config = deadpool_redis::Config::from_url(url); |
| 80 | + config.pool = Some(deadpool_redis::PoolConfig::new(1)); |
| 81 | + config |
| 82 | + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) |
| 83 | + .unwrap() |
| 84 | +} |
| 85 | + |
| 86 | +struct Fixture { |
| 87 | + pool: deadpool_redis::Pool, |
| 88 | + directory: SessionDirectory, |
| 89 | + lease: SessionLease, |
| 90 | +} |
| 91 | +impl Fixture { |
| 92 | + async fn new() -> Self { |
| 93 | + let pool = pool(std::env::var("REDIS_URL").expect("explicit test Redis required")); |
| 94 | + let directory = SessionDirectory::with_lease_ttl(pool.clone(), Duration::from_secs(5)); |
| 95 | + let community = buzz_core::CommunityId::from_uuid(uuid::Uuid::new_v4()); |
| 96 | + let lease = match directory |
| 97 | + .acquire( |
| 98 | + community, |
| 99 | + uuid::Uuid::new_v4(), |
| 100 | + RuntimeId([17; 32]), |
| 101 | + Profile::ReliableStream, |
| 102 | + ) |
| 103 | + .await |
| 104 | + .unwrap() |
| 105 | + { |
| 106 | + AcquireResult::Acquired(lease) => lease, |
| 107 | + _ => panic!("UUID collision"), |
| 108 | + }; |
| 109 | + Self { |
| 110 | + pool, |
| 111 | + directory, |
| 112 | + lease, |
| 113 | + } |
| 114 | + } |
| 115 | + |
| 116 | + async fn cleanup(self) { |
| 117 | + self.directory.release(&self.lease).await.unwrap(); |
| 118 | + let mut conn = self.pool.get().await.unwrap(); |
| 119 | + let community = self.lease.community_id; |
| 120 | + let session = self.lease.session_id; |
| 121 | + let _: () = redis::cmd("DEL") |
| 122 | + .arg(format!("buzz:{community}:tunnel:{session}:generation")) |
| 123 | + .query_async(&mut *conn) |
| 124 | + .await |
| 125 | + .unwrap(); |
| 126 | + } |
| 127 | +} |
| 128 | + |
| 129 | +// Drive the actual consumer ourselves: a held private pool slot guarantees |
| 130 | +// validation is pending, and each poll after 110ms must service a 100ms tick. |
| 131 | +// No spawned-task scheduling assumption decides whether the frame was consumed |
| 132 | +// or the consumer crossed a tick. The Redis server/other pools are never blocked. |
| 133 | +async fn housekeeping(consumer: &mut Pin<&mut impl Future<Output = ()>>) { |
| 134 | + tokio::time::sleep(Duration::from_millis(110)).await; |
| 135 | + assert!(poll!(consumer.as_mut()).is_pending()); |
| 136 | +} |
| 137 | + |
| 138 | +#[tokio::test] |
| 139 | +#[ignore = "requires explicit REDIS_URL"] |
| 140 | +async fn consumed_frame_survives_housekeeping_during_validation() { |
| 141 | + let f = Fixture::new().await; |
| 142 | + let (inbound, mut peer, seen) = streams(f.lease.fenced_header()); |
| 143 | + let held = f.pool.get().await.unwrap(); |
| 144 | + let shutdown = Arc::new(AtomicBool::new(false)); |
| 145 | + let consumer = run_demo_echo(f.directory.clone(), inbound, shutdown); |
| 146 | + tokio::pin!(consumer); |
| 147 | + assert!(poll!(&mut consumer).is_pending()); |
| 148 | + peer.send_bytes(f.lease.community_id, b"first") |
| 149 | + .await |
| 150 | + .unwrap(); |
| 151 | + assert!(poll!(&mut consumer).is_pending()); |
| 152 | + assert_eq!(seen.consumed.load(Ordering::SeqCst), 1); |
| 153 | + for _ in 0..3 { |
| 154 | + housekeeping(&mut consumer).await; |
| 155 | + } |
| 156 | + assert_eq!( |
| 157 | + seen.receives.load(Ordering::SeqCst), |
| 158 | + 1, |
| 159 | + "pending receive was recreated" |
| 160 | + ); |
| 161 | + drop(held); |
| 162 | + tokio::time::timeout(Duration::from_secs(2), async { |
| 163 | + let receive = async { |
| 164 | + assert!(matches!(peer.recv_validated(&f.directory).await.unwrap(), |
| 165 | + Some(ReliableFrame::Data(bytes)) if bytes == b"first")); |
| 166 | + peer.send_bytes(f.lease.community_id, b"second") |
| 167 | + .await |
| 168 | + .unwrap(); |
| 169 | + assert!(matches!(peer.recv_validated(&f.directory).await.unwrap(), |
| 170 | + Some(ReliableFrame::Data(bytes)) if bytes == b"second")); |
| 171 | + peer.send_goodbye(f.lease.community_id, GoodbyeReason::Draining) |
| 172 | + .await |
| 173 | + .unwrap(); |
| 174 | + }; |
| 175 | + tokio::join!(&mut consumer, receive); |
| 176 | + }) |
| 177 | + .await |
| 178 | + .expect("echoes and peer Goodbye must terminate"); |
| 179 | + assert_eq!(seen.consumed.load(Ordering::SeqCst), 3); |
| 180 | + assert!( |
| 181 | + peer.recv_validated(&f.directory).await.unwrap().is_none(), |
| 182 | + "no duplicate echo" |
| 183 | + ); |
| 184 | + f.cleanup().await; |
| 185 | +} |
| 186 | + |
| 187 | +async fn drain_during_validation(latched: bool) { |
| 188 | + let f = Fixture::new().await; |
| 189 | + let (inbound, mut peer, seen) = streams(f.lease.fenced_header()); |
| 190 | + let shutdown = Arc::new(AtomicBool::new(false)); |
| 191 | + let consumer = run_demo_echo(f.directory.clone(), inbound, shutdown.clone()); |
| 192 | + tokio::pin!(consumer); |
| 193 | + if latched { |
| 194 | + peer.send_bytes(f.lease.community_id, b"latch") |
| 195 | + .await |
| 196 | + .unwrap(); |
| 197 | + tokio::time::timeout(Duration::from_secs(2), async { |
| 198 | + tokio::select! { |
| 199 | + _ = &mut consumer => panic!("premature termination"), |
| 200 | + frame = peer.recv_validated(&f.directory) => assert!(matches!(frame.unwrap(), |
| 201 | + Some(ReliableFrame::Data(bytes)) if bytes == b"latch")), |
| 202 | + } |
| 203 | + }) |
| 204 | + .await |
| 205 | + .unwrap(); |
| 206 | + } |
| 207 | + let held = f.pool.get().await.unwrap(); |
| 208 | + peer.send_bytes(f.lease.community_id, b"pending") |
| 209 | + .await |
| 210 | + .unwrap(); |
| 211 | + assert!(poll!(&mut consumer).is_pending()); |
| 212 | + let count = if latched { 2 } else { 1 }; |
| 213 | + assert_eq!(seen.consumed.load(Ordering::SeqCst), count); |
| 214 | + housekeeping(&mut consumer).await; |
| 215 | + shutdown.store(true, Ordering::Relaxed); |
| 216 | + tokio::time::timeout(Duration::from_secs(1), &mut consumer) |
| 217 | + .await |
| 218 | + .unwrap(); |
| 219 | + // Shutdown cannot wait for validation, and must not start another receive. |
| 220 | + assert_eq!(seen.receives.load(Ordering::SeqCst), count); |
| 221 | + assert_eq!(seen.finishes.load(Ordering::SeqCst), 1); |
| 222 | + drop(held); |
| 223 | + if latched { |
| 224 | + assert!(matches!( |
| 225 | + peer.recv_validated(&f.directory).await.unwrap(), |
| 226 | + Some(ReliableFrame::Goodbye(GoodbyeReason::Draining)) |
| 227 | + )); |
| 228 | + } |
| 229 | + assert!( |
| 230 | + peer.recv_validated(&f.directory).await.unwrap().is_none(), |
| 231 | + "unvalidated Data must not echo" |
| 232 | + ); |
| 233 | + f.cleanup().await; |
| 234 | +} |
| 235 | + |
| 236 | +#[tokio::test] |
| 237 | +#[ignore = "requires explicit REDIS_URL"] |
| 238 | +async fn drain_before_latch_cancels_validation_and_finishes() { |
| 239 | + drain_during_validation(false).await; |
| 240 | +} |
| 241 | + |
| 242 | +#[tokio::test] |
| 243 | +#[ignore = "requires explicit REDIS_URL"] |
| 244 | +async fn drain_after_latch_cancels_validation_and_sends_goodbye() { |
| 245 | + drain_during_validation(true).await; |
| 246 | +} |
| 247 | + |
| 248 | +#[tokio::test] |
| 249 | +#[ignore = "requires explicit REDIS_URL"] |
| 250 | +async fn retained_receive_still_rejects_released_fence() { |
| 251 | + let f = Fixture::new().await; |
| 252 | + let (inbound, mut peer, seen) = streams(f.lease.fenced_header()); |
| 253 | + let mut held = f.pool.get().await.unwrap(); |
| 254 | + let consumer = run_demo_echo( |
| 255 | + f.directory.clone(), |
| 256 | + inbound, |
| 257 | + Arc::new(AtomicBool::new(false)), |
| 258 | + ); |
| 259 | + tokio::pin!(consumer); |
| 260 | + peer.send_bytes(f.lease.community_id, b"stale") |
| 261 | + .await |
| 262 | + .unwrap(); |
| 263 | + assert!(poll!(&mut consumer).is_pending()); |
| 264 | + housekeeping(&mut consumer).await; |
| 265 | + // Remove only this test's lease while validation is still waiting for the |
| 266 | + // pool slot. Keep its generation floor, just like a normal release. |
| 267 | + let _: () = redis::cmd("DEL") |
| 268 | + .arg(format!( |
| 269 | + "buzz:{}:tunnel:{}:lease", |
| 270 | + f.lease.community_id, f.lease.session_id |
| 271 | + )) |
| 272 | + .query_async(&mut *held) |
| 273 | + .await |
| 274 | + .unwrap(); |
| 275 | + drop(held); |
| 276 | + tokio::time::timeout(Duration::from_secs(2), &mut consumer) |
| 277 | + .await |
| 278 | + .unwrap(); |
| 279 | + assert_eq!(seen.consumed.load(Ordering::SeqCst), 1); |
| 280 | + assert!(peer.recv_validated(&f.directory).await.unwrap().is_none()); |
| 281 | + f.cleanup().await; |
| 282 | +} |
| 283 | + |
| 284 | +#[tokio::test] |
| 285 | +async fn drain_idle_before_latch_and_eof_terminate_without_redis() { |
| 286 | + for draining in [true, false] { |
| 287 | + let directory = SessionDirectory::new(pool("redis://127.0.0.1:1".into())); |
| 288 | + let fenced = FencedHeader { |
| 289 | + session_id: uuid::Uuid::new_v4(), |
| 290 | + generation: 1, |
| 291 | + owner_runtime_id: RuntimeId([17; 32]), |
| 292 | + }; |
| 293 | + let (inbound, peer, seen) = streams(fenced); |
| 294 | + let peer = if draining { |
| 295 | + Some(peer) |
| 296 | + } else { |
| 297 | + drop(peer); |
| 298 | + None |
| 299 | + }; |
| 300 | + tokio::time::timeout( |
| 301 | + Duration::from_secs(1), |
| 302 | + run_demo_echo(directory, inbound, Arc::new(AtomicBool::new(draining))), |
| 303 | + ) |
| 304 | + .await |
| 305 | + .unwrap(); |
| 306 | + assert_eq!(seen.finishes.load(Ordering::SeqCst), usize::from(draining)); |
| 307 | + assert!(seen.receives.load(Ordering::SeqCst) <= 1); |
| 308 | + drop(peer); |
| 309 | + } |
| 310 | +} |
0 commit comments