Skip to content

Commit 1911c69

Browse files
tlongwell-blocknpub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67dTyler Longwell
authored
fix(relay): send 1012 restart close to all clients on graceful drain (#2575)
Signed-off-by: Tyler Longwell <[email protected]> Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@buzz.block.builderlab.xyz> Co-authored-by: Tyler Longwell <[email protected]>
1 parent 9ec52cf commit 1911c69

3 files changed

Lines changed: 217 additions & 3 deletions

File tree

‎crates/buzz-relay/src/main.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,7 @@ async fn serve(
11271127

11281128
let (shutdown_tx, _) = tokio::sync::watch::channel(false);
11291129
let shutdown_flag = Arc::clone(&state.shutting_down);
1130+
let drain_conn_manager = Arc::clone(&state.conn_manager);
11301131
let tx = shutdown_tx.clone();
11311132
tokio::spawn(async move {
11321133
shutdown_signal().await;
@@ -1136,6 +1137,16 @@ async fn serve(
11361137
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
11371138
info!("Starting graceful drain (30s timeout)");
11381139
let _ = tx.send(true);
1140+
// Tell every connected client to reconnect NOW. Without this, upgraded
1141+
// WebSocket connections outlive the listener drain: clients ride the
1142+
// dying pod until the forced exit below and only learn about the
1143+
// restart from a TCP reset. The 1012 close frame turns a 35s silent
1144+
// death into an immediate, well-attributed reconnect.
1145+
let closed = drain_conn_manager.drain_all();
1146+
info!(
1147+
connections = closed,
1148+
"Sent restart close frame to all live WebSocket connections"
1149+
);
11391150
// Hard timeout: force exit if connections don't drain within 30s.
11401151
tokio::time::sleep(std::time::Duration::from_secs(30)).await;
11411152
tracing::error!("Drain timeout exceeded — forcing exit");

‎crates/buzz-relay/src/router.rs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -299,9 +299,20 @@ async fn nip11_or_ws_handler(
299299

300300
let max_frame_bytes = state.config.max_frame_bytes;
301301
match WebSocketUpgrade::from_request(req, &state).await {
302-
Ok(ws) => limit_relay_websocket(ws, max_frame_bytes)
303-
.on_upgrade(move |socket| handle_connection(socket, state, addr, tenant))
304-
.into_response(),
302+
Ok(ws) => {
303+
// Shutting down: refuse new sockets instead of accepting a
304+
// connection onto a dying pod. Readiness already returns 503, but
305+
// that only stops K8s routing — direct and in-flight upgrades
306+
// still reach here during the pre-drain grace window. Clients
307+
// treat the refusal as a normal dial failure and retry, landing
308+
// on a healthy pod.
309+
if state.shutting_down.load(Ordering::Relaxed) {
310+
return (StatusCode::SERVICE_UNAVAILABLE, "relay restarting").into_response();
311+
}
312+
limit_relay_websocket(ws, max_frame_bytes)
313+
.on_upgrade(move |socket| handle_connection(socket, state, addr, tenant))
314+
.into_response()
315+
}
305316
Err(_) => {
306317
// Browser requesting HTML and Git web GUI is enabled → serve SPA.
307318
if state.config.serve_git_web_gui {

‎crates/buzz-relay/src/state.rs‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,13 +181,18 @@ where
181181
/// Tracks active Nostr WebSocket connections and provides message routing by connection ID.
182182
pub struct ConnectionManager {
183183
connections: DashMap<Uuid, ConnEntry>,
184+
/// Sticky drain flag set by [`Self::drain_all`]. Registrations that land
185+
/// after the drain snapshot self-signal, so no upgrade-vs-shutdown
186+
/// interleaving can produce a connection that misses the restart close.
187+
draining: AtomicBool,
184188
}
185189

186190
impl ConnectionManager {
187191
/// Creates a new, empty connection manager.
188192
pub fn new() -> Self {
189193
Self {
190194
connections: DashMap::new(),
195+
draining: AtomicBool::new(false),
191196
}
192197
}
193198

@@ -208,6 +213,8 @@ impl ConnectionManager {
208213
subscriptions: ConnectionSubscriptions,
209214
grace_limit: u8,
210215
) {
216+
let drain_ctrl_tx = ctrl_tx.clone();
217+
let drain_cancel = cancel.clone();
211218
self.connections.insert(
212219
conn_id,
213220
ConnEntry {
@@ -221,6 +228,14 @@ impl ConnectionManager {
221228
grace_limit,
222229
},
223230
);
231+
// Insert-then-check pairs with drain_all's store-then-iterate: either
232+
// the drain iteration sees this entry, or this check sees the flag.
233+
// A registration that raced past the snapshot self-signals here, so
234+
// no connection can outlive graceful shutdown unclosed.
235+
if self.draining.load(Ordering::SeqCst) {
236+
let _ = drain_ctrl_tx.try_send(Self::restart_close_frame());
237+
drain_cancel.cancel();
238+
}
224239
}
225240

226241
/// Removes a connection from the registry.
@@ -318,6 +333,45 @@ impl ConnectionManager {
318333
closed
319334
}
320335

336+
/// Closes every live connection with a `1012 Service Restart` close frame.
337+
///
338+
/// Called when graceful shutdown starts draining. Without this, upgraded
339+
/// WebSocket connections outlive the axum listener drain: clients ride the
340+
/// dying pod until the forced exit and then learn about the restart from a
341+
/// TCP reset (or, on an abrupt kill, from up to 60s of stall-watchdog
342+
/// silence). The explicit close frame tells them to reconnect immediately
343+
/// — and that the disconnect is a restart, not a policy action.
344+
///
345+
/// Uses the "queue frame on ctrl, then cancel" idiom (see
346+
/// [`ConnectionManager::disconnect_pubkey`]): the send loop drains queued
347+
/// control frames — including this close — before its cancel branch closes
348+
/// the socket. Best-effort: a full control buffer still gets the close via
349+
/// cancel, just without the restart code.
350+
///
351+
/// Returns the number of connections signalled.
352+
pub fn drain_all(&self) -> usize {
353+
// Store-then-iterate pairs with register's insert-then-check: a
354+
// registration that misses this iteration observes the flag and
355+
// self-signals instead. The flag is sticky — drain is one-way.
356+
self.draining.store(true, Ordering::SeqCst);
357+
let frame = Self::restart_close_frame();
358+
let mut closed = 0usize;
359+
for entry in self.connections.iter() {
360+
let _ = entry.ctrl_tx.try_send(frame.clone());
361+
entry.cancel.cancel();
362+
closed += 1;
363+
}
364+
closed
365+
}
366+
367+
/// The WS close frame announcing a graceful restart: 1012 Service Restart.
368+
fn restart_close_frame() -> WsMessage {
369+
WsMessage::Close(Some(axum::extract::ws::CloseFrame {
370+
code: axum::extract::ws::close_code::RESTART,
371+
reason: axum::extract::ws::Utf8Bytes::from_static("relay restarting"),
372+
}))
373+
}
374+
321375
/// Return the server-resolved community that the connection's host bound to.
322376
pub fn community_for_conn(&self, conn_id: Uuid) -> Option<CommunityId> {
323377
self.connections
@@ -1737,4 +1791,142 @@ mod tests {
17371791
"community-B session stays live — ban does not cross the tenant fence"
17381792
);
17391793
}
1794+
1795+
#[tokio::test]
1796+
async fn drain_all_sends_restart_close_and_cancels_every_conn() {
1797+
// Graceful shutdown must tell every live client to reconnect — across
1798+
// all communities — with a 1012 restart close frame queued ahead of
1799+
// the cancel-driven socket close.
1800+
let mgr = ConnectionManager::new();
1801+
1802+
let register = |community| {
1803+
let conn_id = Uuid::new_v4();
1804+
let (tx, _rx) = mpsc::channel(8);
1805+
let (ctrl_tx, ctrl_rx) = mpsc::channel(8);
1806+
let cancel = CancellationToken::new();
1807+
mgr.register(
1808+
conn_id,
1809+
tx,
1810+
ctrl_tx,
1811+
cancel.clone(),
1812+
community,
1813+
Arc::new(AtomicU8::new(0)),
1814+
Arc::new(Mutex::new(HashMap::new())),
1815+
3,
1816+
);
1817+
(ctrl_rx, cancel)
1818+
};
1819+
1820+
let (mut ctrl_a, cancel_a) = register(buzz_core::tenant::CommunityId::from_uuid(
1821+
Uuid::from_u128(0xa),
1822+
));
1823+
let (mut ctrl_b, cancel_b) = register(buzz_core::tenant::CommunityId::from_uuid(
1824+
Uuid::from_u128(0xb),
1825+
));
1826+
1827+
let closed = mgr.drain_all();
1828+
1829+
assert_eq!(closed, 2, "every connection is signalled, no tenant fence");
1830+
assert!(cancel_a.is_cancelled(), "community-A session is cancelled");
1831+
assert!(cancel_b.is_cancelled(), "community-B session is cancelled");
1832+
1833+
for ctrl_rx in [&mut ctrl_a, &mut ctrl_b] {
1834+
let frame = ctrl_rx.try_recv().expect("close frame delivered");
1835+
match frame {
1836+
WsMessage::Close(Some(close)) => {
1837+
assert_eq!(
1838+
close.code,
1839+
axum::extract::ws::close_code::RESTART,
1840+
"close code is 1012 Service Restart"
1841+
);
1842+
assert_eq!(close.reason.as_str(), "relay restarting");
1843+
}
1844+
other => panic!("expected a restart close frame, got {other:?}"),
1845+
}
1846+
}
1847+
}
1848+
1849+
#[tokio::test]
1850+
async fn drain_all_full_control_buffer_still_cancels() {
1851+
// Best-effort delivery: a wedged control channel must not block the
1852+
// drain — the cancel still closes the socket, just without the frame.
1853+
let mgr = ConnectionManager::new();
1854+
let conn_id = Uuid::new_v4();
1855+
let (tx, _rx) = mpsc::channel(8);
1856+
let (ctrl_tx, mut ctrl_rx) = mpsc::channel(1);
1857+
let cancel = CancellationToken::new();
1858+
mgr.register(
1859+
conn_id,
1860+
tx,
1861+
ctrl_tx.clone(),
1862+
cancel.clone(),
1863+
buzz_core::tenant::CommunityId::from_uuid(Uuid::nil()),
1864+
Arc::new(AtomicU8::new(0)),
1865+
Arc::new(Mutex::new(HashMap::new())),
1866+
3,
1867+
);
1868+
// Wedge the 1-slot control channel.
1869+
ctrl_tx
1870+
.try_send(WsMessage::Text("wedge".into()))
1871+
.expect("fill control channel");
1872+
1873+
let closed = mgr.drain_all();
1874+
1875+
assert_eq!(closed, 1);
1876+
assert!(
1877+
cancel.is_cancelled(),
1878+
"cancel fires even when the close frame cannot be queued"
1879+
);
1880+
// Only the wedge frame is present — the close was dropped, not queued.
1881+
assert!(matches!(
1882+
ctrl_rx.try_recv().expect("wedge frame"),
1883+
WsMessage::Text(_)
1884+
));
1885+
assert!(ctrl_rx.try_recv().is_err(), "no second frame queued");
1886+
}
1887+
1888+
#[tokio::test]
1889+
async fn register_after_drain_self_signals_restart_close_and_cancel() {
1890+
// The shutdown-boundary race: an upgrade accepted before SIGTERM can
1891+
// finish its async admission check and register AFTER drain_all's
1892+
// one-shot snapshot. The sticky drain flag makes that interleaving
1893+
// deterministic — register itself queues the 1012 and cancels, so no
1894+
// late registration can ride out graceful shutdown unclosed.
1895+
let mgr = ConnectionManager::new();
1896+
1897+
// Drain with zero connections — sets the sticky flag.
1898+
assert_eq!(mgr.drain_all(), 0);
1899+
1900+
// Late registration lands after the snapshot.
1901+
let conn_id = Uuid::new_v4();
1902+
let (tx, _rx) = mpsc::channel(8);
1903+
let (ctrl_tx, mut ctrl_rx) = mpsc::channel(8);
1904+
let cancel = CancellationToken::new();
1905+
mgr.register(
1906+
conn_id,
1907+
tx,
1908+
ctrl_tx,
1909+
cancel.clone(),
1910+
buzz_core::tenant::CommunityId::from_uuid(Uuid::nil()),
1911+
Arc::new(AtomicU8::new(0)),
1912+
Arc::new(Mutex::new(HashMap::new())),
1913+
3,
1914+
);
1915+
1916+
assert!(
1917+
cancel.is_cancelled(),
1918+
"late registration is cancelled by the sticky drain flag"
1919+
);
1920+
match ctrl_rx.try_recv().expect("close frame delivered") {
1921+
WsMessage::Close(Some(close)) => {
1922+
assert_eq!(
1923+
close.code,
1924+
axum::extract::ws::close_code::RESTART,
1925+
"late registration still gets the 1012 restart close"
1926+
);
1927+
assert_eq!(close.reason.as_str(), "relay restarting");
1928+
}
1929+
other => panic!("expected a restart close frame, got {other:?}"),
1930+
}
1931+
}
17401932
}

0 commit comments

Comments
 (0)