@@ -181,13 +181,18 @@ where
181181/// Tracks active Nostr WebSocket connections and provides message routing by connection ID.
182182pub struct ConnectionManager {
183183 connections : DashMap < Uuid , ConnEntry > ,
184+ /// Sticky drain flag set by [`Self::drain_all`]. Registrations that land
185+ /// after the drain snapshot self-signal, so no upgrade-vs-shutdown
186+ /// interleaving can produce a connection that misses the restart close.
187+ draining : AtomicBool ,
184188}
185189
186190impl ConnectionManager {
187191 /// Creates a new, empty connection manager.
188192 pub fn new ( ) -> Self {
189193 Self {
190194 connections : DashMap :: new ( ) ,
195+ draining : AtomicBool :: new ( false ) ,
191196 }
192197 }
193198
@@ -208,6 +213,8 @@ impl ConnectionManager {
208213 subscriptions : ConnectionSubscriptions ,
209214 grace_limit : u8 ,
210215 ) {
216+ let drain_ctrl_tx = ctrl_tx. clone ( ) ;
217+ let drain_cancel = cancel. clone ( ) ;
211218 self . connections . insert (
212219 conn_id,
213220 ConnEntry {
@@ -221,6 +228,14 @@ impl ConnectionManager {
221228 grace_limit,
222229 } ,
223230 ) ;
231+ // Insert-then-check pairs with drain_all's store-then-iterate: either
232+ // the drain iteration sees this entry, or this check sees the flag.
233+ // A registration that raced past the snapshot self-signals here, so
234+ // no connection can outlive graceful shutdown unclosed.
235+ if self . draining . load ( Ordering :: SeqCst ) {
236+ let _ = drain_ctrl_tx. try_send ( Self :: restart_close_frame ( ) ) ;
237+ drain_cancel. cancel ( ) ;
238+ }
224239 }
225240
226241 /// Removes a connection from the registry.
@@ -318,6 +333,45 @@ impl ConnectionManager {
318333 closed
319334 }
320335
336+ /// Closes every live connection with a `1012 Service Restart` close frame.
337+ ///
338+ /// Called when graceful shutdown starts draining. Without this, upgraded
339+ /// WebSocket connections outlive the axum listener drain: clients ride the
340+ /// dying pod until the forced exit and then learn about the restart from a
341+ /// TCP reset (or, on an abrupt kill, from up to 60s of stall-watchdog
342+ /// silence). The explicit close frame tells them to reconnect immediately
343+ /// — and that the disconnect is a restart, not a policy action.
344+ ///
345+ /// Uses the "queue frame on ctrl, then cancel" idiom (see
346+ /// [`ConnectionManager::disconnect_pubkey`]): the send loop drains queued
347+ /// control frames — including this close — before its cancel branch closes
348+ /// the socket. Best-effort: a full control buffer still gets the close via
349+ /// cancel, just without the restart code.
350+ ///
351+ /// Returns the number of connections signalled.
352+ pub fn drain_all ( & self ) -> usize {
353+ // Store-then-iterate pairs with register's insert-then-check: a
354+ // registration that misses this iteration observes the flag and
355+ // self-signals instead. The flag is sticky — drain is one-way.
356+ self . draining . store ( true , Ordering :: SeqCst ) ;
357+ let frame = Self :: restart_close_frame ( ) ;
358+ let mut closed = 0usize ;
359+ for entry in self . connections . iter ( ) {
360+ let _ = entry. ctrl_tx . try_send ( frame. clone ( ) ) ;
361+ entry. cancel . cancel ( ) ;
362+ closed += 1 ;
363+ }
364+ closed
365+ }
366+
367+ /// The WS close frame announcing a graceful restart: 1012 Service Restart.
368+ fn restart_close_frame ( ) -> WsMessage {
369+ WsMessage :: Close ( Some ( axum:: extract:: ws:: CloseFrame {
370+ code : axum:: extract:: ws:: close_code:: RESTART ,
371+ reason : axum:: extract:: ws:: Utf8Bytes :: from_static ( "relay restarting" ) ,
372+ } ) )
373+ }
374+
321375 /// Return the server-resolved community that the connection's host bound to.
322376 pub fn community_for_conn ( & self , conn_id : Uuid ) -> Option < CommunityId > {
323377 self . connections
@@ -1737,4 +1791,142 @@ mod tests {
17371791 "community-B session stays live — ban does not cross the tenant fence"
17381792 ) ;
17391793 }
1794+
1795+ #[ tokio:: test]
1796+ async fn drain_all_sends_restart_close_and_cancels_every_conn ( ) {
1797+ // Graceful shutdown must tell every live client to reconnect — across
1798+ // all communities — with a 1012 restart close frame queued ahead of
1799+ // the cancel-driven socket close.
1800+ let mgr = ConnectionManager :: new ( ) ;
1801+
1802+ let register = |community| {
1803+ let conn_id = Uuid :: new_v4 ( ) ;
1804+ let ( tx, _rx) = mpsc:: channel ( 8 ) ;
1805+ let ( ctrl_tx, ctrl_rx) = mpsc:: channel ( 8 ) ;
1806+ let cancel = CancellationToken :: new ( ) ;
1807+ mgr. register (
1808+ conn_id,
1809+ tx,
1810+ ctrl_tx,
1811+ cancel. clone ( ) ,
1812+ community,
1813+ Arc :: new ( AtomicU8 :: new ( 0 ) ) ,
1814+ Arc :: new ( Mutex :: new ( HashMap :: new ( ) ) ) ,
1815+ 3 ,
1816+ ) ;
1817+ ( ctrl_rx, cancel)
1818+ } ;
1819+
1820+ let ( mut ctrl_a, cancel_a) = register ( buzz_core:: tenant:: CommunityId :: from_uuid (
1821+ Uuid :: from_u128 ( 0xa ) ,
1822+ ) ) ;
1823+ let ( mut ctrl_b, cancel_b) = register ( buzz_core:: tenant:: CommunityId :: from_uuid (
1824+ Uuid :: from_u128 ( 0xb ) ,
1825+ ) ) ;
1826+
1827+ let closed = mgr. drain_all ( ) ;
1828+
1829+ assert_eq ! ( closed, 2 , "every connection is signalled, no tenant fence" ) ;
1830+ assert ! ( cancel_a. is_cancelled( ) , "community-A session is cancelled" ) ;
1831+ assert ! ( cancel_b. is_cancelled( ) , "community-B session is cancelled" ) ;
1832+
1833+ for ctrl_rx in [ & mut ctrl_a, & mut ctrl_b] {
1834+ let frame = ctrl_rx. try_recv ( ) . expect ( "close frame delivered" ) ;
1835+ match frame {
1836+ WsMessage :: Close ( Some ( close) ) => {
1837+ assert_eq ! (
1838+ close. code,
1839+ axum:: extract:: ws:: close_code:: RESTART ,
1840+ "close code is 1012 Service Restart"
1841+ ) ;
1842+ assert_eq ! ( close. reason. as_str( ) , "relay restarting" ) ;
1843+ }
1844+ other => panic ! ( "expected a restart close frame, got {other:?}" ) ,
1845+ }
1846+ }
1847+ }
1848+
1849+ #[ tokio:: test]
1850+ async fn drain_all_full_control_buffer_still_cancels ( ) {
1851+ // Best-effort delivery: a wedged control channel must not block the
1852+ // drain — the cancel still closes the socket, just without the frame.
1853+ let mgr = ConnectionManager :: new ( ) ;
1854+ let conn_id = Uuid :: new_v4 ( ) ;
1855+ let ( tx, _rx) = mpsc:: channel ( 8 ) ;
1856+ let ( ctrl_tx, mut ctrl_rx) = mpsc:: channel ( 1 ) ;
1857+ let cancel = CancellationToken :: new ( ) ;
1858+ mgr. register (
1859+ conn_id,
1860+ tx,
1861+ ctrl_tx. clone ( ) ,
1862+ cancel. clone ( ) ,
1863+ buzz_core:: tenant:: CommunityId :: from_uuid ( Uuid :: nil ( ) ) ,
1864+ Arc :: new ( AtomicU8 :: new ( 0 ) ) ,
1865+ Arc :: new ( Mutex :: new ( HashMap :: new ( ) ) ) ,
1866+ 3 ,
1867+ ) ;
1868+ // Wedge the 1-slot control channel.
1869+ ctrl_tx
1870+ . try_send ( WsMessage :: Text ( "wedge" . into ( ) ) )
1871+ . expect ( "fill control channel" ) ;
1872+
1873+ let closed = mgr. drain_all ( ) ;
1874+
1875+ assert_eq ! ( closed, 1 ) ;
1876+ assert ! (
1877+ cancel. is_cancelled( ) ,
1878+ "cancel fires even when the close frame cannot be queued"
1879+ ) ;
1880+ // Only the wedge frame is present — the close was dropped, not queued.
1881+ assert ! ( matches!(
1882+ ctrl_rx. try_recv( ) . expect( "wedge frame" ) ,
1883+ WsMessage :: Text ( _)
1884+ ) ) ;
1885+ assert ! ( ctrl_rx. try_recv( ) . is_err( ) , "no second frame queued" ) ;
1886+ }
1887+
1888+ #[ tokio:: test]
1889+ async fn register_after_drain_self_signals_restart_close_and_cancel ( ) {
1890+ // The shutdown-boundary race: an upgrade accepted before SIGTERM can
1891+ // finish its async admission check and register AFTER drain_all's
1892+ // one-shot snapshot. The sticky drain flag makes that interleaving
1893+ // deterministic — register itself queues the 1012 and cancels, so no
1894+ // late registration can ride out graceful shutdown unclosed.
1895+ let mgr = ConnectionManager :: new ( ) ;
1896+
1897+ // Drain with zero connections — sets the sticky flag.
1898+ assert_eq ! ( mgr. drain_all( ) , 0 ) ;
1899+
1900+ // Late registration lands after the snapshot.
1901+ let conn_id = Uuid :: new_v4 ( ) ;
1902+ let ( tx, _rx) = mpsc:: channel ( 8 ) ;
1903+ let ( ctrl_tx, mut ctrl_rx) = mpsc:: channel ( 8 ) ;
1904+ let cancel = CancellationToken :: new ( ) ;
1905+ mgr. register (
1906+ conn_id,
1907+ tx,
1908+ ctrl_tx,
1909+ cancel. clone ( ) ,
1910+ buzz_core:: tenant:: CommunityId :: from_uuid ( Uuid :: nil ( ) ) ,
1911+ Arc :: new ( AtomicU8 :: new ( 0 ) ) ,
1912+ Arc :: new ( Mutex :: new ( HashMap :: new ( ) ) ) ,
1913+ 3 ,
1914+ ) ;
1915+
1916+ assert ! (
1917+ cancel. is_cancelled( ) ,
1918+ "late registration is cancelled by the sticky drain flag"
1919+ ) ;
1920+ match ctrl_rx. try_recv ( ) . expect ( "close frame delivered" ) {
1921+ WsMessage :: Close ( Some ( close) ) => {
1922+ assert_eq ! (
1923+ close. code,
1924+ axum:: extract:: ws:: close_code:: RESTART ,
1925+ "late registration still gets the 1012 restart close"
1926+ ) ;
1927+ assert_eq ! ( close. reason. as_str( ) , "relay restarting" ) ;
1928+ }
1929+ other => panic ! ( "expected a restart close frame, got {other:?}" ) ,
1930+ }
1931+ }
17401932}
0 commit comments