Skip to content

Commit 7db92b7

Browse files
author
am
committed
fix: isolate finite websocket window work from shared handler capacity
Bound in-flight finite delivery by process, community, and principal before the shared handler permit. Preserve live queue headroom under concurrent finite senders and restore the terminal EOSE failure fixture. Co-authored-by: Kalvin Chau <[email protected]> Signed-off-by: Kalvin Chau <[email protected]>
1 parent abbe89e commit 7db92b7

4 files changed

Lines changed: 298 additions & 9 deletions

File tree

‎crates/buzz-relay/src/api/bridge/thread_window/postgres_tests.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ async fn websocket_batch_enforces_token_scope_and_rejects_queue_truncation() {
498498

499499
// A queue with room for the page but not EOSE must close, not complete.
500500
let (terminal, mut terminal_rx) = f.ws_conn(None);
501-
for _ in 0..2 {
501+
for _ in 0..1 {
502502
assert!(terminal.send("padding".into()));
503503
}
504504
crate::handlers::req::handle_req(

‎crates/buzz-relay/src/connection.rs‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -856,6 +856,14 @@ async fn handle_text_message(text: String, conn: Arc<ConnectionState>, state: Ar
856856
return;
857857
}
858858

859+
dispatch_client_message(msg, conn, state).await;
860+
}
861+
862+
async fn dispatch_client_message(
863+
msg: ClientMessage,
864+
conn: Arc<ConnectionState>,
865+
state: Arc<AppState>,
866+
) {
859867
match msg {
860868
ClientMessage::Auth(event) => {
861869
// AUTH remains inline so only one frame can race the connection's
@@ -907,6 +915,36 @@ async fn handle_text_message(text: String, conn: Arc<ConnectionState>, state: Ar
907915
} => {
908916
let conn = Arc::clone(&conn);
909917
let state = Arc::clone(&state);
918+
// Admission precedes the shared handler permit. A slow finite
919+
// reader must not be able to occupy the entire process-wide pool.
920+
// Invalid/unauthenticated filters still follow normal REQ validation.
921+
let finite_window = if raw_filters.iter().any(|filter| {
922+
filter
923+
.get("thread_window")
924+
.is_some_and(|value| value != false)
925+
}) {
926+
match conn.auth_state_snapshot() {
927+
AuthState::Authenticated(ctx) => {
928+
let principal = ctx.agent_owner_pubkey.unwrap_or(ctx.pubkey);
929+
match state
930+
.finite_window_work
931+
.try_acquire((conn.tenant.community(), principal.to_bytes()))
932+
{
933+
Some(permit) => Some(permit),
934+
None => {
935+
conn.send(request_rejection_message(
936+
RejectionTarget::Subscription(&sub_id),
937+
"rate-limited: too many concurrent thread windows",
938+
));
939+
return;
940+
}
941+
}
942+
}
943+
_ => None,
944+
}
945+
} else {
946+
None
947+
};
910948
let permit = match state.handler_semaphore.clone().try_acquire_owned() {
911949
Ok(p) => p,
912950
Err(_) => {
@@ -930,6 +968,7 @@ async fn handle_text_message(text: String, conn: Arc<ConnectionState>, state: Ar
930968
)
931969
.await;
932970
drop(permit);
971+
drop(finite_window);
933972
}
934973
.instrument(span),
935974
);
@@ -1557,6 +1596,96 @@ pub(crate) mod tests {
15571596
assert_eq!(frame[1], "history-abc");
15581597
}
15591598

1599+
/// A saturated tenant is rejected before occupying another tenant's
1600+
/// handler permits. This binds the actual REQ dispatcher, not a quota-only
1601+
/// helper; Redis admission is outside this isolated test seam.
1602+
#[tokio::test]
1603+
async fn finite_window_quota_preserves_other_community_handler_capacity() {
1604+
let state = crate::state::tests::test_state().await;
1605+
let (mut attacker, mut attacker_rx) = test_conn_with_auth(authenticated_state());
1606+
let a = buzz_core::CommunityId::from_uuid(Uuid::new_v4());
1607+
let b = buzz_core::CommunityId::from_uuid(Uuid::new_v4());
1608+
Arc::get_mut(&mut attacker).unwrap().tenant = TenantContext::resolved(a, "attacker.local");
1609+
let AuthState::Authenticated(ctx) = attacker.auth_state_snapshot() else {
1610+
panic!("authenticated test connection");
1611+
};
1612+
let key = (a, ctx.pubkey.to_bytes());
1613+
let held: Vec<_> = (0..4)
1614+
.map(|_| state.finite_window_work.try_acquire(key).unwrap())
1615+
.collect();
1616+
let saturation: Vec<_> = (4..state.finite_window_work.community_limit())
1617+
.map(|n| {
1618+
state
1619+
.finite_window_work
1620+
.try_acquire((a, [n as u8; 32]))
1621+
.unwrap()
1622+
})
1623+
.collect();
1624+
let slots_before = state.handler_semaphore.available_permits();
1625+
let raw = serde_json::json!(["REQ", "saturated", {"thread_window": true}]).to_string();
1626+
dispatch_client_message(
1627+
ClientMessage::parse(&raw).unwrap(),
1628+
Arc::clone(&attacker),
1629+
Arc::clone(&state),
1630+
)
1631+
.await;
1632+
let rejected = read_frame(&mut attacker_rx);
1633+
assert_eq!(rejected[0], "CLOSED");
1634+
assert_eq!(rejected[1], "saturated");
1635+
assert_eq!(
1636+
rejected[2],
1637+
"rate-limited: too many concurrent thread windows"
1638+
);
1639+
assert_eq!(state.handler_semaphore.available_permits(), slots_before);
1640+
let neighbor = state.finite_window_work.try_acquire((b, [8; 32])).unwrap();
1641+
assert_eq!(state.handler_semaphore.available_permits(), slots_before);
1642+
drop(neighbor);
1643+
drop(saturation);
1644+
drop(held);
1645+
1646+
// The other community remains able to enter the same production REQ
1647+
// dispatcher while the first tenant's finite quota is saturated.
1648+
// Hold all but one handler permit to make admission observable.
1649+
let attacker_held: Vec<_> = (0..state.finite_window_work.community_limit())
1650+
.map(|n| {
1651+
state
1652+
.finite_window_work
1653+
.try_acquire((a, [n as u8; 32]))
1654+
.unwrap()
1655+
})
1656+
.collect();
1657+
let capacity = state.handler_semaphore.available_permits();
1658+
let _busy = Arc::clone(&state.handler_semaphore)
1659+
.acquire_many_owned(capacity as u32 - 1)
1660+
.await
1661+
.unwrap();
1662+
// An invalid finite query terminates without touching Postgres but
1663+
// travels through the same permit acquisition and release.
1664+
let (mut neighbor_conn, mut neighbor_rx) = test_conn_with_auth(authenticated_state());
1665+
Arc::get_mut(&mut neighbor_conn).unwrap().tenant =
1666+
TenantContext::resolved(b, "neighbor.local");
1667+
let neighbor_raw =
1668+
serde_json::json!(["REQ", "neighbor", {"thread_window": true}]).to_string();
1669+
dispatch_client_message(
1670+
ClientMessage::parse(&neighbor_raw).unwrap(),
1671+
Arc::clone(&neighbor_conn),
1672+
Arc::clone(&state),
1673+
)
1674+
.await;
1675+
let response = tokio::time::timeout(Duration::from_secs(2), neighbor_rx.recv())
1676+
.await
1677+
.unwrap()
1678+
.unwrap();
1679+
let response: serde_json::Value =
1680+
serde_json::from_str(response.to_text().unwrap()).unwrap();
1681+
assert_eq!(response[0], "CLOSED");
1682+
assert_eq!(response[1], "neighbor");
1683+
assert!(response[2].as_str().unwrap().starts_with("invalid:"));
1684+
assert_eq!(state.handler_semaphore.available_permits(), 1);
1685+
drop(attacker_held);
1686+
assert!(state.finite_window_work.try_acquire(key).is_some());
1687+
}
1688+
15601689
/// COUNT refusals follow NIP-45 and close the named query.
15611690
#[tokio::test]
15621691
async fn saturated_handler_rejects_a_count_on_the_closed_channel() {

‎crates/buzz-relay/src/handlers/req.rs‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -600,7 +600,7 @@ async fn send_thread_window_frame(
600600
let reserve = (conn.send_tx.max_capacity() / 4)
601601
.max(1)
602602
.min(conn.send_tx.max_capacity().saturating_sub(1));
603-
let mut frame = axum::extract::ws::Message::Text(frame.into());
603+
let frame = axum::extract::ws::Message::Text(frame.into());
604604
loop {
605605
if conn.cancel.is_cancelled() || tokio::time::Instant::now() >= deadline {
606606
return false;
@@ -609,16 +609,18 @@ async fn send_thread_window_frame(
609609
return false;
610610
}
611611
if conn.send_tx.capacity() > reserve {
612-
match conn.send_tx.try_send(frame) {
613-
Ok(()) => {
612+
if let Ok(permit) = conn.send_tx.try_reserve() {
613+
// The permit is already subtracted from capacity. Holding it
614+
// during this check makes concurrent finite senders respect
615+
// the reserve; live producers only see transient contention.
616+
if conn.send_tx.capacity() >= reserve {
617+
permit.send(frame);
614618
conn.backpressure_count
615619
.store(0, std::sync::atomic::Ordering::Relaxed);
616620
return true;
617621
}
618-
Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => return false,
619-
Err(tokio::sync::mpsc::error::TrySendError::Full(message)) => {
620-
frame = message;
621-
}
622+
} else if conn.send_tx.is_closed() {
623+
return false;
622624
}
623625
}
624626
// No reservation: a live producer can take the reserved slots before
@@ -2777,6 +2779,28 @@ mod thread_window_queue_tests {
27772779
assert!(!conn.cancel.is_cancelled());
27782780
}
27792781

2782+
#[tokio::test]
2783+
async fn concurrent_finite_senders_preserve_live_headroom() {
2784+
let (conn, _rx) = crate::connection::tests::test_conn_with_auth(
2785+
crate::connection::tests::authenticated_state(),
2786+
);
2787+
let results = futures_util::future::join_all((0..12).map(|n| {
2788+
let conn = conn.clone();
2789+
async move {
2790+
send_thread_window_frame(
2791+
&conn,
2792+
format!("finite-{n}"),
2793+
Instant::now() + Duration::from_millis(30),
2794+
)
2795+
.await
2796+
}
2797+
}))
2798+
.await;
2799+
assert_eq!(results.iter().filter(|&&sent| sent).count(), 3);
2800+
assert_eq!(conn.send_tx.capacity(), 1);
2801+
assert!(conn.send("live".into()), "reserved slot remains available");
2802+
}
2803+
27802804
#[tokio::test]
27812805
async fn finite_frame_stalled_reader_expires_without_filling_reserved_slot() {
27822806
let (conn, mut rx) = crate::connection::tests::test_conn_with_auth(

‎crates/buzz-relay/src/state.rs‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use std::collections::{HashMap, HashSet};
44
use std::future::Future;
55
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering};
6-
use std::sync::Arc;
6+
use std::sync::{Arc, Mutex as StdMutex, PoisonError};
77
use std::time::Instant;
88

99
use axum::body::Bytes;
@@ -36,6 +36,139 @@ use crate::subscription::SubscriptionRegistry;
3636

3737
pub(crate) type ScopedPubkeyKey = (CommunityId, [u8; 32]);
3838

39+
// Finite WebSocket windows hold a handler permit until their output is queued.
40+
// Bound that lifetime before acquiring the shared handler permit. The total
41+
// cap leaves capacity for EVENT/COUNT/legacy REQ; the community cap prevents
42+
// multiple identities in one tenant from consuming every finite slot. At the
43+
// default 1024-handler capacity: 256 total, 128/community, 4/principal.
44+
// Reduced test deployments (8 handlers) admit one/community, two total.
45+
// These are per-process limits; a multi-pod deployment needs a shared quota
46+
// to make a fleet-wide fairness guarantee.
47+
pub(crate) struct FiniteWindowWork {
48+
limits: (usize, usize, usize), // global, community, principal
49+
counts: StdMutex<FiniteWindowCounts>,
50+
}
51+
52+
#[derive(Default)]
53+
struct FiniteWindowCounts {
54+
total: usize,
55+
communities: HashMap<CommunityId, usize>,
56+
principals: HashMap<ScopedPubkeyKey, usize>,
57+
}
58+
59+
pub(crate) struct FiniteWindowPermit {
60+
work: Arc<FiniteWindowWork>,
61+
key: ScopedPubkeyKey,
62+
}
63+
64+
impl FiniteWindowWork {
65+
fn new(handler_capacity: usize) -> Self {
66+
Self {
67+
limits: (
68+
(handler_capacity / 4).max(1),
69+
(handler_capacity / 8).max(1),
70+
4,
71+
),
72+
counts: StdMutex::new(FiniteWindowCounts::default()),
73+
}
74+
}
75+
76+
#[cfg(test)]
77+
pub(crate) fn community_limit(&self) -> usize {
78+
self.limits.1
79+
}
80+
81+
pub(crate) fn try_acquire(
82+
self: &Arc<Self>,
83+
key: ScopedPubkeyKey,
84+
) -> Option<FiniteWindowPermit> {
85+
let mut counts = self.counts.lock().unwrap_or_else(PoisonError::into_inner);
86+
if counts.total >= self.limits.0
87+
|| counts.communities.get(&key.0).copied().unwrap_or(0) >= self.limits.1
88+
|| counts.principals.get(&key).copied().unwrap_or(0) >= self.limits.2
89+
{
90+
return None;
91+
}
92+
counts.total += 1;
93+
*counts.communities.entry(key.0).or_default() += 1;
94+
*counts.principals.entry(key).or_default() += 1;
95+
Some(FiniteWindowPermit {
96+
work: Arc::clone(self),
97+
key,
98+
})
99+
}
100+
}
101+
102+
impl Drop for FiniteWindowPermit {
103+
fn drop(&mut self) {
104+
let mut counts = self
105+
.work
106+
.counts
107+
.lock()
108+
.unwrap_or_else(PoisonError::into_inner);
109+
counts.total -= 1;
110+
if let Some(community) = counts.communities.get_mut(&self.key.0) {
111+
*community -= 1;
112+
if *community == 0 {
113+
counts.communities.remove(&self.key.0);
114+
}
115+
}
116+
if let Some(principal) = counts.principals.get_mut(&self.key) {
117+
*principal -= 1;
118+
if *principal == 0 {
119+
counts.principals.remove(&self.key);
120+
}
121+
}
122+
}
123+
}
124+
125+
#[cfg(test)]
126+
mod finite_window_work_tests {
127+
use super::*;
128+
129+
#[test]
130+
fn finite_work_is_bounded_by_principal_community_and_global_capacity() {
131+
let work = Arc::new(FiniteWindowWork::new(64));
132+
let a = CommunityId::from_uuid(Uuid::new_v4());
133+
let b = CommunityId::from_uuid(Uuid::new_v4());
134+
let permits: Vec<_> = (0..4)
135+
.map(|_| work.try_acquire((a, [1; 32])).unwrap())
136+
.collect();
137+
assert!(work.try_acquire((a, [1; 32])).is_none(), "principal cap");
138+
let neighbors: Vec<_> = (2..6)
139+
.map(|n| work.try_acquire((a, [n; 32])).unwrap())
140+
.collect();
141+
assert!(work.try_acquire((a, [9; 32])).is_none(), "community cap");
142+
let neighbor: Vec<_> = (0..8)
143+
.map(|n| work.try_acquire((b, [n; 32])).unwrap())
144+
.collect();
145+
let c = CommunityId::from_uuid(Uuid::new_v4());
146+
assert!(work.try_acquire((c, [9; 32])).is_none(), "global cap");
147+
drop(permits);
148+
assert!(
149+
work.try_acquire((a, [1; 32])).is_some(),
150+
"release on completion"
151+
);
152+
drop(neighbors);
153+
drop(neighbor);
154+
assert_eq!(work.counts.lock().unwrap().total, 0);
155+
}
156+
157+
#[test]
158+
fn reduced_handler_limit_leaves_non_finite_capacity() {
159+
let work = Arc::new(FiniteWindowWork::new(8));
160+
let a = CommunityId::from_uuid(Uuid::new_v4());
161+
let b = CommunityId::from_uuid(Uuid::new_v4());
162+
let attacker = work.try_acquire((a, [1; 32])).unwrap();
163+
assert!(work.try_acquire((a, [2; 32])).is_none());
164+
let other = work.try_acquire((b, [3; 32])).unwrap();
165+
assert!(work.try_acquire((b, [4; 32])).is_none());
166+
drop(attacker);
167+
assert!(work.try_acquire((a, [2; 32])).is_some());
168+
drop(other);
169+
}
170+
}
171+
39172
/// Why a community-bound socket is being asked to stop.
40173
///
41174
/// Only deletion is externally attributed today. Ordinary lifecycle exits keep
@@ -656,6 +789,8 @@ pub struct AppState {
656789
pub conn_semaphore: Arc<Semaphore>,
657790
/// Semaphore limiting concurrent message handler tasks.
658791
pub handler_semaphore: Arc<Semaphore>,
792+
/// In-flight finite WS windows, held through the last EVENT/EOSE or cancellation.
793+
pub(crate) finite_window_work: Arc<FiniteWindowWork>,
659794
/// Semaphore limiting concurrent git subprocess operations across
660795
/// the whole relay. Bounds resource use; **not** writer
661796
/// serialization — that's the CAS at the manifest pointer (spec
@@ -881,6 +1016,7 @@ impl AppState {
8811016
community_disconnect_publish_attempts: Arc::new(AtomicU64::new(0)),
8821017
conn_semaphore: Arc::new(Semaphore::new(max_connections)),
8831018
handler_semaphore: Arc::new(Semaphore::new(max_concurrent_handlers)),
1019+
finite_window_work: Arc::new(FiniteWindowWork::new(max_concurrent_handlers)),
8841020
git_semaphore: Arc::new(Semaphore::new(git_max_concurrent_ops)),
8851021
media_upload_semaphore: Arc::new(Semaphore::new(media_max_concurrent_uploads)),
8861022
workflow_engine,

0 commit comments

Comments
 (0)