Skip to content

Commit d723518

Browse files
TheSentinel454Elrondclaude
committed
fix(relay): keep index buckets another connection refilled during cleanup
Index cleanup emptied a bucket, released its guard, then removed the key unconditionally. A REQ on another connection could push into the bucket in that gap, and the remove then deleted its live entry, so the new subscriber stopped receiving events. Per-author keys make the last-watcher case ordinary presence churn. The bucket is now dropped only if it is still empty under the shard lock, on every index. Global selector keys are the product of a filter's kinds and its authors or #p values, so one REQ could allocate millions of them. A subscription past 4,096 keys now stays on the kind/wildcard indexes. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 7390ac4 commit d723518

1 file changed

Lines changed: 213 additions & 51 deletions

File tree

‎crates/buzz-relay/src/subscription.rs‎

Lines changed: 213 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
//! Subscription registry with active WebSocket indexes for targeted fan-out.
22
33
use std::collections::{HashMap, HashSet};
4+
use std::hash::Hash;
45

56
use dashmap::DashMap;
67
use nostr::{Alphabet, Filter, Kind, PublicKey, SingleLetterTag};
@@ -589,39 +590,28 @@ impl SubscriptionRegistry {
589590
let ch_id = *ch_id;
590591
match extract_kinds_from_filters(filters) {
591592
// None = wildcard (at least one filter had no kinds constraint).
592-
None => {
593-
// Was in wildcard index.
594-
if let Some(mut entries) =
595-
self.channel_wildcard_index.get_mut(&(community_id, ch_id))
596-
{
597-
entries.retain(|(cid, sid)| !(*cid == conn_id && sid == sub_id));
598-
if entries.is_empty() {
599-
drop(entries);
600-
self.channel_wildcard_index.remove(&(community_id, ch_id));
601-
}
602-
}
603-
}
593+
None => remove_index_entry(
594+
&self.channel_wildcard_index,
595+
&(community_id, ch_id),
596+
conn_id,
597+
sub_id,
598+
),
604599
Some(kinds) if kinds.is_empty() => {
605600
// `kinds: []` subscriptions are never indexed (they match nothing),
606601
// so there is nothing to remove here.
607602
}
608603
Some(kinds) => {
609-
// Was in kind-specific index.
610604
for kind in kinds {
611605
let key = IndexKey {
612606
channel_id: ch_id,
613607
kind,
614608
};
615-
if let Some(mut entries) = self
616-
.channel_kind_index
617-
.get_mut(&(community_id, key.clone()))
618-
{
619-
entries.retain(|(cid, sid)| !(*cid == conn_id && sid == sub_id));
620-
if entries.is_empty() {
621-
drop(entries);
622-
self.channel_kind_index.remove(&(community_id, key));
623-
}
624-
}
609+
remove_index_entry(
610+
&self.channel_kind_index,
611+
&(community_id, key),
612+
conn_id,
613+
sub_id,
614+
);
625615
}
626616
}
627617
}
@@ -630,38 +620,25 @@ impl SubscriptionRegistry {
630620
// Global subscription — remove from the same global index chosen at registration.
631621
if let Some(keys) = extract_global_selector_index_keys(community_id, filters) {
632622
for key in keys {
633-
if let Some(mut entries) = self.global_selector_index.get_mut(&key) {
634-
entries.retain(|(cid, sid)| !(*cid == conn_id && sid == sub_id));
635-
if entries.is_empty() {
636-
drop(entries);
637-
self.global_selector_index.remove(&key);
638-
}
639-
}
623+
remove_index_entry(&self.global_selector_index, &key, conn_id, sub_id);
640624
}
641625
} else {
642626
match extract_kinds_from_filters(filters) {
643-
None => {
644-
if let Some(mut entries) = self.global_wildcard_index.get_mut(&community_id)
645-
{
646-
entries.retain(|(cid, sid)| !(*cid == conn_id && sid == sub_id));
647-
if entries.is_empty() {
648-
drop(entries);
649-
self.global_wildcard_index.remove(&community_id);
650-
}
651-
}
652-
}
627+
None => remove_index_entry(
628+
&self.global_wildcard_index,
629+
&community_id,
630+
conn_id,
631+
sub_id,
632+
),
653633
Some(kinds) if kinds.is_empty() => {}
654634
Some(kinds) => {
655635
for kind in kinds {
656-
if let Some(mut entries) =
657-
self.global_kind_index.get_mut(&(community_id, kind))
658-
{
659-
entries.retain(|(cid, sid)| !(*cid == conn_id && sid == sub_id));
660-
if entries.is_empty() {
661-
drop(entries);
662-
self.global_kind_index.remove(&(community_id, kind));
663-
}
664-
}
636+
remove_index_entry(
637+
&self.global_kind_index,
638+
&(community_id, kind),
639+
conn_id,
640+
sub_id,
641+
);
665642
}
666643
}
667644
}
@@ -670,13 +647,66 @@ impl SubscriptionRegistry {
670647
}
671648
}
672649

650+
/// Removes one subscription from an index bucket, then drops the bucket if it
651+
/// is still empty. The bucket guard is released between the two steps, and a
652+
/// registration from another connection can refill the bucket in that gap, so
653+
/// the drop re-checks emptiness under the shard lock.
654+
fn remove_index_entry<K: Eq + Hash>(
655+
index: &DashMap<K, Vec<(ConnId, SubId)>>,
656+
key: &K,
657+
conn_id: ConnId,
658+
sub_id: &str,
659+
) {
660+
let emptied = match index.get_mut(key) {
661+
Some(mut entries) => {
662+
entries.retain(|(cid, sid)| !(*cid == conn_id && sid == sub_id));
663+
entries.is_empty()
664+
}
665+
None => return,
666+
};
667+
if emptied {
668+
#[cfg(test)]
669+
test_hooks::before_empty_bucket_drop();
670+
index.remove_if(key, |_, entries| entries.is_empty());
671+
}
672+
}
673+
674+
/// Test-only hook into the gap between emptying an index bucket and dropping it.
675+
#[cfg(test)]
676+
mod test_hooks {
677+
use std::cell::RefCell;
678+
679+
thread_local! {
680+
static BEFORE_EMPTY_BUCKET_DROP: RefCell<Option<Box<dyn FnOnce()>>> =
681+
const { RefCell::new(None) };
682+
}
683+
684+
/// Runs `hook` once, on this thread, the next time a bucket is emptied.
685+
pub(super) fn arm_before_empty_bucket_drop(hook: impl FnOnce() + 'static) {
686+
BEFORE_EMPTY_BUCKET_DROP.with(|slot| *slot.borrow_mut() = Some(Box::new(hook)));
687+
}
688+
689+
pub(super) fn before_empty_bucket_drop() {
690+
if let Some(hook) = BEFORE_EMPTY_BUCKET_DROP.with(|slot| slot.borrow_mut().take()) {
691+
hook();
692+
}
693+
}
694+
}
695+
673696
fn p_tag() -> SingleLetterTag {
674697
SingleLetterTag::lowercase(Alphabet::P)
675698
}
676699

700+
/// Most selector-index keys one global subscription may occupy. Keys are the
701+
/// product of a filter's kinds and its `#p` values or authors, so a large
702+
/// filter would otherwise allocate millions of them; past this budget the
703+
/// subscription stays on the kind/wildcard indexes instead.
704+
const MAX_GLOBAL_SELECTOR_KEYS: usize = 4096;
705+
677706
/// Index keys for a global subscription whose every filter is constrained to
678-
/// `#p` recipients, or else to authors, with or without kinds. `None` means it
679-
/// needs the broader kind/wildcard indexes. `#p` wins when a filter names both.
707+
/// `#p` recipients, or else to authors, with or without kinds, within
708+
/// [`MAX_GLOBAL_SELECTOR_KEYS`]. `None` means it needs the broader
709+
/// kind/wildcard indexes. `#p` wins when a filter names both.
680710
fn extract_global_selector_index_keys(
681711
community_id: CommunityId,
682712
filters: &[Filter],
@@ -707,6 +737,7 @@ fn extract_selector_keys(
707737
) -> Option<Vec<GlobalSelectorIndexKey>> {
708738
let mut seen = HashSet::new();
709739
let mut keys = Vec::new();
740+
let mut budget = MAX_GLOBAL_SELECTOR_KEYS;
710741

711742
for filter in filters {
712743
let kinds: Vec<Option<Kind>> = match &filter.kinds {
@@ -719,6 +750,7 @@ fn extract_selector_keys(
719750
if selectors.is_empty() {
720751
return None;
721752
}
753+
budget = budget.checked_sub(kinds.len().saturating_mul(selectors.len()))?;
722754

723755
for kind in kinds {
724756
for selector in &selectors {
@@ -1785,6 +1817,136 @@ mod tests {
17851817
.is_empty());
17861818
}
17871819

1820+
#[test]
1821+
fn index_cleanup_keeps_a_registration_from_another_connection() {
1822+
let presence = Kind::Custom(buzz_core::kind::KIND_PRESENCE_UPDATE as u16);
1823+
let alice = Keys::generate();
1824+
let channel_id = Uuid::new_v4();
1825+
// (case, filter, channel scope, an event the filter routes)
1826+
let cases = [
1827+
(
1828+
"author selector",
1829+
Filter::new().kind(presence).author(alice.public_key()),
1830+
None,
1831+
make_stored_event_by(&alice, presence),
1832+
),
1833+
(
1834+
"global kind",
1835+
Filter::new().kind(Kind::TextNote),
1836+
None,
1837+
make_stored_event(Kind::TextNote, None),
1838+
),
1839+
(
1840+
"global wildcard",
1841+
Filter::new(),
1842+
None,
1843+
make_stored_event(Kind::TextNote, None),
1844+
),
1845+
(
1846+
"channel kind",
1847+
Filter::new().kind(Kind::TextNote),
1848+
Some(channel_id),
1849+
make_stored_event(Kind::TextNote, Some(channel_id)),
1850+
),
1851+
(
1852+
"channel wildcard",
1853+
Filter::new(),
1854+
Some(channel_id),
1855+
make_stored_event(Kind::TextNote, Some(channel_id)),
1856+
),
1857+
];
1858+
1859+
for (case, filter, channel, event) in cases {
1860+
let registry = Arc::new(SubscriptionRegistry::new());
1861+
let leaving = Uuid::new_v4();
1862+
let joining = Uuid::new_v4();
1863+
registry.register(leaving, "s".to_string(), vec![filter.clone()], channel);
1864+
1865+
// The last subscriber leaves; another connection subscribes to the
1866+
// same bucket after it is emptied and before it is dropped.
1867+
let joined = Arc::new(std::sync::atomic::AtomicBool::new(false));
1868+
{
1869+
let registry = Arc::clone(&registry);
1870+
let joined = Arc::clone(&joined);
1871+
test_hooks::arm_before_empty_bucket_drop(move || {
1872+
registry.register(joining, "s".to_string(), vec![filter], channel);
1873+
joined.store(true, std::sync::atomic::Ordering::SeqCst);
1874+
});
1875+
}
1876+
registry.remove_connection(leaving);
1877+
1878+
assert!(
1879+
joined.load(std::sync::atomic::Ordering::SeqCst),
1880+
"{case}: hook did not run"
1881+
);
1882+
assert_eq!(
1883+
registry.fan_out(&event),
1884+
vec![(joining, "s".to_string())],
1885+
"{case}: cleanup dropped the new subscriber's bucket"
1886+
);
1887+
}
1888+
}
1889+
1890+
#[test]
1891+
fn global_selector_index_falls_back_past_the_key_budget() {
1892+
let registry = SubscriptionRegistry::new();
1893+
let kinds = [Kind::Custom(20001), Kind::Custom(20002)];
1894+
let authors: Vec<Keys> = (0..MAX_GLOBAL_SELECTOR_KEYS)
1895+
.map(|_| Keys::generate())
1896+
.collect();
1897+
let pubkeys: Vec<PublicKey> = authors.iter().map(Keys::public_key).collect();
1898+
// One past budget across both kinds.
1899+
let over = MAX_GLOBAL_SELECTOR_KEYS / kinds.len() + 1;
1900+
let p_values: Vec<String> = pubkeys[..over].iter().map(PublicKey::to_hex).collect();
1901+
let conn_id = Uuid::new_v4();
1902+
1903+
// Exactly at budget: one kind per author key.
1904+
registry.register(
1905+
conn_id,
1906+
"at".to_string(),
1907+
vec![Filter::new()
1908+
.kind(kinds[0])
1909+
.authors(pubkeys.iter().copied())],
1910+
None,
1911+
);
1912+
assert_eq!(
1913+
registry.global_selector_index.len(),
1914+
MAX_GLOBAL_SELECTOR_KEYS
1915+
);
1916+
registry.remove_subscription(conn_id, "at");
1917+
assert!(registry.global_selector_index.is_empty());
1918+
1919+
// One past budget, by authors and by `#p`: the generic kind index
1920+
// carries them and still routes every author or recipient.
1921+
for filter in [
1922+
Filter::new()
1923+
.kinds(kinds)
1924+
.authors(pubkeys[..over].iter().copied()),
1925+
Filter::new()
1926+
.kinds(kinds)
1927+
.custom_tags(p_tag(), p_values.iter().cloned()),
1928+
] {
1929+
registry.register(conn_id, "over".to_string(), vec![filter], None);
1930+
assert!(registry.global_selector_index.is_empty());
1931+
assert_eq!(registry.global_kind_index.len(), kinds.len());
1932+
registry.remove_connection(conn_id);
1933+
assert!(registry.global_kind_index.is_empty());
1934+
}
1935+
1936+
registry.register(
1937+
conn_id,
1938+
"over".to_string(),
1939+
vec![Filter::new()
1940+
.kinds(kinds)
1941+
.authors(pubkeys[..over].iter().copied())],
1942+
None,
1943+
);
1944+
assert_eq!(
1945+
registry.fan_out(&make_stored_event_by(&authors[over - 1], kinds[1])),
1946+
vec![(conn_id, "over".to_string())]
1947+
);
1948+
}
1949+
17881950
#[test]
17891951
fn test_global_selector_index_falls_back_unless_every_filter_is_constrained() {
17901952
let registry = SubscriptionRegistry::new();

0 commit comments

Comments
 (0)