Skip to content

Commit 69237ef

Browse files
author
npub1mprnacetjua2xx3p5eddmhxyk6wv929ymm5py8kd2xfxurxahspqqlgyta
committed
feat(pubsub): scope redis topics by community
Co-authored-by: npub1mprnacetjua2xx3p5eddmhxyk6wv929ymm5py8kd2xfxurxahspqqlgyta <d8473ee32b973aa31a21a65adddcc4b69cc2a8a4dee8121ecd51926e0cddbc02@sprout-oss.stage.blox.sqprod.co> Signed-off-by: npub1mprnacetjua2xx3p5eddmhxyk6wv929ymm5py8kd2xfxurxahspqqlgyta <d8473ee32b973aa31a21a65adddcc4b69cc2a8a4dee8121ecd51926e0cddbc02@sprout-oss.stage.blox.sqprod.co>
1 parent e349d76 commit 69237ef

6 files changed

Lines changed: 770 additions & 133 deletions

File tree

‎crates/buzz-pubsub/src/cache_invalidation.rs‎

Lines changed: 101 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,48 @@
1111
//! universal delivery-enforcement point, so dropping the stale key is
1212
//! sufficient: the next read re-fetches authoritative state from the DB.
1313
14+
use buzz_core::{CommunityId, TenantContext};
1415
use futures_util::StreamExt;
1516
use serde::{Deserialize, Serialize};
1617
use tokio::sync::broadcast;
1718
use uuid::Uuid;
1819

19-
/// Redis pub/sub channel for cache-invalidation messages. Distinct from the
20-
/// `buzz:channel:*` event topic so the two streams never interfere.
21-
pub const CACHE_INVALIDATION_CHANNEL: &str = "buzz:cache-invalidate";
20+
use crate::topic::BUZZ_PREFIX;
21+
22+
/// Tenant-local Redis pub/sub channel suffix for cache-invalidation messages.
23+
pub const CACHE_INVALIDATION_SUFFIX: &str = "cache-invalidate";
24+
25+
/// Pattern used by the subscriber to receive cache invalidations for all
26+
/// communities this pod may have cached locally.
27+
pub const CACHE_INVALIDATION_PATTERN: &str = "buzz:*:cache-invalidate";
28+
29+
/// Redis pub/sub channel for cache-invalidation messages under `ctx`.
30+
pub fn cache_invalidation_channel(ctx: &TenantContext) -> String {
31+
format!(
32+
"{BUZZ_PREFIX}:{}:{CACHE_INVALIDATION_SUFFIX}",
33+
ctx.community()
34+
)
35+
}
36+
37+
/// Parse a cache-invalidation Redis channel into its scoped community id.
38+
pub fn parse_cache_invalidation_channel(channel: &str) -> Option<CommunityId> {
39+
let mut parts = channel.split(':');
40+
if parts.next()? != BUZZ_PREFIX {
41+
return None;
42+
}
43+
let community_id = Uuid::parse_str(parts.next()?).ok()?;
44+
if parts.next()? != CACHE_INVALIDATION_SUFFIX {
45+
return None;
46+
}
47+
if parts.next().is_some() {
48+
return None;
49+
}
50+
Some(CommunityId::from_uuid(community_id))
51+
}
2252

2353
/// A cache-key drop to apply on every pod. Each variant mirrors exactly one of
24-
/// the relay's local `invalidate_*` operations.
54+
/// the relay's local `invalidate_*` operations. The community is carried by
55+
/// [`ScopedCacheInvalidation`], not by the tenant-local operation.
2556
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
2657
#[serde(tag = "op")]
2758
pub enum CacheInvalidation {
@@ -47,19 +78,28 @@ pub enum CacheInvalidation {
4778
ChannelDeleted,
4879
}
4980

81+
/// A cache invalidation received from a community-scoped Redis channel.
82+
#[derive(Debug, Clone, PartialEq, Eq)]
83+
pub struct ScopedCacheInvalidation {
84+
/// Community whose local cache key should be dropped.
85+
pub community_id: CommunityId,
86+
/// Tenant-local cache invalidation operation.
87+
pub invalidation: CacheInvalidation,
88+
}
89+
5090
/// Initial reconnect backoff (1 second).
5191
const BACKOFF_INITIAL_SECS: u64 = 1;
5292
/// Maximum reconnect backoff (30 seconds).
5393
const BACKOFF_MAX_SECS: u64 = 30;
5494

55-
/// Subscribes to `buzz:cache-invalidate` and forwards drops to the broadcast.
95+
/// Subscribes to `buzz:*:cache-invalidate` and forwards scoped drops to the broadcast.
5696
///
5797
/// Mirrors `subscriber::run_subscriber`: a reconnect loop with exponential
5898
/// backoff (1s → 2s → 4s → … → 30s max). Never returns — runs for the lifetime
5999
/// of the relay.
60100
pub async fn run_cache_invalidation_subscriber(
61101
redis_url: String,
62-
broadcast_tx: broadcast::Sender<CacheInvalidation>,
102+
broadcast_tx: broadcast::Sender<ScopedCacheInvalidation>,
63103
) {
64104
let mut backoff_secs = BACKOFF_INITIAL_SECS;
65105

@@ -87,19 +127,25 @@ pub async fn run_cache_invalidation_subscriber(
87127

88128
async fn connect_and_subscribe(
89129
redis_url: &str,
90-
broadcast_tx: &broadcast::Sender<CacheInvalidation>,
130+
broadcast_tx: &broadcast::Sender<ScopedCacheInvalidation>,
91131
) -> Result<(), redis::RedisError> {
92132
let client = redis::Client::open(redis_url)?;
93133
let mut conn = client.get_async_pubsub().await?;
94134

95-
conn.subscribe(CACHE_INVALIDATION_CHANNEL).await?;
135+
conn.psubscribe(CACHE_INVALIDATION_PATTERN).await?;
96136

97137
tracing::info!(
98-
"Redis cache-invalidation subscriber connected — listening on {CACHE_INVALIDATION_CHANNEL}"
138+
"Redis cache-invalidation subscriber connected — listening on {CACHE_INVALIDATION_PATTERN}"
99139
);
100140

101141
let mut stream = conn.on_message();
102142
while let Some(msg) = stream.next().await {
143+
let channel = msg.get_channel_name();
144+
let Some(community_id) = parse_cache_invalidation_channel(channel) else {
145+
tracing::warn!("Received cache-invalidation message on unexpected channel: {channel}");
146+
continue;
147+
};
148+
103149
let payload: String = match msg.get_payload() {
104150
Ok(p) => p,
105151
Err(e) => {
@@ -116,7 +162,12 @@ async fn connect_and_subscribe(
116162
}
117163
};
118164

119-
if broadcast_tx.send(invalidation).is_err() {
165+
let scoped = ScopedCacheInvalidation {
166+
community_id,
167+
invalidation,
168+
};
169+
170+
if broadcast_tx.send(scoped).is_err() {
120171
tracing::trace!("No cache-invalidation receivers — message dropped");
121172
}
122173
}
@@ -128,6 +179,46 @@ async fn connect_and_subscribe(
128179
mod tests {
129180
use super::*;
130181

182+
fn ctx(id: u128, host: &str) -> TenantContext {
183+
TenantContext::resolved(CommunityId::from_uuid(Uuid::from_u128(id)), host)
184+
}
185+
186+
#[test]
187+
fn cache_invalidation_channel_is_community_scoped() {
188+
let community_a = ctx(0xaaaa, "a.example");
189+
let community_b = ctx(0xbbbb, "b.example");
190+
191+
assert_eq!(
192+
cache_invalidation_channel(&community_a),
193+
format!("buzz:{}:cache-invalidate", community_a.community())
194+
);
195+
assert_ne!(
196+
cache_invalidation_channel(&community_a),
197+
cache_invalidation_channel(&community_b)
198+
);
199+
}
200+
201+
#[test]
202+
fn parses_cache_invalidation_channel() {
203+
let community_id = CommunityId::from_uuid(Uuid::from_u128(0xaaaa));
204+
let raw = format!("buzz:{community_id}:cache-invalidate");
205+
206+
assert_eq!(parse_cache_invalidation_channel(&raw), Some(community_id));
207+
}
208+
209+
#[test]
210+
fn rejects_bad_cache_invalidation_channels() {
211+
for raw in [
212+
"buzz:cache-invalidate",
213+
"buzz:not-a-uuid:cache-invalidate",
214+
"not-buzz:00000000-0000-0000-0000-00000000aaaa:cache-invalidate",
215+
"buzz:00000000-0000-0000-0000-00000000aaaa:cache-invalidate:extra",
216+
"buzz:00000000-0000-0000-0000-00000000aaaa:channel:00000000-0000-0000-0000-00000000bbbb",
217+
] {
218+
assert_eq!(parse_cache_invalidation_channel(raw), None);
219+
}
220+
}
221+
131222
#[test]
132223
fn membership_roundtrips_through_json() {
133224
let msg = CacheInvalidation::Membership {

0 commit comments

Comments
 (0)