Skip to content

Commit e5621f9

Browse files
TheSentinel454Elrondclaude
committed
fix(relay): close a REQ whose history read failed instead of sending EOSE
A REQ whose history query failed with anything other than a statement timeout ended with EOSE. EOSE tells the client it has every stored match, so a client resuming from its live point skips the history that failed. Send CLOSED "error: database error" instead, retiring the subscription the same way the timeout path does, and reuse that one reason string for the handler's existing pre-registration DB failures and COUNT. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 7c165c3 commit e5621f9

4 files changed

Lines changed: 130 additions & 21 deletions

File tree

‎ARCHITECTURE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ This prevents a race where a non-member receives live fan-out events from a priv
378378

379379
### Historical Query (EOSE)
380380

381-
After registering, the REQ handler queries Postgres for stored events matching the filters (up to 500 per filter, hard cap). These are sent as `["EVENT", sub_id, event]` frames before `["EOSE", sub_id]`. New events arriving after EOSE are delivered via the fan-out path.
381+
After registering, the REQ handler queries Postgres for stored events matching the filters (up to 500 per filter, hard cap). These are sent as `["EVENT", sub_id, event]` frames before `["EOSE", sub_id]`. New events arriving after EOSE are delivered via the fan-out path. EOSE means every stored match was sent, so a failed history read never ends in EOSE: it retires the subscription with `["CLOSED", sub_id, "error: query timed out"]` for a statement timeout or `"error: database error"` for anything else.
382382

383383
**Client consumption invariant.** A client rebuilding channel state must
384384
open its live subscription before (or overlapping) the finite history

‎crates/buzz-relay/src/api/bridge.rs‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9448,6 +9448,100 @@ pub(crate) mod postgres_tests {
94489448
fx.drop().await;
94499449
}
94509450

9451+
/// A history read that fails with an ordinary DB error ends the REQ with a
9452+
/// retryable CLOSED, never EOSE: EOSE would tell a resuming client it holds
9453+
/// every stored event, so its next REQ would skip the history that failed.
9454+
/// The CLOSED also retires the subscription, and once the DB recovers a
9455+
/// reconnect with the same REQ replays the missed event.
9456+
/// Mutation: send EOSE on the non-timeout error again → RED.
9457+
#[tokio::test]
9458+
#[ignore = "requires Postgres"]
9459+
async fn failed_history_read_closes_and_reconnect_replays_the_history() {
9460+
use crate::handlers::req::handle_req;
9461+
let fx = cancel_fixture().await;
9462+
let frames = |rx: &mut tokio::sync::mpsc::Receiver<axum::extract::ws::Message>| {
9463+
std::iter::from_fn(|| rx.try_recv().ok())
9464+
.map(|msg| match msg {
9465+
axum::extract::ws::Message::Text(t) => t.to_string(),
9466+
other => panic!("expected text frame, got {other:?}"),
9467+
})
9468+
.collect::<Vec<_>>()
9469+
};
9470+
let req = || -> nostr::Filter {
9471+
serde_json::from_value(serde_json::json!({"kinds": [9], "#h": [&fx.channel]}))
9472+
.expect("history filter")
9473+
};
9474+
let channel: uuid::Uuid = fx.channel.parse().expect("channel uuid");
9475+
let topic = buzz_pubsub::EventTopic::Channel(channel);
9476+
let tenant = TenantContext::resolved(fx.community, fx.host.clone());
9477+
9478+
// `events` gone on both pools: the history read fails with 42P01 on the
9479+
// replica and again on the writer re-run. Not a statement timeout.
9480+
let rename = |from: &'static str, to: &'static str| {
9481+
let pools = [fx.writer.clone(), fx.replica.clone()];
9482+
async move {
9483+
for pool in pools {
9484+
sqlx::query(sqlx::AssertSqlSafe(format!(
9485+
"ALTER TABLE {from} RENAME TO {to}"
9486+
)))
9487+
.execute(&pool)
9488+
.await
9489+
.expect("rename events");
9490+
}
9491+
}
9492+
};
9493+
rename("events", "events_gone").await;
9494+
let (conn, mut rx) = fx.ws_conn();
9495+
handle_req(
9496+
"h".into(),
9497+
vec![req()],
9498+
vec![None],
9499+
conn.clone(),
9500+
fx.state.clone(),
9501+
)
9502+
.await;
9503+
assert_eq!(
9504+
frames(&mut rx),
9505+
vec![format!(
9506+
r#"["CLOSED","h","{}"]"#,
9507+
crate::handlers::req::DATABASE_ERROR_CLOSED
9508+
)],
9509+
"a failed history read must CLOSE, not EOSE"
9510+
);
9511+
assert!(conn.subscriptions.lock().await.is_empty(), "conn map");
9512+
assert!(
9513+
fx.state
9514+
.sub_registry
9515+
.get_filters(conn.conn_id, "h")
9516+
.is_none(),
9517+
"CLOSED must retire the fan-out registration"
9518+
);
9519+
assert_eq!(fx.state.pubsub.topic_refcount(&tenant, topic).await, 0);
9520+
drop(conn);
9521+
9522+
// The DB recovers and the client reconnects with the same REQ.
9523+
rename("events_gone", "events").await;
9524+
let (conn, mut rx) = fx.ws_conn();
9525+
handle_req(
9526+
"h".into(),
9527+
vec![req()],
9528+
vec![None],
9529+
conn.clone(),
9530+
fx.state.clone(),
9531+
)
9532+
.await;
9533+
let replay = frames(&mut rx);
9534+
assert_eq!(replay.len(), 2, "event then EOSE: {replay:?}");
9535+
assert!(
9536+
replay[0].starts_with(r#"["EVENT","h","#) && replay[0].contains(&fx.root),
9537+
"the reconnect must replay the missed event: {replay:?}"
9538+
);
9539+
assert_eq!(replay[1], r#"["EOSE","h"]"#);
9540+
9541+
drop(conn);
9542+
fx.drop().await;
9543+
}
9544+
94519545
/// HTTP agents reach `materialize_nip_oa_owner` on every request. Once the
94529546
/// owner link is confirmed in `observer_owner_cache` the call writes
94539547
/// nothing: the link is set once and never cleared.

‎crates/buzz-relay/src/handlers/count.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use crate::connection::{AuthState, ConnectionState};
1010
use crate::handlers::req::{
1111
event_visible_to_reader, filter_can_match_result_gated_kinds,
1212
filter_can_match_shared_gated_kinds, result_gated_count_safe_for_pushdown,
13+
DATABASE_ERROR_CLOSED,
1314
};
1415
use crate::protocol::RelayMessage;
1516
use crate::state::AppState;
@@ -88,7 +89,7 @@ pub async fn handle_count(
8889
Ok(ids) => ids,
8990
Err(e) => {
9091
warn!(sub_id = %sub_id, "Failed to get accessible channels: {e}");
91-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
92+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
9293
return;
9394
}
9495
};
@@ -116,7 +117,7 @@ pub async fn handle_count(
116117
Ok(ids) => ids,
117118
Err(e) => {
118119
warn!(sub_id = %sub_id, "Failed to get readable named-only channels: {e}");
119-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
120+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
120121
return;
121122
}
122123
};
@@ -178,7 +179,7 @@ pub async fn handle_count(
178179
.await
179180
{
180181
warn!(sub_id = %sub_id, "Channel membership confirmation failed: {e}");
181-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
182+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
182183
return;
183184
}
184185
if let Err(e) = super::system_channels::keep_repaired_named_only_channels_named(
@@ -191,7 +192,7 @@ pub async fn handle_count(
191192
.await
192193
{
193194
warn!(sub_id = %sub_id, "Failed to classify repaired channels: {e}");
194-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
195+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
195196
return;
196197
}
197198
let filter_scope = super::system_channels::filter_channel_scope(

‎crates/buzz-relay/src/handlers/req.rs‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ pub async fn handle_req(
122122
Ok(ids) => ids,
123123
Err(e) => {
124124
warn!(conn_id = %conn_id, "Failed to get accessible channels: {e}");
125-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
125+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
126126
return;
127127
}
128128
}
@@ -146,7 +146,7 @@ pub async fn handle_req(
146146
Ok(ids) => ids,
147147
Err(e) => {
148148
warn!(conn_id = %conn_id, "Failed to get readable named-only channels: {e}");
149-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
149+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
150150
return;
151151
}
152152
};
@@ -185,7 +185,7 @@ pub async fn handle_req(
185185
Ok(confirmed) => confirmed,
186186
Err(e) => {
187187
warn!(conn_id = %conn_id, "Channel membership confirmation failed: {e}");
188-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
188+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
189189
return;
190190
}
191191
};
@@ -204,7 +204,7 @@ pub async fn handle_req(
204204
.await
205205
{
206206
warn!(conn_id = %conn_id, "Failed to classify repaired channels: {e}");
207-
conn.send(RelayMessage::closed(&sub_id, "error: database error"));
207+
conn.send(RelayMessage::closed(&sub_id, DATABASE_ERROR_CLOSED));
208208
return;
209209
}
210210
}
@@ -401,8 +401,9 @@ pub async fn handle_req(
401401
// Registration above is one uncancellable unit; the history read below is
402402
// read-only delivery and races gate cancellation, so expiry drops it and
403403
// releases the permit without sending another EVENT or EOSE. It returns
404-
// whether a statement timeout ended it: retirement then runs after the
405-
// race, because a dropped retirement would leak its unreleased topics.
404+
// the CLOSED reason when a failed read ended it: retirement then runs
405+
// after the race, because a dropped retirement would leak its unreleased
406+
// topics.
406407
let history = async {
407408
#[cfg(test)]
408409
crate::nip_fi_test_hooks::before_req_history(conn.tenant.community()).await;
@@ -489,11 +490,9 @@ pub async fn handle_req(
489490
Ok(evs) => evs,
490491
Err(e) => {
491492
warn!(conn_id = %conn_id, sub_id = %sub_id, "Historical query failed: {e}");
492-
if e.is_statement_cancelled() {
493-
return true;
494-
}
495-
conn.send(RelayMessage::eose(&sub_id));
496-
return false;
493+
// CLOSED, never EOSE: EOSE tells the client it has every
494+
// stored event, so a resuming client would skip this history.
495+
return Some(history_read_closed_reason(&e));
497496
}
498497
};
499498

@@ -570,7 +569,7 @@ pub async fn handle_req(
570569

571570
let msg = RelayMessage::event(&sub_id, &stored.event);
572571
if !conn.send(msg) {
573-
return false;
572+
return None;
574573
}
575574
total_sent += 1;
576575
if total_sent.is_multiple_of(100) {
@@ -587,13 +586,15 @@ pub async fn handle_req(
587586
count = total_sent,
588587
"EOSE sent after historical delivery"
589588
);
590-
false
589+
None
591590
};
592591
let outcome = unless_cancelled(&conn, history).await;
593592
drop(_req_permit);
594593
match outcome {
595-
Some(false) => {}
596-
Some(true) => close_timed_out_subscription(&sub_id, owner, &conn, &state).await,
594+
Some(None) => {}
595+
Some(Some(reason)) => {
596+
super::close::close_if_owner(&sub_id, owner, Some(reason), &conn, &state).await;
597+
}
597598
None => {
598599
super::close::close_if_owner(&sub_id, owner, None, &conn, &state).await;
599600
}
@@ -1439,7 +1440,7 @@ async fn handle_huddle_liveness_req(
14391440
Ok(links) => links,
14401441
Err(error) => {
14411442
warn!("Huddle liveness linkage batch failed: {error}");
1442-
conn.send(RelayMessage::closed(sub_id, "error: database error"));
1443+
conn.send(RelayMessage::closed(sub_id, DATABASE_ERROR_CLOSED));
14431444
return;
14441445
}
14451446
};
@@ -1600,6 +1601,19 @@ async fn close_timed_out_subscription(
16001601
/// Clients match it to skip retrying the same expensive query.
16011602
pub(crate) const QUERY_TIMED_OUT_CLOSED: &str = "error: query timed out";
16021603

1604+
/// CLOSED reason for any other failed DB read. Generic on purpose: the raw
1605+
/// error stays in the relay log, and clients treat it as retryable.
1606+
pub(crate) const DATABASE_ERROR_CLOSED: &str = "error: database error";
1607+
1608+
/// CLOSED reason for a failed REQ history read.
1609+
fn history_read_closed_reason(e: &buzz_db::DbError) -> &'static str {
1610+
if e.is_statement_cancelled() {
1611+
QUERY_TIMED_OUT_CLOSED
1612+
} else {
1613+
DATABASE_ERROR_CLOSED
1614+
}
1615+
}
1616+
16031617
/// CLOSED reason for a failed one-shot DB read (COUNT): a statement cancel gets
16041618
/// the stable timeout reason; anything else keeps the raw error.
16051619
pub(crate) fn db_read_closed_reason(e: &buzz_db::DbError) -> String {

0 commit comments

Comments
 (0)