Skip to content

Commit efcdf00

Browse files
relay: open the search pool lazily so a dead reader can't block boot (#8662)
1 parent 64312fa commit efcdf00

6 files changed

Lines changed: 187 additions & 44 deletions

File tree

‎Justfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,7 @@ test-unit:
546546
# never connects.
547547
# `^config::` is anchored so it does not also select nip_fi_config.
548548
cargo nextest run -p buzz-relay --lib --bin buzz-relay \
549-
-E 'test(/^api::admin::/) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::coalesced_refresh::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::profiles::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^mesh_boot::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::accessible_channel_invalidation_is_scoped_to_community) + test(=state::tests::accessible_channels_join_member_and_open_halves) + test(=state::tests::channel_deleted_invalidation_is_scoped_to_community) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + test(=api::invites::postgres_tests::restriction_tests::postgres_tests::destructive_fixture_rejects_shared_database_urls) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))'
549+
-E 'test(/^api::admin::/) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::coalesced_refresh::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::profiles::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^mesh_boot::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::accessible_channel_invalidation_is_scoped_to_community) + test(=state::tests::accessible_channels_join_member_and_open_halves) + test(=state::tests::channel_deleted_invalidation_is_scoped_to_community) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::failed_search_closes_without_eose_or_trying_later_filters) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + test(=api::invites::postgres_tests::restriction_tests::postgres_tests::destructive_fixture_rejects_shared_database_urls) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))'
550550
# Note on audio::join::tests scope: the full suite is infra-free (no DB,
551551
# no Redis). The infra-free audio/FI regression witnesses — bootstrap
552552
# ordering barrier, CommitConfirmed arm, pending-close invisibility,

‎crates/buzz-relay/src/handlers/req.rs‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -876,8 +876,19 @@ async fn handle_search_req(
876876
let search_result = match state.search.search(&search_query).await {
877877
Ok(r) => r,
878878
Err(e) => {
879+
// CLOSED, not EOSE: an empty EOSE would read as "no
880+
// matches". Stop the whole REQ so later filters do not
881+
// each wait out the search pool's acquire timeout too.
879882
warn!(sub_id = %sub_id, "NIP-50 search failed: {e}");
880-
break;
883+
super::close::close_if_owner(
884+
sub_id,
885+
owner,
886+
Some(SEARCH_FAILED_CLOSED),
887+
conn,
888+
state,
889+
)
890+
.await;
891+
return;
881892
}
882893
};
883894

@@ -1614,6 +1625,10 @@ fn history_read_closed_reason(e: &buzz_db::DbError) -> &'static str {
16141625
}
16151626
}
16161627

1628+
/// CLOSED reason for a NIP-50 search the search service could not answer
1629+
/// (e.g. the read replica it queries is down).
1630+
pub(crate) const SEARCH_FAILED_CLOSED: &str = "error: search failed";
1631+
16171632
/// CLOSED reason for a failed one-shot DB read (COUNT): a statement cancel gets
16181633
/// the stable timeout reason; anything else keeps the raw error.
16191634
pub(crate) fn db_read_closed_reason(e: &buzz_db::DbError) -> String {
@@ -2051,6 +2066,63 @@ mod tests {
20512066
assert!(closed_frames(&mut send_rx).is_empty());
20522067
}
20532068

2069+
/// A NIP-50 search the search pool cannot serve (its reader is down) ends
2070+
/// the REQ with a CLOSED error, never an EOSE that reads as "no matches",
2071+
/// and the REQ's later search filters never wait on the pool.
2072+
#[tokio::test]
2073+
async fn failed_search_closes_without_eose_or_trying_later_filters() {
2074+
const ACQUIRE: std::time::Duration = std::time::Duration::from_millis(500);
2075+
let state = crate::state::tests::test_state_with_database_url_and_pool_options(
2076+
"postgres://127.0.0.1:1/none",
2077+
sqlx::postgres::PgPoolOptions::new().acquire_timeout(ACQUIRE),
2078+
)
2079+
.await;
2080+
let (conn, mut send_rx) = lifecycle_conn();
2081+
let owner = claim_search_subscription("s", &conn, &state)
2082+
.await
2083+
.expect("claim");
2084+
let filters: Vec<Filter> = (0..4)
2085+
.map(|i| {
2086+
Filter::new()
2087+
.kind(nostr::Kind::TextNote)
2088+
.search(format!("needle{i}"))
2089+
})
2090+
.collect();
2091+
2092+
let started = std::time::Instant::now();
2093+
handle_search_req(
2094+
"s",
2095+
owner,
2096+
&filters,
2097+
&[],
2098+
&super::super::system_channels::NamedOnlyChannels::default(),
2099+
true,
2100+
&conn.tenant,
2101+
&[0; 32],
2102+
&conn,
2103+
&state,
2104+
None,
2105+
)
2106+
.await;
2107+
let elapsed = started.elapsed();
2108+
2109+
assert_eq!(
2110+
closed_frames(&mut send_rx),
2111+
vec![format!(r#"["CLOSED","s","{SEARCH_FAILED_CLOSED}"]"#)],
2112+
"exactly one CLOSED error and no EOSE"
2113+
);
2114+
assert!(
2115+
conn.subscriptions.lock().await.is_empty(),
2116+
"the failed search must release its ID"
2117+
);
2118+
// Each filter that reaches the pool waits out ACQUIRE on the dead port.
2119+
assert!(
2120+
elapsed < ACQUIRE * 3,
2121+
"later filters must not try the pool: took {elapsed:?} for {} filters",
2122+
filters.len()
2123+
);
2124+
}
2125+
20542126
/// Concurrent same-ID claims and stale teardowns: whichever claim lands
20552127
/// last owns the ID, and every stale teardown is a silent no-op.
20562128
#[tokio::test]

‎crates/buzz-relay/src/main.rs‎

Lines changed: 37 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,22 @@ async fn connect_audit_pool(config: &DbConfig) -> anyhow::Result<sqlx::PgPool> {
5151
.map_err(Into::into)
5252
}
5353

54+
/// The search pool. Search is lag-tolerant, so it prefers the read replica
55+
/// when one is configured. Like the main read pool it is lazy, so a reader
56+
/// that is down at boot fails searches, not startup, and it waits no longer
57+
/// than the writer pool for a connection, so a dead reader fails each search
58+
/// within that bound instead of sqlx's 30s default.
59+
fn connect_search_pool(config: &DbConfig) -> anyhow::Result<sqlx::PgPool> {
60+
let url = config
61+
.read_database_url
62+
.as_deref()
63+
.unwrap_or(&config.database_url);
64+
sqlx::postgres::PgPoolOptions::new()
65+
.acquire_timeout(std::time::Duration::from_secs(config.acquire_timeout_secs))
66+
.connect_lazy(url)
67+
.map_err(|e| anyhow::anyhow!("Search DB URL is invalid: {e}"))
68+
}
69+
5470
fn relay_keypair_from_config(relay_private_key: Option<&str>) -> anyhow::Result<nostr::Keys> {
5571
let hex = relay_private_key.ok_or_else(|| {
5672
anyhow::anyhow!(
@@ -834,18 +850,8 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
834850
// Postgres FTS: the searchable row IS the persisted event row (its
835851
// `tsvector` column is populated by the `insert_event` write), so there is
836852
// no external collection to provision — the search service just queries the
837-
// same Postgres over its own pool. Search is lag-tolerant, so it prefers
838-
// the read replica when one is configured.
839-
let search_db_url = config
840-
.read_database_url
841-
.as_deref()
842-
.unwrap_or(&config.database_url);
843-
let step = StepTimer::start(StartupStep::SearchConnect);
844-
let search_pool = sqlx::postgres::PgPoolOptions::new()
845-
.connect(search_db_url)
846-
.await
847-
.map_err(|e| anyhow::anyhow!("Search DB connection failed: {e}"))?;
848-
step.finish();
853+
// same Postgres over its own pool (see `connect_search_pool`).
854+
let search_pool = connect_search_pool(&db_config)?;
849855
let search_metrics_pool = search_pool.clone();
850856
let search = SearchService::new(search_pool);
851857
info!(
@@ -3222,9 +3228,9 @@ mod tests {
32223228

32233229
use super::{
32243230
apply_activity_refresh, apply_stock_refresh, buzz_auto_migrate_enabled, connect_audit_pool,
3225-
counted_partition_audit, dropped_in_memory_keys, emit_cached_fleet_usage_metrics,
3226-
idle_timeout_secs, jwks_next_retry_after_failed_refresh, nip_fi_jwks_refresh_loop,
3227-
record_legacy_usage_collection, refresh_due_fleet_usage,
3231+
connect_search_pool, counted_partition_audit, dropped_in_memory_keys,
3232+
emit_cached_fleet_usage_metrics, idle_timeout_secs, jwks_next_retry_after_failed_refresh,
3233+
nip_fi_jwks_refresh_loop, record_legacy_usage_collection, refresh_due_fleet_usage,
32283234
refresh_legacy_active_gauge_recency, relay_keypair_from_config,
32293235
run_jwks_refresh_supervisor, run_periodic_until_cancelled, startup_partition_audit,
32303236
EmissionScope, FleetUsageSchedule, InMemoryMetricKey, PartitionAuditSchedule,
@@ -3237,6 +3243,22 @@ mod tests {
32373243
registry::{GenerationalAtomicStorage, Registry},
32383244
};
32393245

3246+
#[tokio::test]
3247+
async fn search_pool_is_lazy_prefers_the_reader_and_bounds_acquire() {
3248+
// Unroutable per RFC 5737 TEST-NET-1: construction must not dial it.
3249+
let config = DbConfig {
3250+
read_database_url: Some("postgres://u:[email protected]:5432/none".to_owned()), // sadscan:disable np.postgres.1
3251+
..DbConfig::default()
3252+
};
3253+
let pool = connect_search_pool(&config).expect("lazy construction must not dial");
3254+
assert_eq!(pool.connect_options().get_host(), "192.0.2.1");
3255+
assert_eq!(
3256+
pool.options().get_acquire_timeout(),
3257+
Duration::from_secs(config.acquire_timeout_secs),
3258+
"a dead reader must fail a search within the writer's acquire bound"
3259+
);
3260+
}
3261+
32403262
#[tokio::test(start_paused = true)]
32413263
async fn periodic_loop_exits_immediately_on_cancellation() {
32423264
let cancel = CancellationToken::new();

‎crates/buzz-relay/src/startup_steps.rs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,6 @@ pub enum StartupStep {
5959
AuditConnect,
6060
/// Connect Redis commands and pub/sub.
6161
RedisConnect,
62-
/// Connect the search database pool.
63-
SearchConnect,
6462
/// Warm NIP-FI JWKS snapshots (only when NIP-FI is configured).
6563
NipFiJwksWarm,
6664
/// Load the first NIP-FI binding snapshot (only for the `database` source).
@@ -94,7 +92,6 @@ impl StartupStep {
9492
Self::MembershipBootstrap,
9593
Self::AuditConnect,
9694
Self::RedisConnect,
97-
Self::SearchConnect,
9895
Self::NipFiJwksWarm,
9996
Self::NipFiSnapshotLoad,
10097
Self::MeshBoot,
@@ -118,7 +115,6 @@ impl StartupStep {
118115
Self::MembershipBootstrap => "membership_bootstrap",
119116
Self::AuditConnect => "audit_connect",
120117
Self::RedisConnect => "redis_connect",
121-
Self::SearchConnect => "search_connect",
122118
Self::NipFiJwksWarm => "nip_fi_jwks_warm",
123119
Self::NipFiSnapshotLoad => "nip_fi_snapshot_load",
124120
Self::MeshBoot => "mesh_boot",

0 commit comments

Comments
 (0)