Skip to content

Commit 9a403d3

Browse files
fix: channel member search via NIP-50 + Typesense indexer fix for kind:0 (#569)
Signed-off-by: Tyler Longwell <[email protected]>
1 parent 8716aef commit 9a403d3

9 files changed

Lines changed: 661 additions & 36 deletions

File tree

‎crates/sprout-relay/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ description = "WebSocket relay server for the Sprout communications platform"
1111
name = "sprout-relay"
1212
path = "src/main.rs"
1313

14+
[[bin]]
15+
name = "sprout-reindex-kind0"
16+
path = "src/bin/reindex_kind0.rs"
17+
1418
[dependencies]
1519
sprout-core = { workspace = true }
1620
sprout-db = { workspace = true }
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
//! One-shot admin tool: re-index all kind:0 (user metadata) events in Typesense.
2+
//!
3+
//! Necessary after the indexer change that appends `display_name`/`name`/`nip05`
4+
//! values to the indexed content for kind:0 docs (see `sprout-search`'s
5+
//! `flatten_kind0_for_indexing`). Existing docs need to be rewritten with the
6+
//! appended tokens before they become searchable by display name.
7+
//!
8+
//! New / updated kind:0 events index correctly automatically — this tool only
9+
//! exists to backfill the existing population.
10+
//!
11+
//! Usage (from the repo root, with .env sourced):
12+
//!
13+
//! ```
14+
//! cargo run --release -p sprout-relay --bin sprout-reindex-kind0
15+
//! ```
16+
//!
17+
//! Idempotent — Typesense uses upsert semantics, so running twice is safe.
18+
//! Streams in batches so memory stays bounded regardless of relay size.
19+
//!
20+
//! ## Paging
21+
//!
22+
//! Walks `query_events` with a snapshot ceiling (`until = now()` at start) plus
23+
//! a keyset cursor over `(created_at, id)` matching the underlying
24+
//! `ORDER BY created_at DESC, id ASC` index. This guarantees:
25+
//!
26+
//! - No rows are skipped if new kind:0 events arrive during the run
27+
//! (they're newer than the snapshot, so they fall outside the predicate).
28+
//! - No rows are double-counted at page boundaries (the cursor advances
29+
//! strictly past the last row of each batch).
30+
//! - Bounded total work — won't chase its own tail under live write traffic.
31+
//!
32+
//! Newly-arrived kind:0 events that fall outside the snapshot are indexed by
33+
//! the relay's live write path anyway, so this backfill plus the live path
34+
//! together cover the full population.
35+
36+
use anyhow::Context;
37+
use chrono::{DateTime, Utc};
38+
use tracing::{info, warn};
39+
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
40+
41+
use sprout_db::{Db, DbConfig, EventQuery};
42+
use sprout_relay::config::Config;
43+
use sprout_search::{SearchConfig, SearchService};
44+
45+
/// Page size for the SQL → Typesense pipeline. Small enough to keep DB and
46+
/// Typesense memory comfortable, large enough to amortise per-batch overhead.
47+
const BATCH: i64 = 500;
48+
49+
#[tokio::main]
50+
async fn main() -> anyhow::Result<()> {
51+
tracing_subscriber::registry()
52+
.with(fmt::layer())
53+
.with(
54+
EnvFilter::from_default_env()
55+
.add_directive("sprout_reindex_kind0=info".parse()?)
56+
.add_directive("sprout_relay=info".parse()?),
57+
)
58+
.init();
59+
60+
let config = Config::from_env().context("loading relay config from environment")?;
61+
62+
let db_config = DbConfig {
63+
database_url: config.database_url.clone(),
64+
..DbConfig::default()
65+
};
66+
let db = Db::new(&db_config)
67+
.await
68+
.context("connecting to postgres")?;
69+
70+
// SearchConfig::default() reads TYPESENSE_URL / TYPESENSE_API_KEY /
71+
// TYPESENSE_COLLECTION from the environment, same as the relay does.
72+
let search = SearchService::new(SearchConfig::default());
73+
search
74+
.ensure_collection()
75+
.await
76+
.context("ensuring Typesense collection")?;
77+
78+
// Snapshot ceiling: we only reindex events that already exist at start.
79+
// Anything newer is handled by the relay's live indexing path.
80+
let snapshot: DateTime<Utc> = Utc::now();
81+
82+
// Keyset cursor over (created_at, id) — matches the underlying
83+
// `ORDER BY created_at DESC, id ASC` index. On the first iteration both
84+
// cursor fields are None and the predicate reduces to `created_at <= snapshot`.
85+
// Subsequent iterations advance to strictly past the last row of the prior batch.
86+
let mut cursor_until: DateTime<Utc> = snapshot;
87+
let mut cursor_before_id: Option<Vec<u8>> = None;
88+
89+
let mut total_indexed: usize = 0;
90+
let mut total_failed: usize = 0;
91+
let mut batches: usize = 0;
92+
93+
info!(?snapshot, "starting kind:0 reindex");
94+
95+
loop {
96+
let q = EventQuery {
97+
kinds: Some(vec![0]),
98+
limit: Some(BATCH),
99+
max_limit: Some(BATCH),
100+
until: Some(cursor_until),
101+
before_id: cursor_before_id.clone(),
102+
..EventQuery::default()
103+
};
104+
105+
let batch = db
106+
.query_events(&q)
107+
.await
108+
.context("querying kind:0 events")?;
109+
110+
if batch.is_empty() {
111+
break;
112+
}
113+
114+
let batch_len = batch.len();
115+
116+
// Capture the tail of the batch for cursor advance *before* the index
117+
// call, so we still advance even if indexing fails for this batch.
118+
// (We'd otherwise loop forever on a poisoned batch.)
119+
let tail = batch
120+
.last()
121+
.map(|ev| {
122+
let ts = ev.event.created_at.as_u64() as i64;
123+
let dt = DateTime::<Utc>::from_timestamp(ts, 0).unwrap_or(cursor_until);
124+
let id_bytes = ev.event.id.to_bytes().to_vec();
125+
(dt, id_bytes)
126+
})
127+
.expect("batch is non-empty (checked above)");
128+
129+
match search.index_batch(&batch).await {
130+
Ok(indexed) => {
131+
total_indexed += indexed;
132+
if indexed < batch_len {
133+
let failed = batch_len - indexed;
134+
total_failed += failed;
135+
warn!(failed, batch_len, "some events failed to index in batch");
136+
}
137+
info!(indexed, batch_len, batches, total_indexed, "indexed batch");
138+
}
139+
Err(e) => {
140+
warn!(error = %e, batch_len, batches, "batch index failed entirely");
141+
total_failed += batch_len;
142+
}
143+
}
144+
145+
batches += 1;
146+
147+
// Tail of the prior batch becomes the cursor for the next page.
148+
// `query_events` will use the composite predicate
149+
// created_at < cursor_until OR (created_at = cursor_until AND id > cursor_before_id)
150+
// which exactly skips past the last row we just processed.
151+
cursor_until = tail.0;
152+
cursor_before_id = Some(tail.1);
153+
154+
// If we got fewer than BATCH back, we're at the tail of the table.
155+
if (batch_len as i64) < BATCH {
156+
break;
157+
}
158+
}
159+
160+
info!(
161+
total_indexed,
162+
total_failed, batches, "kind:0 reindex complete"
163+
);
164+
if total_failed > 0 {
165+
std::process::exit(1);
166+
}
167+
Ok(())
168+
}

‎crates/sprout-search/src/index.rs‎

Lines changed: 188 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,36 @@ pub fn event_to_document(event: &StoredEvent) -> Result<Value, SearchError> {
4343
.map(|id| id.to_string())
4444
.unwrap_or_else(|| "__global__".to_string());
4545

46+
// For kind:0 (user metadata) we append the parsed JSON values to `content`
47+
// so Typesense's word-tokenizer can index them cleanly. Without this, a
48+
// raw blob like `{"display_name":"alice","about":"loves cats"}` does not
49+
// produce a clean `alice` token — the leading `"` glues onto the next
50+
// word, so the doc is unreachable for the obvious `q=alice` search.
51+
//
52+
// We only flatten kind:0 (the structured-metadata kind defined by NIP-01)
53+
// and only the small set of fields the member-picker uses. Bio / about /
54+
// website are intentionally left out so they don't pollute name-prefix
55+
// searches with false positives. Stays consistent with `display_name >
56+
// nip05 > pubkey` ranking applied on the desktop side.
57+
//
58+
// The Typesense `content` field is write-only as far as the relay's read
59+
// paths go (the bridge fetches the canonical event from Postgres by id
60+
// after Typesense returns hits), so appending derived tokens here doesn't
61+
// affect any consumer's view of the event's actual content.
62+
//
63+
// NOTE: existing kind:0 docs indexed before this change won't have the
64+
// appended tokens. Running `just reindex-search` (or the
65+
// `sprout-relay reindex-search` admin path) repopulates them. New /
66+
// updated profiles get the tokens automatically.
67+
let content_indexed = if event_kind_i32(nostr_event) == 0 {
68+
flatten_kind0_for_indexing(nostr_event.content.as_str())
69+
} else {
70+
nostr_event.content.as_str().to_string()
71+
};
72+
4673
let doc = json!({
4774
"id": nostr_event.id.to_string(),
48-
"content": nostr_event.content.as_str(),
75+
"content": content_indexed,
4976
// Cast to i32 for Typesense schema (int32 field). nostr Kind is u16; all Sprout kinds fit in i32.
5077
"kind": event_kind_i32(nostr_event),
5178
"pubkey": nostr_event.pubkey.to_string(),
@@ -57,6 +84,38 @@ pub fn event_to_document(event: &StoredEvent) -> Result<Value, SearchError> {
5784
Ok(doc)
5885
}
5986

87+
/// For kind:0 events, return the original content with the searchable fields
88+
/// (`display_name`, `name`, `nip05`) appended as space-separated plain words.
89+
///
90+
/// Tolerant of malformed input: anything that fails JSON parsing returns the
91+
/// original content unchanged, never an error.
92+
fn flatten_kind0_for_indexing(raw_content: &str) -> String {
93+
let Ok(parsed) = serde_json::from_str::<Value>(raw_content) else {
94+
return raw_content.to_string();
95+
};
96+
let Some(obj) = parsed.as_object() else {
97+
return raw_content.to_string();
98+
};
99+
100+
let mut extracted: Vec<&str> = Vec::with_capacity(3);
101+
for key in ["display_name", "name", "nip05"] {
102+
if let Some(val) = obj.get(key).and_then(Value::as_str) {
103+
let trimmed = val.trim();
104+
if !trimmed.is_empty() {
105+
extracted.push(trimmed);
106+
}
107+
}
108+
}
109+
110+
if extracted.is_empty() {
111+
raw_content.to_string()
112+
} else {
113+
// Single leading space ensures we don't smash the closing `}` of the
114+
// original JSON into the first appended token.
115+
format!("{} {}", raw_content, extracted.join(" "))
116+
}
117+
}
118+
60119
/// Indexes a single event via Typesense upsert.
61120
pub async fn index_event(
62121
client: &reqwest::Client,
@@ -269,6 +328,134 @@ mod tests {
269328
assert_eq!(doc["channel_id"].as_str().unwrap(), "__global__");
270329
}
271330

331+
// ── kind:0 flattening for searchability ─────────────────────────────────
332+
333+
#[test]
334+
fn kind0_appends_display_name_for_tokenization() {
335+
let stored = make_stored_event(
336+
r#"{"display_name":"alice","about":"loves cats"}"#,
337+
Kind::Metadata,
338+
None,
339+
);
340+
let doc = event_to_document(&stored).unwrap();
341+
let content = doc["content"].as_str().unwrap();
342+
// Original JSON is preserved (read paths don't depend on this but it
343+
// costs nothing and makes debugging the index cheaper).
344+
assert!(content.contains(r#""display_name":"alice""#));
345+
// The display name is also present as a free-standing token so the
346+
// default Typesense tokenizer can index it without the leading-quote
347+
// gluing onto the next character.
348+
assert!(content.ends_with(" alice"), "got: {content:?}");
349+
}
350+
351+
#[test]
352+
fn kind0_appends_name_when_display_name_absent() {
353+
// NIP-01 allows `name` as the canonical display field too.
354+
let stored = make_stored_event(r#"{"name":"bob","about":"x"}"#, Kind::Metadata, None);
355+
let doc = event_to_document(&stored).unwrap();
356+
let content = doc["content"].as_str().unwrap();
357+
assert!(content.ends_with(" bob"), "got: {content:?}");
358+
}
359+
360+
#[test]
361+
fn kind0_includes_both_display_name_and_name_when_present() {
362+
let stored = make_stored_event(
363+
r#"{"display_name":"Alice","name":"alice"}"#,
364+
Kind::Metadata,
365+
None,
366+
);
367+
let doc = event_to_document(&stored).unwrap();
368+
let content = doc["content"].as_str().unwrap();
369+
assert!(content.ends_with(" Alice alice"), "got: {content:?}");
370+
}
371+
372+
#[test]
373+
fn kind0_includes_nip05_in_appended_tokens() {
374+
let stored = make_stored_event(
375+
r#"{"display_name":"alice","nip05":"[email protected]"}"#,
376+
Kind::Metadata,
377+
None,
378+
);
379+
let doc = event_to_document(&stored).unwrap();
380+
let content = doc["content"].as_str().unwrap();
381+
assert!(
382+
content.ends_with(" alice [email protected]"),
383+
"got: {content:?}"
384+
);
385+
}
386+
387+
#[test]
388+
fn kind0_excludes_about_and_website_from_appended_tokens() {
389+
// `about` and `website` deliberately do not get appended — including
390+
// them would cause name-prefix searches to return false positives from
391+
// bios. The user's own display_name still appears.
392+
let stored = make_stored_event(
393+
r#"{"display_name":"alice","about":"I work with bob on x","website":"https://carol.example"}"#,
394+
Kind::Metadata,
395+
None,
396+
);
397+
let doc = event_to_document(&stored).unwrap();
398+
let content = doc["content"].as_str().unwrap();
399+
assert!(content.ends_with(" alice"), "got: {content:?}");
400+
// Sanity: the about/website are still in the doc because we preserve
401+
// the original JSON — they just don't appear in the trailing tokens.
402+
assert!(content.contains("bob"));
403+
assert!(content.contains("carol"));
404+
}
405+
406+
#[test]
407+
fn kind0_malformed_json_is_passed_through_unchanged() {
408+
let stored = make_stored_event("not json at all", Kind::Metadata, None);
409+
let doc = event_to_document(&stored).unwrap();
410+
assert_eq!(doc["content"].as_str().unwrap(), "not json at all");
411+
}
412+
413+
#[test]
414+
fn kind0_non_object_json_is_passed_through_unchanged() {
415+
// Defensive: NIP-01 says content is a JSON object, but a malformed
416+
// client could publish e.g. a JSON array. We don't crash, we just
417+
// skip the flattening for that doc.
418+
let stored = make_stored_event(r#"["nope"]"#, Kind::Metadata, None);
419+
let doc = event_to_document(&stored).unwrap();
420+
assert_eq!(doc["content"].as_str().unwrap(), r#"["nope"]"#);
421+
}
422+
423+
#[test]
424+
fn kind0_empty_string_values_skipped() {
425+
let stored = make_stored_event(
426+
r#"{"display_name":"","name":"alice","nip05":" "}"#,
427+
Kind::Metadata,
428+
None,
429+
);
430+
let doc = event_to_document(&stored).unwrap();
431+
let content = doc["content"].as_str().unwrap();
432+
// Only `name` is non-empty; whitespace-only `nip05` is also skipped.
433+
assert!(content.ends_with(" alice"), "got: {content:?}");
434+
}
435+
436+
#[test]
437+
fn kind0_no_searchable_fields_is_passed_through() {
438+
// Profile with only fields we don't extract.
439+
let stored = make_stored_event(
440+
r#"{"about":"just a bio","picture":"https://x"}"#,
441+
Kind::Metadata,
442+
None,
443+
);
444+
let doc = event_to_document(&stored).unwrap();
445+
let content = doc["content"].as_str().unwrap();
446+
// No trailing space-separated tokens added; original content unchanged.
447+
assert_eq!(content, r#"{"about":"just a bio","picture":"https://x"}"#);
448+
}
449+
450+
#[test]
451+
fn non_kind0_events_not_flattened() {
452+
// kind:1 (note) with a JSON-looking body must be left strictly alone.
453+
let json_looking = r#"{"display_name":"alice"}"#;
454+
let stored = make_stored_event(json_looking, Kind::TextNote, None);
455+
let doc = event_to_document(&stored).unwrap();
456+
assert_eq!(doc["content"].as_str().unwrap(), json_looking);
457+
}
458+
272459
#[test]
273460
fn tag_flattening_uses_unit_separator() {
274461
let keys = Keys::generate();

0 commit comments

Comments
 (0)