Skip to content

Commit 9b131a5

Browse files
TheSentinel454Elrondclaude
committed
fix(search): apply the prefix minimum to parsed lexemes
The three-character minimum counted characters of the trailing whitespace token, but push_tsquery gives `:*` to every lexeme the `simple` parser yields from that token. Punctuation padded the count: `pr.`, `@pr`, `c++`, `don't` and `pre-p` still ran one- or two-character prefix scans, and a punctuation-only trailing token (`project ?`) was blocked although it yields no `:*` lexeme. Move the check into the prefix tsquery SQL so it reads the parser's own output: if any trailing lexeme is shorter than the minimum, the tsquery is NULL and matches nothing. The Rust pre-check is gone. kind:0-only search and search_profile_names pass 0 and are unchanged. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Luke Tornquist <[email protected]>
1 parent 194ad32 commit 9b131a5

2 files changed

Lines changed: 200 additions & 36 deletions

File tree

‎crates/buzz-search/src/query.rs‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,9 @@ pub enum SearchMode {
6363
///
6464
/// Intended for bounded typeahead surfaces such as the desktop topbar. The
6565
/// relay still refetches and re-authorizes every hit; this mode changes only
66-
/// the candidate tsquery, not the access boundary. A trailing token shorter
67-
/// than three characters returns no hits unless the query is kind:0-only.
66+
/// the candidate tsquery, not the access boundary. A trailing token that
67+
/// yields a lexeme shorter than three characters returns no hits unless the
68+
/// query is kind:0-only (see `PREFIX_MIN_CHARS`).
6869
Prefix,
6970
}
7071

@@ -150,17 +151,26 @@ const SEARCH_TEXT_MAX_CHARS: usize = 4096;
150151
/// bridge uses page 1), but clamp here too so a future caller cannot accidentally
151152
/// wire untrusted input into a multi-trillion-row OFFSET.
152153
const PAGE_MAX: u32 = 1000;
153-
/// Shortest trailing token (the one that gets `:*`) [`SearchMode::Prefix`]
154-
/// runs for anything but a kind:0-only (profile) query; a shorter one returns
155-
/// no hits without SQL, whatever completed words precede it.
154+
/// Shortest lexeme [`SearchMode::Prefix`] gives `:*` for anything but a
155+
/// kind:0-only (profile) query. If the trailing token yields a shorter one,
156+
/// the tsquery is NULL and the search returns no hits, whatever completed
157+
/// words precede it. The check runs on Postgres' own `simple` parser output,
158+
/// so punctuation such as `c++`, `@al` or `don't` cannot slip a short `:*`
159+
/// past it.
156160
/// The GIN index cannot stop early or return rows in rank order, so a one- or
157161
/// two-character `:*` prefix makes Postgres fetch, recheck and rank a large
158162
/// share of every indexed message before it can apply `LIMIT`, and typeahead
159163
/// sends one on every typing pause. Profiles are few enough to stay
160164
/// one-character typeahead.
161-
const PREFIX_MIN_CHARS: usize = 3;
162-
163-
fn push_tsquery(qb: &mut QueryBuilder<sqlx::Postgres>, mode: SearchMode, search_text: &str) {
165+
const PREFIX_MIN_CHARS: i32 = 3;
166+
167+
/// `min_prefix_chars` applies to [`SearchMode::Prefix`] only; 0 disables it.
168+
fn push_tsquery(
169+
qb: &mut QueryBuilder<sqlx::Postgres>,
170+
mode: SearchMode,
171+
search_text: &str,
172+
min_prefix_chars: i32,
173+
) {
164174
match mode {
165175
SearchMode::FullText => {
166176
qb.push("websearch_to_tsquery('simple', ");
@@ -174,14 +184,20 @@ fn push_tsquery(qb: &mut QueryBuilder<sqlx::Postgres>, mode: SearchMode, search_
174184
// tsquery construction so query-side normalization matches the
175185
// `search_tsv` generated column. `quote_literal` prevents tsquery
176186
// syntax injection from punctuation/operators in the raw topbar input.
187+
// A trailing lexeme shorter than `min_prefix_chars` makes the whole
188+
// tsquery NULL, which matches nothing (see `PREFIX_MIN_CHARS`).
189+
qb.push(
190+
"(SELECT CASE WHEN bool_or(token_ord = max_token_ord AND char_length(lexeme) < ",
191+
);
192+
qb.push_bind(min_prefix_chars);
177193
qb.push(
178-
"(SELECT COALESCE(\
194+
") THEN NULL::tsquery ELSE COALESCE(\
179195
string_agg(\
180196
quote_literal(lexeme) || CASE WHEN token_ord = max_token_ord THEN ':*' ELSE '' END, \
181197
' & ' ORDER BY token_ord, lex_ord\
182198
), \
183199
''\
184-
)::tsquery \
200+
)::tsquery END \
185201
FROM (\
186202
SELECT raw_token.token_ord, normalized.lexeme, normalized.lex_ord, raw_token.max_token_ord \
187203
FROM (\
@@ -239,14 +255,7 @@ fn normalized_search_text(q: &str) -> Option<String> {
239255
#[datastore_span(name = "search", system = "postgresql")]
240256
pub async fn search(pool: &PgPool, query: &SearchQuery) -> Result<SearchResult, SearchError> {
241257
let profile_only = query.kinds.as_deref() == Some(&[0][..]);
242-
let Some(search_text) = normalized_search_text(&query.q).filter(|text| {
243-
query.mode != SearchMode::Prefix
244-
|| profile_only
245-
|| text
246-
.split_whitespace()
247-
.last()
248-
.is_some_and(|token| token.chars().count() >= PREFIX_MIN_CHARS)
249-
}) else {
258+
let Some(search_text) = normalized_search_text(&query.q) else {
250259
return Ok(SearchResult {
251260
hits: Vec::new(),
252261
page: query.page.clamp(1, PAGE_MAX),
@@ -277,7 +286,8 @@ pub async fn search(pool: &PgPool, query: &SearchQuery) -> Result<SearchResult,
277286
ts_rank_cd(search_tsv, search_query.query) AS rank \
278287
FROM events CROSS JOIN LATERAL (SELECT ",
279288
);
280-
push_tsquery(&mut qb, query.mode, &search_text);
289+
let min_prefix_chars = if profile_only { 0 } else { PREFIX_MIN_CHARS };
290+
push_tsquery(&mut qb, query.mode, &search_text, min_prefix_chars);
281291
qb.push(" AS query) AS search_query WHERE community_id = ");
282292
qb.push_bind(*query.community.as_uuid());
283293
qb.push(" AND deleted_at IS NULL AND search_tsv @@ search_query.query");
@@ -441,7 +451,7 @@ pub async fn search_profile_names(
441451
") AS tier \
442452
FROM events e CROSS JOIN LATERAL (SELECT ",
443453
);
444-
push_tsquery(&mut qb, SearchMode::Prefix, &search_text);
454+
push_tsquery(&mut qb, SearchMode::Prefix, &search_text, 0);
445455
qb.push(" AS query, ");
446456
qb.push(TIER_TEXT[0]);
447457
qb.push_bind(search_text.clone());

‎crates/buzz-search/tests/postgres_fts_integration.rs‎

Lines changed: 170 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,44 @@ async fn prefix_mode_matches_final_token_prefix_without_changing_full_text() {
349349
1_700_000_002,
350350
)
351351
.await;
352+
// Rows whose own parser output contains a short lexeme, so a short
353+
// trailing `:*` from punctuation in the query would match them.
354+
let contraction = rand_bytes32();
355+
let hyphenated = rand_bytes32();
356+
let profile = rand_bytes32();
357+
insert_event(
358+
&pool,
359+
c,
360+
contraction,
361+
rand_bytes32(),
362+
9,
363+
"don't panic",
364+
None,
365+
1_700_000_003,
366+
)
367+
.await;
368+
insert_event(
369+
&pool,
370+
c,
371+
hyphenated,
372+
rand_bytes32(),
373+
9,
374+
"pre-processing done",
375+
None,
376+
1_700_000_004,
377+
)
378+
.await;
379+
insert_event(
380+
&pool,
381+
c,
382+
profile,
383+
rand_bytes32(),
384+
0,
385+
r#"{"display_name":"prof"}"#,
386+
None,
387+
1_700_000_005,
388+
)
389+
.await;
352390

353391
let svc = SearchService::new(pool.clone());
354392
let full_text = svc
@@ -397,28 +435,117 @@ async fn prefix_mode_matches_final_token_prefix_without_changing_full_text() {
397435
assert!(prefix_ids.contains(&project_plan));
398436
assert!(prefix_ids.contains(&project_archive));
399437

400-
// A trailing token below three characters returns no hits for non-profile
401-
// prefix search, in either sort, whatever the kinds and whatever completed
402-
// words precede it; `pro` above proves the rows exist.
403-
// kind:0-only stays one-character typeahead (see the `jm` test above).
404-
for (q, kinds, sort) in [
405-
("pr", Some(vec![9]), buzz_search::SearchSort::Relevance),
406-
(" p ", Some(vec![9]), buzz_search::SearchSort::Relevance),
438+
// Non-profile prefix search returns no hits, in either sort, whatever the
439+
// kinds and whatever completed words precede it, when the trailing token's
440+
// `simple` parser output has a lexeme below three characters. Punctuation
441+
// must not pad the count: `pr.`, `@pr` and `#pr` still yield `pr`, `don't`
442+
// yields `t` and `pre-p` yields `p`. Each row would match without the
443+
// minimum. kind:0-only stays one-character typeahead.
444+
for (q, kinds, sort, expected) in [
445+
(
446+
"pr",
447+
Some(vec![9]),
448+
buzz_search::SearchSort::Relevance,
449+
vec![],
450+
),
451+
(
452+
" p ",
453+
Some(vec![9]),
454+
buzz_search::SearchSort::Relevance,
455+
vec![],
456+
),
407457
(
408458
"project pr",
409459
Some(vec![9]),
410460
buzz_search::SearchSort::Relevance,
461+
vec![],
411462
),
412463
(
413464
"project p ",
414465
Some(vec![9]),
415466
buzz_search::SearchSort::Relevance,
467+
vec![],
468+
),
469+
("pr", None, buzz_search::SearchSort::Relevance, vec![]),
470+
(
471+
"pr",
472+
Some(vec![0, 9]),
473+
buzz_search::SearchSort::Relevance,
474+
vec![],
475+
),
476+
("pr", Some(vec![9]), buzz_search::SearchSort::Newest, vec![]),
477+
(
478+
"pr.",
479+
Some(vec![9]),
480+
buzz_search::SearchSort::Relevance,
481+
vec![],
482+
),
483+
(
484+
"@pr",
485+
Some(vec![9]),
486+
buzz_search::SearchSort::Relevance,
487+
vec![],
488+
),
489+
(
490+
"#pr",
491+
Some(vec![9]),
492+
buzz_search::SearchSort::Relevance,
493+
vec![],
494+
),
495+
(
496+
"...p",
497+
Some(vec![9]),
498+
buzz_search::SearchSort::Relevance,
499+
vec![],
500+
),
501+
(
502+
"don't",
503+
Some(vec![9]),
504+
buzz_search::SearchSort::Relevance,
505+
vec![],
506+
),
507+
(
508+
"pre-p",
509+
Some(vec![9]),
510+
buzz_search::SearchSort::Relevance,
511+
vec![],
512+
),
513+
// Controls: padded but long enough, punctuation-only trailing token
514+
// (no `:*` lexeme, so the completed word matches exactly), long
515+
// hyphenated parts, a short completed word (exact, so not limited),
516+
// and a one-character profile query.
517+
(
518+
"pro.",
519+
Some(vec![9]),
520+
buzz_search::SearchSort::Relevance,
521+
vec![project_plan, project_archive, projectile_plan, hyphenated],
522+
),
523+
(
524+
"project ?",
525+
Some(vec![9]),
526+
buzz_search::SearchSort::Relevance,
527+
vec![project_plan, project_archive],
528+
),
529+
(
530+
"pre-pro",
531+
Some(vec![9]),
532+
buzz_search::SearchSort::Relevance,
533+
vec![hyphenated],
534+
),
535+
(
536+
"t pan",
537+
Some(vec![9]),
538+
buzz_search::SearchSort::Relevance,
539+
vec![contraction],
540+
),
541+
(
542+
"p",
543+
Some(vec![0]),
544+
buzz_search::SearchSort::Relevance,
545+
vec![profile],
416546
),
417-
("pr", None, buzz_search::SearchSort::Relevance),
418-
("pr", Some(vec![0, 9]), buzz_search::SearchSort::Relevance),
419-
("pr", Some(vec![9]), buzz_search::SearchSort::Newest),
420547
] {
421-
let short = svc
548+
let result = svc
422549
.search(&SearchQuery {
423550
community: c,
424551
q: q.into(),
@@ -433,12 +560,39 @@ async fn prefix_mode_matches_final_token_prefix_without_changing_full_text() {
433560
sort,
434561
})
435562
.await
436-
.expect("short prefix search ok");
437-
assert!(
438-
short.hits.is_empty(),
439-
"{q:?} {kinds:?} {sort:?} must not run a short prefix search"
440-
);
563+
.expect("prefix search ok");
564+
let mut got: Vec<[u8; 32]> = result.hits.iter().map(|h| h.event_id).collect();
565+
let mut want = expected.clone();
566+
got.sort();
567+
want.sort();
568+
assert_eq!(got, want, "{q:?} {kinds:?} {sort:?}");
441569
}
570+
// A short trailing lexeme is a prefix-mode rule only; full text still
571+
// matches the contraction exactly.
572+
let full_text_contraction = svc
573+
.search(&SearchQuery {
574+
community: c,
575+
q: "don't".into(),
576+
channel_scope: ChannelScope::Any,
577+
kinds: Some(vec![9]),
578+
authors: None,
579+
since: None,
580+
until: None,
581+
page: 1,
582+
per_page: 10,
583+
mode: buzz_search::SearchMode::FullText,
584+
sort: buzz_search::SearchSort::Relevance,
585+
})
586+
.await
587+
.expect("full text search ok");
588+
assert_eq!(
589+
full_text_contraction
590+
.hits
591+
.iter()
592+
.map(|h| h.event_id)
593+
.collect::<Vec<_>>(),
594+
vec![contraction]
595+
);
442596

443597
let multi_token_prefix = svc
444598
.search(&SearchQuery {

0 commit comments

Comments
 (0)