@@ -67,6 +67,8 @@ const SearchRequest = Schema.Struct({
6767 match : Schema . NullOr ( Schema . String ) ,
6868 /** Tokens every hit must contain, checked on the text. */
6969 substrings : TokenList ,
70+ /** For each short word outside ASCII, the spellings a hit must contain one of. */
71+ shortSpellings : Schema . fromJsonString ( Schema . Array ( Schema . Array ( Schema . String ) ) ) ,
7072 projectId : Schema . NullOr ( ProjectId ) ,
7173 /** Threads the caller already lists, left out before the hit limit. */
7274 excludedThreadIds : TokenList ,
@@ -101,6 +103,19 @@ const MESSAGE_CANDIDATE_LIMIT = 10_000;
101103const MESSAGE_SCAN_LIMIT = 20_000 ;
102104const NON_ASCII = / [ \u0080 - \uffff ] / ;
103105
106+ /** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */
107+ function shortTokenSpellings ( token : string ) : ReadonlyArray < string > {
108+ const forms = ( char : string ) => {
109+ const plain = char . normalize ( "NFD" ) . replace ( / \p{ M} / gu, "" ) ;
110+ return [ char , char . toUpperCase ( ) , plain , plain . toUpperCase ( ) ] ;
111+ } ;
112+ const spellings = Array . from ( token ) . reduce < ReadonlyArray < string > > (
113+ ( prefixes , char ) => prefixes . flatMap ( ( prefix ) => forms ( char ) . map ( ( form ) => prefix + form ) ) ,
114+ [ "" ] ,
115+ ) ;
116+ return [ ...new Set ( spellings ) ] ;
117+ }
118+
104119/**
105120 * The SQL inputs for a message search, or null when the query has no token
106121 * long enough to search. Tokens of three or more characters go to the
@@ -115,12 +130,11 @@ function buildMessageSearch(
115130 if ( long . length === 0 ) return null ;
116131 // Two-character CJK words are too short for trigrams.
117132 const indexed = hasIndex && long . every ( ( token ) => Array . from ( token ) . length >= 3 ) ;
118- // SQLite's lower() folds ASCII only, so "в" would reject a message that has "В". The matcher
119- // still requires these tokens, ignoring case and accents.
120133 const narrowing = short . filter ( ( token ) => ! NON_ASCII . test ( token ) ) ;
121134 return {
122135 match : indexed ? long . map ( ( token ) => `"${ token . replaceAll ( '"' , '""' ) } "` ) . join ( " " ) : null ,
123136 substrings : indexed ? narrowing : [ ...long , ...narrowing ] ,
137+ shortSpellings : short . filter ( ( token ) => NON_ASCII . test ( token ) ) . map ( shortTokenSpellings ) ,
124138 } ;
125139}
126140
@@ -164,7 +178,7 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
164178// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
165179export const messageHitsQuery = (
166180 sql : SqlClient . SqlClient ,
167- { match, substrings, projectId, excludedThreadIds } : typeof SearchRequest . Encoded ,
181+ { match, substrings, shortSpellings , projectId, excludedThreadIds } : typeof SearchRequest . Encoded ,
168182) => sql `
169183 WITH eligible AS MATERIALIZED (
170184 SELECT threads.thread_id, threads.project_id
@@ -229,6 +243,13 @@ export const messageHitsQuery = (
229243 SELECT 1 FROM json_each(${ substrings } ) AS token
230244 WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
231245 )
246+ AND NOT EXISTS (
247+ SELECT 1 FROM json_each(${ shortSpellings } ) AS token
248+ WHERE NOT EXISTS (
249+ SELECT 1 FROM json_each(token.value) AS spelling
250+ WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
251+ )
252+ )
232253 -- The index's own rowid, so FTS5 returns hits newest first and stops at
233254 -- the limit; ordering by the messages table's rowid sorts every hit.
234255 ORDER BY "messageRowid" DESC
0 commit comments