Skip to content

Commit 6c9fa82

Browse files
MarcoDotIOclaude
andcommitted
fix(server): short non-Latin search words narrow hits before the limit
Skipping the SQL check let newer messages with only the long word fill the hit limit before the matcher rejected them. Each short word is now checked as the case and accent spellings it can match, so the filter still runs first. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent a00ca2a commit 6c9fa82

2 files changed

Lines changed: 68 additions & 4 deletions

File tree

‎apps/server/src/orchestration-v2/ThreadSearch.test.ts‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ it.layer(layerTest)("ThreadSearch", (it) => {
270270
for (const match of ['"needle"', null]) {
271271
const plan = yield* sql<{
272272
readonly detail: string;
273-
}>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
273+
}>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
274274
assert.isFalse(
275275
plan.some((row) => row.detail.includes("TEMP B-TREE")),
276276
plan.map((row) => row.detail).join("\n"),
@@ -732,6 +732,49 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
732732
);
733733
});
734734

735+
it.effect.each([true, false])(
736+
"ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
737+
(indexed) =>
738+
Effect.gen(function* () {
739+
const projections = yield* ProjectionStore.ProjectionStoreV2;
740+
const search = yield* ThreadSearch.ThreadSearch;
741+
const sql = yield* SqlClient.SqlClient;
742+
const project = ProjectId.make("project:short-limit");
743+
const older = ThreadId.make("thread:short-older");
744+
const newer = ThreadId.make("thread:short-newer");
745+
yield* createProject(project);
746+
yield* Effect.forEach(
747+
[
748+
thread(older, project),
749+
message(older, "older", "user", "В логах ошибка"),
750+
thread(newer, project),
751+
],
752+
projections.apply,
753+
{ discard: true },
754+
);
755+
if (!indexed) {
756+
yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
757+
yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
758+
yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
759+
yield* sql`DROP TABLE orchestration_v2_message_search`;
760+
}
761+
// More newer hits for the long word than one read holds, none with the short one.
762+
yield* sql`
763+
WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
764+
INSERT INTO orchestration_v2_projection_messages
765+
(message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
766+
SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
767+
${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
768+
FROM n
769+
`;
770+
const { matches } = yield* search.search({ query: "ошибка в" });
771+
assert.deepEqual(
772+
matches.map((match) => match.threadId),
773+
[older],
774+
);
775+
}).pipe(Effect.provide(layerTest)),
776+
);
777+
735778
it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
736779
it.effect("finds them in any case or accent, with or without the index", () =>
737780
Effect.gen(function* () {

‎apps/server/src/orchestration-v2/ThreadSearch.ts‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ const SearchRequest = Schema.Struct({
6767
match: Schema.NullOr(Schema.String),
6868
/** Tokens every hit must contain, checked on the text. */
6969
substrings: TokenList,
70+
/** For each short word outside ASCII, the spellings a hit must contain one of. */
71+
shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
7072
projectId: Schema.NullOr(ProjectId),
7173
/** Threads the caller already lists, left out before the hit limit. */
7274
excludedThreadIds: TokenList,
@@ -101,6 +103,19 @@ const MESSAGE_CANDIDATE_LIMIT = 10_000;
101103
const MESSAGE_SCAN_LIMIT = 20_000;
102104
const NON_ASCII = /[\u0080-\uffff]/;
103105

106+
/** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */
107+
function shortTokenSpellings(token: string): ReadonlyArray<string> {
108+
const forms = (char: string) => {
109+
const plain = char.normalize("NFD").replace(/\p{M}/gu, "");
110+
return [char, char.toUpperCase(), plain, plain.toUpperCase()];
111+
};
112+
const spellings = Array.from(token).reduce<ReadonlyArray<string>>(
113+
(prefixes, char) => prefixes.flatMap((prefix) => forms(char).map((form) => prefix + form)),
114+
[""],
115+
);
116+
return [...new Set(spellings)];
117+
}
118+
104119
/**
105120
* The SQL inputs for a message search, or null when the query has no token
106121
* long enough to search. Tokens of three or more characters go to the
@@ -115,12 +130,11 @@ function buildMessageSearch(
115130
if (long.length === 0) return null;
116131
// Two-character CJK words are too short for trigrams.
117132
const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);
118-
// SQLite's lower() folds ASCII only, so "в" would reject a message that has "В". The matcher
119-
// still requires these tokens, ignoring case and accents.
120133
const narrowing = short.filter((token) => !NON_ASCII.test(token));
121134
return {
122135
match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
123136
substrings: indexed ? narrowing : [...long, ...narrowing],
137+
shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
124138
};
125139
}
126140

@@ -164,7 +178,7 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
164178
// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
165179
export const messageHitsQuery = (
166180
sql: SqlClient.SqlClient,
167-
{ match, substrings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
181+
{ match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
168182
) => sql`
169183
WITH eligible AS MATERIALIZED (
170184
SELECT threads.thread_id, threads.project_id
@@ -229,6 +243,13 @@ export const messageHitsQuery = (
229243
SELECT 1 FROM json_each(${substrings}) AS token
230244
WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
231245
)
246+
AND NOT EXISTS (
247+
SELECT 1 FROM json_each(${shortSpellings}) AS token
248+
WHERE NOT EXISTS (
249+
SELECT 1 FROM json_each(token.value) AS spelling
250+
WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
251+
)
252+
)
232253
-- The index's own rowid, so FTS5 returns hits newest first and stops at
233254
-- the limit; ordering by the messages table's rowid sorts every hit.
234255
ORDER BY "messageRowid" DESC

0 commit comments

Comments
 (0)