Skip to content

Commit a00ca2a

Browse files
MarcoDotIOclaude
andcommitted
fix(server): message search keeps results when a short non-Latin word differs in case
A short word after a longer one ("ошибка в") is checked in SQL with SQLite's lower(), which folds ASCII only, so a message that only has "В" was dropped before the matcher, which ignores case and accents, saw it. Non-ASCII short words now skip that SQL check; the matcher still requires them. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent bff68fb commit a00ca2a

2 files changed

Lines changed: 41 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/ThreadSearch.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,42 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
732732
);
733733
});
734734

735+
it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
736+
it.effect("finds them in any case or accent, with or without the index", () =>
737+
Effect.gen(function* () {
738+
const projections = yield* ProjectionStore.ProjectionStoreV2;
739+
const search = yield* ThreadSearch.ThreadSearch;
740+
const sql = yield* SqlClient.SqlClient;
741+
const project = ProjectId.make("project:short-non-ascii");
742+
const russian = ThreadId.make("thread:short-russian");
743+
const french = ThreadId.make("thread:short-french");
744+
yield* createProject(project);
745+
yield* Effect.forEach(
746+
[
747+
thread(russian, project),
748+
message(russian, "russian", "user", "В логах ошибка"),
749+
thread(french, project),
750+
message(french, "french", "user", "École fermée à Lyon"),
751+
],
752+
projections.apply,
753+
{ discard: true },
754+
);
755+
const threadIds = (query: string) =>
756+
search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
757+
758+
const assertMatches = Effect.gen(function* () {
759+
assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
760+
assert.deepEqual(yield* threadIds("fermée éc"), [french]);
761+
// They still narrow: a word the message lacks rules it out.
762+
assert.deepEqual(yield* threadIds("ошибка я"), []);
763+
});
764+
yield* assertMatches;
765+
yield* sql`DROP TABLE orchestration_v2_message_search`;
766+
yield* assertMatches;
767+
}),
768+
);
769+
});
770+
735771
it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {
736772
it.effect(
737773
"matches Korean, Japanese, and two-character Chinese words, and whole short words",

‎apps/server/src/orchestration-v2/ThreadSearch.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ const MESSAGE_HIT_LIMIT = 2000;
9999
const MESSAGE_CANDIDATE_LIMIT = 10_000;
100100
// Without the index, how many of the newest messages a search scans.
101101
const MESSAGE_SCAN_LIMIT = 20_000;
102+
const NON_ASCII = /[\u0080-\uffff]/;
102103

103104
/**
104105
* The SQL inputs for a message search, or null when the query has no token
@@ -114,9 +115,12 @@ function buildMessageSearch(
114115
if (long.length === 0) return null;
115116
// Two-character CJK words are too short for trigrams.
116117
const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);
118+
// SQLite's lower() folds ASCII only, so "в" would reject a message that has "В". The matcher
119+
// still requires these tokens, ignoring case and accents.
120+
const narrowing = short.filter((token) => !NON_ASCII.test(token));
117121
return {
118122
match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
119-
substrings: indexed ? short : [...long, ...short],
123+
substrings: indexed ? narrowing : [...long, ...narrowing],
120124
};
121125
}
122126

0 commit comments

Comments
 (0)