fix(memory-core): score CJK keyword search instead of textScore=0 (trigram FTS5)

draix · claude · draix · commit 2511c76abf1a · 2026-06-11T09:21:17.000-03:00
Multi-character CJK query tokens were emitted as a single quoted trigram MATCH phrase that only matches an exact contiguous substring, so MATCH returned zero rows and (because the LIKE fallback only fired on a thrown MATCH) the keyword channel collapsed to textScore=0 for every CJK query. - planKeywordSearch: decompose any CJK-bearing token into per-character LIKE substring terms (keeping contiguous non-CJK runs whole), not just tokens shorter than 3 characters. - searchKeyword: fall back to per-term LIKE when MATCH succeeds but returns zero rows, via a shared runLikeFallback helper reused by the exception path so both use the correctly decomposed term set. Adds CJK FTS regression tests (per-character AND semantics, empty-MATCH fallback, mixed ASCII+CJK token splitting); the failing cases reproduce textScore=0 against main. Fixes #92061 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
diff --git a/extensions/memory-core/src/memory/manager-search.test.ts b/extensions/memory-core/src/memory/manager-search.test.ts
@@ -222,6 +222,103 @@ describe("searchKeyword trigram fallback", () => {
 
     expect(repeated[0]?.score).toBe(unique[0]?.score);
   });
+
+  // ===== CJK FTS textScore regression coverage (issue #92061) =====
+  //
+  // A multi-character CJK query used to be emitted as a single quoted trigram
+  // MATCH phrase (e.g. MATCH "飞书插件配置"), which only matches the exact
+  // contiguous substring. Real documents rarely contain that verbatim run, so
+  // MATCH returned zero rows, no LIKE fallback fired, and every hybrid result
+  // collapsed to textScore=0. The fix decomposes CJK tokens into per-character
+  // LIKE terms and adds a LIKE fallback when MATCH returns zero rows.
+
+  itWithTrigramFts(
+    "scores a multi-character CJK query with textScore=1 instead of 0 (issue #92061)",
+    async () => {
+      // The query characters are all present, but never as the contiguous
+      // substring "飞书插件配置" — the doc has "...配置飞书插件". Pre-fix this
+      // produced an empty keyword result (textScore=0); post-fix it matches.
+      const results = await runSearch({
+        rows: [{ id: "feishu", path: "memory/zh.md", text: "如何配置飞书插件" }],
+        query: "飞书插件配置",
+      });
+      expect(results.map((row) => row.id)).toEqual(["feishu"]);
+      expect(results[0]?.textScore).toBe(1);
+      expect(results[0]?.textScore).toBeGreaterThan(0);
+    },
+  );
+
+  itWithTrigramFts("requires every CJK character of the query to be present", async () => {
+    const results = await runSearch({
+      rows: [
+        { id: "full", path: "memory/full.md", text: "如何配置飞书插件" },
+        // Contains 飞书 but not 插件配置 — must be excluded by the per-character
+        // AND semantics rather than matched on a partial overlap.
+        { id: "partial", path: "memory/partial.md", text: "飞书云文档简介" },
+      ],
+      query: "飞书插件配置",
+    });
+    expect(results.map((row) => row.id)).toEqual(["full"]);
+  });
+
+  itWithTrigramFts(
+    "covers Japanese kanji queries that are not a contiguous substring",
+    async () => {
+      const results = await runSearch({
+        rows: [{ id: "jp", path: "memory/jp.md", text: "東京の駅の周辺を歩く" }],
+        query: "東京駅周辺",
+      });
+      expect(results.map((row) => row.id)).toEqual(["jp"]);
+      expect(results[0]?.textScore).toBe(1);
+    },
+  );
+
+  itWithTrigramFts(
+    "falls back to LIKE when a trigram MATCH returns zero rows without throwing",
+    async () => {
+      // "id" is a sub-trigram (<3 chars) Latin token, so MATCH "id" returns
+      // zero rows (without throwing) under the trigram tokenizer, while the CJK
+      // token splits into substring terms. Pre-fix the empty MATCH short-
+      // circuited to no results because the LIKE fallback only ran on
+      // exceptions; post-fix the zero-row MATCH triggers the LIKE fallback.
+      const results = await runSearch({
+        rows: [{ id: "doc", path: "memory/mixed.md", text: "id配置文档说明" }],
+        query: "id 配置",
+      });
+      expect(results.map((row) => row.id)).toEqual(["doc"]);
+      expect(results[0]?.textScore).toBe(1);
+    },
+  );
+
+  itWithTrigramFts("keeps non-CJK runs whole when splitting a mixed CJK token", async () => {
+    const results = await runSearch({
+      rows: [
+        // Contains the substring "API" and the characters 飞书 (not contiguous).
+        { id: "hit", path: "memory/api.md", text: "查看API文档和飞书设置" },
+        // Has 飞书 and scattered letters but no "API" substring — must be
+        // excluded, proving the ASCII run is matched whole rather than as the
+        // individual letters A, P, I.
+        { id: "miss", path: "memory/apple.md", text: "苹果Apple的飞书通知" },
+      ],
+      query: "API飞书",
+    });
+    expect(results.map((row) => row.id)).toEqual(["hit"]);
+  });
+
+  itWithTrigramFts(
+    "still BM25-scores a mixed Latin+CJK query through MATCH (no regression)",
+    async () => {
+      // The Latin term is trigram-eligible and present, so MATCH succeeds and
+      // the keyword score stays a real BM25 value rather than the LIKE default.
+      const results = await runSearch({
+        rows: [{ id: "doc", path: "memory/mixed.md", text: "feishu 配置文档说明" }],
+        query: "feishu 配置",
+      });
+      expect(results.map((row) => row.id)).toEqual(["doc"]);
+      expect(results[0]?.textScore).toBeGreaterThan(0);
+      expect(results[0]?.textScore).toBeLessThan(1);
+    },
+  );
 });
 
 describe("searchKeyword FTS MATCH fallback", () => {
diff --git a/extensions/memory-core/src/memory/manager-search.ts b/extensions/memory-core/src/memory/manager-search.ts
@@ -94,28 +94,61 @@ function readCount(row: { count?: number | bigint } | undefined): number {
   return 0;
 }
 
+// Decompose a token that contains CJK characters into LIKE substring terms.
+// Each CJK character becomes its own term while contiguous non-CJK runs (e.g.
+// the "API" in "API配置") are kept whole, so a doc only has to contain each
+// character/run somewhere — not the exact concatenation — to match.
+function splitCjkToken(token: string): string[] {
+  const terms: string[] = [];
+  let asciiRun = "";
+  for (const char of Array.from(token)) {
+    if (SHORT_CJK_TRIGRAM_RE.test(char)) {
+      if (asciiRun) {
+        terms.push(asciiRun);
+        asciiRun = "";
+      }
+      terms.push(char);
+    } else {
+      asciiRun += char;
+    }
+  }
+  if (asciiRun) {
+    terms.push(asciiRun);
+  }
+  return terms;
+}
+
 function planKeywordSearch(params: {
   query: string;
   ftsTokenizer?: "unicode61" | "trigram";
   buildFtsQuery: (raw: string) => string | null;
-}): { matchQuery: string | null; substringTerms: string[] } {
+}): { matchQuery: string | null; substringTerms: string[]; fallbackLikeTerms: string[] } {
   if (params.ftsTokenizer !== "trigram") {
+    const tokens = normalizeStringEntries(params.query.match(FTS_QUERY_TOKEN_RE) ?? []);
     return {
       matchQuery: params.buildFtsQuery(params.query),
       substringTerms: [],
+      fallbackLikeTerms: uniqueStrings(tokens),
     };
   }
 
   const tokens = normalizeStringEntries(params.query.match(FTS_QUERY_TOKEN_RE) ?? []);
   if (tokens.length === 0) {
-    return { matchQuery: null, substringTerms: [] };
+    return { matchQuery: null, substringTerms: [], fallbackLikeTerms: [] };
   }
 
   const matchTerms: string[] = [];
   const substringTerms: string[] = [];
   for (const token of tokens) {
-    if (SHORT_CJK_TRIGRAM_RE.test(token) && Array.from(token).length < 3) {
-      substringTerms.push(token);
+    // The trigram tokenizer turns a multi-character CJK token into a quoted
+    // MATCH phrase that only matches the exact contiguous substring, which
+    // rarely exists verbatim in documents — so every CJK query scored
+    // textScore=0 (issue #92061). Route any CJK-bearing token through per-
+    // character LIKE terms instead, which reliably match regardless of where
+    // the characters appear. Single/double CJK characters already needed this
+    // because trigram requires >=3 characters to form a searchable token.
+    if (SHORT_CJK_TRIGRAM_RE.test(token)) {
+      substringTerms.push(...splitCjkToken(token));
       continue;
     }
     matchTerms.push(token);
@@ -124,6 +157,7 @@ function planKeywordSearch(params: {
   return {
     matchQuery: buildMatchQueryFromTerms(matchTerms),
     substringTerms,
+    fallbackLikeTerms: uniqueStrings([...matchTerms, ...substringTerms]),
   };
 }
 
@@ -346,6 +380,29 @@ export async function searchKeyword(params: {
   }>;
   let usedMatch = false;
 
+  // Per-term LIKE substring search used whenever MATCH is unusable: either it
+  // threw, or it succeeded but returned no rows for a query whose CJK tokens
+  // were decomposed into substring terms (issue #92061).
+  const runLikeFallback = (): typeof rows => {
+    const fallbackLikeClause = plan.fallbackLikeTerms
+      .map(() => " AND text LIKE ? ESCAPE '\\'")
+      .join("");
+    const fallbackLikeParams = plan.fallbackLikeTerms.map((term) => `%${escapeLikePattern(term)}%`);
+    return params.db
+      .prepare(
+        `SELECT id, path, source, start_line, end_line, text,\n` +
+          `       0 AS rank\n` +
+          `  FROM ${params.ftsTable}\n` +
+          ` WHERE 1=1${fallbackLikeClause}${liveChunkClause}${params.sourceFilter.sql}\n` +
+          ` LIMIT ?`,
+      )
+      .all(
+        ...fallbackLikeParams,
+        ...params.sourceFilter.params,
+        params.limit,
+      ) as typeof rows;
+  };
+
   if (plan.matchQuery) {
     try {
       rows = params.db
@@ -364,29 +421,22 @@ export async function searchKeyword(params: {
           params.limit,
         ) as typeof rows;
       usedMatch = true;
+      // A successful MATCH that returns nothing is the CJK failure mode: the
+      // quoted trigram phrase had no exact substring hit. Retry with LIKE on
+      // the decomposed per-character terms so the keyword component still
+      // contributes instead of collapsing every hybrid result to textScore=0.
+      if (rows.length === 0 && plan.substringTerms.length > 0) {
+        rows = runLikeFallback();
+        usedMatch = false;
+      }
     } catch (matchErr) {
       // FTS5 MATCH can fail on certain token patterns depending on the
       // Node.js sqlite runtime and tokenizer (e.g. unicode61 vs trigram).
       // Log the root cause, then fall back to per-token LIKE-based substring
       // search so results are still returned instead of being silently dropped.
       console.warn(`memory search: FTS5 MATCH failed, falling back to LIKE: ${String(matchErr)}`);
-      const queryTokens = normalizeStringEntries(params.query.match(FTS_QUERY_TOKEN_RE) ?? []);
-      const allTerms = uniqueStrings([...queryTokens, ...plan.substringTerms]);
-      const fallbackLikeClause = allTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join("");
-      const fallbackLikeParams = allTerms.map((term) => `%${escapeLikePattern(term)}%`);
-      rows = params.db
-        .prepare(
-          `SELECT id, path, source, start_line, end_line, text,\n` +
-            `       0 AS rank\n` +
-            `  FROM ${params.ftsTable}\n` +
-            ` WHERE 1=1${fallbackLikeClause}${liveChunkClause}${params.sourceFilter.sql}\n` +
-            ` LIMIT ?`,
-        )
-        .all(
-          ...fallbackLikeParams,
-          ...params.sourceFilter.params,
-          params.limit,
-        ) as typeof rows;
+      rows = runLikeFallback();
+      usedMatch = false;
     }
   } else {
     rows = params.db
