Skip to content

Commit 3ce48af

Browse files
TakhoffmanMitsuyuki Osabe
Takhoffman
and
Mitsuyuki Osabe
authored
Memory: add configurable FTS5 tokenizer for CJK text support (#56707)
Verified: - pnpm build - pnpm check - pnpm test -- extensions/memory-core/src/memory/manager-search.test.ts packages/memory-host-sdk/src/host/query-expansion.test.ts - pnpm test -- extensions/memory-core/src/memory/index.test.ts -t "reindexes when extraPaths change" - pnpm test -- src/config/schema.base.generated.test.ts - pnpm test -- src/media-understanding/image.test.ts - pnpm test Co-authored-by: Mitsuyuki Osabe <[email protected]>
1 parent 6f7ff54 commit 3ce48af

12 files changed

Lines changed: 310 additions & 24 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ Docs: https://docs.openclaw.ai
2020
- Memory/QMD: honor `memory.qmd.update.embedInterval` even when regular QMD update cadence is disabled or slower by arming a dedicated embed-cadence maintenance timer, while avoiding redundant timers when regular updates are already frequent enough. (#37326) Thanks @barronlroth.
2121
- Agents/memory flush: keep daily memory flush files append-only during embedded attempts so compaction writes do not overwrite earlier notes. (#53725) Thanks @HPluseven.
2222
- Web UI/markdown: stop bare auto-links from swallowing adjacent CJK text while preserving valid mixed-script path and query characters in rendered links. (#48410) Thanks @jnuyao.
23+
- Memory/FTS: add configurable trigram tokenization plus short-CJK substring fallback so memory search can find Chinese, Japanese, and Korean text without breaking mixed long-and-short queries. Thanks @carrotRakko.
2324

2425
## 2026.3.28
2526

‎extensions/memory-core/src/memory/manager-search.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import {
2+
ensureMemoryIndexSchema,
3+
requireNodeSqlite,
4+
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
5+
import { describe, expect, it } from "vitest";
6+
import { bm25RankToScore, buildFtsQuery } from "./hybrid.js";
7+
import { searchKeyword } from "./manager-search.js";
8+
9+
describe("searchKeyword trigram fallback", () => {
10+
const { DatabaseSync } = requireNodeSqlite();
11+
12+
function createTrigramDb() {
13+
const db = new DatabaseSync(":memory:");
14+
ensureMemoryIndexSchema({
15+
db,
16+
embeddingCacheTable: "embedding_cache",
17+
cacheEnabled: false,
18+
ftsTable: "chunks_fts",
19+
ftsEnabled: true,
20+
ftsTokenizer: "trigram",
21+
});
22+
return db;
23+
}
24+
25+
async function runSearch(params: {
26+
rows: Array<{ id: string; path: string; text: string }>;
27+
query: string;
28+
}) {
29+
const db = createTrigramDb();
30+
try {
31+
const insert = db.prepare(
32+
"INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
33+
);
34+
for (const row of params.rows) {
35+
insert.run(row.text, row.id, row.path, "memory", "mock-embed", 1, 1);
36+
}
37+
return await searchKeyword({
38+
db,
39+
ftsTable: "chunks_fts",
40+
providerModel: "mock-embed",
41+
query: params.query,
42+
ftsTokenizer: "trigram",
43+
limit: 10,
44+
snippetMaxChars: 200,
45+
sourceFilter: { sql: "", params: [] },
46+
buildFtsQuery,
47+
bm25RankToScore,
48+
});
49+
} finally {
50+
db.close();
51+
}
52+
}
53+
54+
it("finds short Chinese queries with substring fallback", async () => {
55+
const results = await runSearch({
56+
rows: [{ id: "1", path: "memory/zh.md", text: "今天玩成语接龙游戏" }],
57+
query: "成语",
58+
});
59+
expect(results.map((row) => row.id)).toContain("1");
60+
expect(results[0]?.textScore).toBe(1);
61+
});
62+
63+
it("finds short Japanese and Korean queries with substring fallback", async () => {
64+
const japaneseResults = await runSearch({
65+
rows: [{ id: "jp", path: "memory/jp.md", text: "今日はしりとり大会" }],
66+
query: "しり とり",
67+
});
68+
expect(japaneseResults.map((row) => row.id)).toEqual(["jp"]);
69+
70+
const koreanResults = await runSearch({
71+
rows: [{ id: "ko", path: "memory/ko.md", text: "오늘 끝말잇기 게임을 했다" }],
72+
query: "끝말",
73+
});
74+
expect(koreanResults.map((row) => row.id)).toEqual(["ko"]);
75+
});
76+
77+
it("keeps MATCH semantics for long trigram terms while requiring short CJK substrings", async () => {
78+
const results = await runSearch({
79+
rows: [
80+
{ id: "match", path: "memory/good.md", text: "今天玩成语接龙游戏" },
81+
{ id: "partial", path: "memory/partial.md", text: "今天玩成语接龙" },
82+
],
83+
query: "成语接龙 游戏",
84+
});
85+
expect(results.map((row) => row.id)).toEqual(["match"]);
86+
expect(results[0]?.textScore).toBeGreaterThan(0);
87+
});
88+
});

‎extensions/memory-core/src/memory/manager-search.ts‎

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import {
77

88
const vectorToBlob = (embedding: number[]): Buffer =>
99
Buffer.from(new Float32Array(embedding).buffer);
10+
const FTS_QUERY_TOKEN_RE = /[\p{L}\p{N}_]+/gu;
11+
const SHORT_CJK_TRIGRAM_RE = /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\u3131-\u3163]/u;
1012

1113
export type SearchSource = string;
1214

@@ -20,6 +22,55 @@ export type SearchRowResult = {
2022
source: SearchSource;
2123
};
2224

25+
function escapeLikePattern(term: string): string {
26+
return term.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
27+
}
28+
29+
function buildMatchQueryFromTerms(terms: string[]): string | null {
30+
if (terms.length === 0) {
31+
return null;
32+
}
33+
const quoted = terms.map((term) => `"${term.replaceAll('"', "")}"`);
34+
return quoted.join(" AND ");
35+
}
36+
37+
function planKeywordSearch(params: {
38+
query: string;
39+
ftsTokenizer?: "unicode61" | "trigram";
40+
buildFtsQuery: (raw: string) => string | null;
41+
}): { matchQuery: string | null; substringTerms: string[] } {
42+
if (params.ftsTokenizer !== "trigram") {
43+
return {
44+
matchQuery: params.buildFtsQuery(params.query),
45+
substringTerms: [],
46+
};
47+
}
48+
49+
const tokens =
50+
params.query
51+
.match(FTS_QUERY_TOKEN_RE)
52+
?.map((token) => token.trim())
53+
.filter(Boolean) ?? [];
54+
if (tokens.length === 0) {
55+
return { matchQuery: null, substringTerms: [] };
56+
}
57+
58+
const matchTerms: string[] = [];
59+
const substringTerms: string[] = [];
60+
for (const token of tokens) {
61+
if (SHORT_CJK_TRIGRAM_RE.test(token) && Array.from(token).length < 3) {
62+
substringTerms.push(token);
63+
continue;
64+
}
65+
matchTerms.push(token);
66+
}
67+
68+
return {
69+
matchQuery: buildMatchQueryFromTerms(matchTerms),
70+
substringTerms,
71+
};
72+
}
73+
2374
export async function searchVector(params: {
2475
db: DatabaseSync;
2576
vectorTable: string;
@@ -141,6 +192,7 @@ export async function searchKeyword(params: {
141192
ftsTable: string;
142193
providerModel: string | undefined;
143194
query: string;
195+
ftsTokenizer?: "unicode61" | "trigram";
144196
limit: number;
145197
snippetMaxChars: number;
146198
sourceFilter: { sql: string; params: SearchSource[] };
@@ -150,25 +202,42 @@ export async function searchKeyword(params: {
150202
if (params.limit <= 0) {
151203
return [];
152204
}
153-
const ftsQuery = params.buildFtsQuery(params.query);
154-
if (!ftsQuery) {
205+
const plan = planKeywordSearch({
206+
query: params.query,
207+
ftsTokenizer: params.ftsTokenizer,
208+
buildFtsQuery: params.buildFtsQuery,
209+
});
210+
if (!plan.matchQuery && plan.substringTerms.length === 0) {
155211
return [];
156212
}
157213

158214
// When providerModel is undefined (FTS-only mode), search all models
159215
const modelClause = params.providerModel ? " AND model = ?" : "";
160216
const modelParams = params.providerModel ? [params.providerModel] : [];
217+
const substringClause = plan.substringTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join("");
218+
const substringParams = plan.substringTerms.map((term) => `%${escapeLikePattern(term)}%`);
219+
const whereClause = plan.matchQuery
220+
? `${params.ftsTable} MATCH ?${substringClause}${modelClause}${params.sourceFilter.sql}`
221+
: `1=1${substringClause}${modelClause}${params.sourceFilter.sql}`;
222+
const queryParams = [
223+
...(plan.matchQuery ? [plan.matchQuery] : []),
224+
...substringParams,
225+
...modelParams,
226+
...params.sourceFilter.params,
227+
params.limit,
228+
];
229+
const rankExpression = plan.matchQuery ? `bm25(${params.ftsTable})` : "0";
161230

162231
const rows = params.db
163232
.prepare(
164233
`SELECT id, path, source, start_line, end_line, text,\n` +
165-
` bm25(${params.ftsTable}) AS rank\n` +
234+
` ${rankExpression} AS rank\n` +
166235
` FROM ${params.ftsTable}\n` +
167-
` WHERE ${params.ftsTable} MATCH ?${modelClause}${params.sourceFilter.sql}\n` +
236+
` WHERE ${whereClause}\n` +
168237
` ORDER BY rank ASC\n` +
169238
` LIMIT ?`,
170239
)
171-
.all(ftsQuery, ...modelParams, ...params.sourceFilter.params, params.limit) as Array<{
240+
.all(...queryParams) as Array<{
172241
id: string;
173242
path: string;
174243
source: SearchSource;
@@ -179,7 +248,7 @@ export async function searchKeyword(params: {
179248
}>;
180249

181250
return rows.map((row) => {
182-
const textScore = params.bm25RankToScore(row.rank);
251+
const textScore = plan.matchQuery ? params.bm25RankToScore(row.rank) : 1;
183252
return {
184253
id: row.id,
185254
path: row.path,

‎extensions/memory-core/src/memory/manager-sync-ops.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ type MemoryIndexMeta = {
5656
chunkTokens: number;
5757
chunkOverlap: number;
5858
vectorDims?: number;
59+
ftsTokenizer?: string;
5960
};
6061

6162
type MemorySyncProgressState = {
@@ -362,6 +363,7 @@ export abstract class MemoryManagerSyncOps {
362363
cacheEnabled: this.cache.enabled,
363364
ftsTable: FTS_TABLE,
364365
ftsEnabled: this.fts.enabled,
366+
ftsTokenizer: this.settings.store.fts.tokenizer,
365367
});
366368
this.fts.available = result.ftsAvailable;
367369
if (result.ftsError) {
@@ -1028,7 +1030,8 @@ export abstract class MemoryManagerSyncOps {
10281030
meta.scopeHash !== configuredScopeHash ||
10291031
meta.chunkTokens !== this.settings.chunking.tokens ||
10301032
meta.chunkOverlap !== this.settings.chunking.overlap ||
1031-
(vectorReady && !meta?.vectorDims);
1033+
(vectorReady && !meta?.vectorDims) ||
1034+
(meta.ftsTokenizer ?? "unicode61") !== this.settings.store.fts.tokenizer;
10321035
try {
10331036
if (needsFullReindex) {
10341037
if (
@@ -1220,6 +1223,7 @@ export abstract class MemoryManagerSyncOps {
12201223
scopeHash: this.resolveConfiguredScopeHash(),
12211224
chunkTokens: this.settings.chunking.tokens,
12221225
chunkOverlap: this.settings.chunking.overlap,
1226+
ftsTokenizer: this.settings.store.fts.tokenizer,
12231227
};
12241228
if (!nextMeta) {
12251229
throw new Error("Failed to compute memory index metadata for reindexing.");
@@ -1292,6 +1296,7 @@ export abstract class MemoryManagerSyncOps {
12921296
scopeHash: this.resolveConfiguredScopeHash(),
12931297
chunkTokens: this.settings.chunking.tokens,
12941298
chunkOverlap: this.settings.chunking.overlap,
1299+
ftsTokenizer: this.settings.store.fts.tokenizer,
12951300
};
12961301
if (this.vector.available && this.vector.dims) {
12971302
nextMeta.vectorDims = this.vector.dims;
@@ -1306,9 +1311,10 @@ export abstract class MemoryManagerSyncOps {
13061311
this.db.exec(`DELETE FROM chunks`);
13071312
if (this.fts.enabled && this.fts.available) {
13081313
try {
1309-
this.db.exec(`DELETE FROM ${FTS_TABLE}`);
1314+
this.db.exec(`DROP TABLE IF EXISTS ${FTS_TABLE}`);
13101315
} catch {}
13111316
}
1317+
this.ensureSchema();
13121318
this.dropVectorTable();
13131319
this.vector.dims = undefined;
13141320
this.sessionsDirtyFiles.clear();

‎extensions/memory-core/src/memory/manager.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,9 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
352352

353353
// Extract keywords for better FTS matching on conversational queries
354354
// e.g., "that thing we discussed about the API" → ["discussed", "API"]
355-
const keywords = extractKeywords(cleaned);
355+
const keywords = extractKeywords(cleaned, {
356+
ftsTokenizer: this.settings.store.fts.tokenizer,
357+
});
356358
const searchTerms = keywords.length > 0 ? keywords : [cleaned];
357359

358360
// Search with each keyword and merge results
@@ -488,6 +490,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
488490
ftsTable: FTS_TABLE,
489491
providerModel,
490492
query,
493+
ftsTokenizer: this.settings.store.fts.tokenizer,
491494
limit,
492495
snippetMaxChars: SNIPPET_MAX_CHARS,
493496
sourceFilter,

‎packages/memory-host-sdk/src/host/memory-schema.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export function ensureMemoryIndexSchema(params: {
66
cacheEnabled: boolean;
77
ftsTable: string;
88
ftsEnabled: boolean;
9+
ftsTokenizer?: "unicode61" | "trigram";
910
}): { ftsAvailable: boolean; ftsError?: string } {
1011
params.db.exec(`
1112
CREATE TABLE IF NOT EXISTS meta (
@@ -58,6 +59,8 @@ export function ensureMemoryIndexSchema(params: {
5859
let ftsError: string | undefined;
5960
if (params.ftsEnabled) {
6061
try {
62+
const tokenizer = params.ftsTokenizer ?? "unicode61";
63+
const tokenizeClause = tokenizer === "trigram" ? `, tokenize='trigram case_sensitive 0'` : "";
6164
params.db.exec(
6265
`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n` +
6366
` text,\n` +
@@ -67,7 +70,7 @@ export function ensureMemoryIndexSchema(params: {
6770
` model UNINDEXED,\n` +
6871
` start_line UNINDEXED,\n` +
6972
` end_line UNINDEXED\n` +
70-
`);`,
73+
`${tokenizeClause});`,
7174
);
7275
ftsAvailable = true;
7376
} catch (err) {

‎packages/memory-host-sdk/src/host/query-expansion.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,51 @@ describe("extractKeywords", () => {
174174
const testCount = keywords.filter((k) => k === "test").length;
175175
expect(testCount).toBe(1);
176176
});
177+
178+
describe("with trigram tokenizer", () => {
179+
const trigramOpts = { ftsTokenizer: "trigram" as const };
180+
181+
it("emits whole CJK block instead of unigrams in trigram mode", () => {
182+
const defaultKeywords = extractKeywords("之前讨论的那个方案");
183+
const trigramKeywords = extractKeywords("之前讨论的那个方案", trigramOpts);
184+
// Default mode produces bigrams
185+
expect(defaultKeywords).toContain("讨论");
186+
expect(defaultKeywords).toContain("方案");
187+
// Trigram mode emits the whole contiguous CJK block (FTS5 trigram
188+
// requires >= 3 chars per term; individual characters return no results)
189+
expect(trigramKeywords).toContain("之前讨论的那个方案");
190+
expect(trigramKeywords).not.toContain("讨论");
191+
expect(trigramKeywords).not.toContain("方案");
192+
});
193+
194+
it("skips Japanese kanji bigrams in trigram mode", () => {
195+
const defaultKeywords = extractKeywords("経済政策について");
196+
const trigramKeywords = extractKeywords("経済政策について", trigramOpts);
197+
// Default mode adds kanji bigrams: 経済, 済政, 政策
198+
expect(defaultKeywords).toContain("経済");
199+
expect(defaultKeywords).toContain("済政");
200+
expect(defaultKeywords).toContain("政策");
201+
// Trigram mode keeps the full kanji block but skips bigram splitting
202+
expect(trigramKeywords).toContain("経済政策");
203+
expect(trigramKeywords).not.toContain("済政");
204+
});
205+
206+
it("still filters stop words in trigram mode", () => {
207+
const keywords = extractKeywords("これ それ そして どう", trigramOpts);
208+
expect(keywords).not.toContain("これ");
209+
expect(keywords).not.toContain("それ");
210+
expect(keywords).not.toContain("そして");
211+
expect(keywords).not.toContain("どう");
212+
});
213+
214+
it("does not affect English keyword extraction", () => {
215+
const keywords = extractKeywords("that thing we discussed about the API", trigramOpts);
216+
expect(keywords).toContain("discussed");
217+
expect(keywords).toContain("api");
218+
expect(keywords).not.toContain("that");
219+
expect(keywords).not.toContain("the");
220+
});
221+
});
177222
});
178223

179224
describe("expandQueryForFts", () => {

0 commit comments

Comments
 (0)