Skip to content

Commit 4666d56

Browse files
authored
fix(vector): boot 回填不再被模型指纹短路 + 只补活跃行 (#128) (#141)
Issue #128 两处根因(数据损坏类): 1. needsEmbedding 补 forgotten = 0 AND archived = 0 过滤(与 embeddedCount/count 口径对齐):归档/遗忘行不参与召回与聚类,不该吃掉 回填配额——报告实测 /vector-reindex 的 50 个名额全部落在归档行, 88 条活跃记忆长期无向量。boot 回填与 /vector-reindex(rebuildIndex 同样走 needsEmbedding)一并生效。 2. boot 回填的模型指纹门删除:原实现「指纹匹配即 return」,任意一条 嵌入成功写入 vector_meta 后,此后每次启动都在门上短路——写入时嵌入 失败(embedder 未就绪/服务商限流)的行永远补不上。回填循环本就有 MAX_TOTAL 上限,抽成导出的 backfillMissingEmbeddings()(指纹匹配时 markModel 幂等),指纹只作记账不再拦路。 测试:+3(needsEmbedding 活跃行过滤、指纹匹配场景下活跃行补回 + 归档行跳过、maxTotal 上限与剩余排队)。全量 839 项通过。
1 parent 94a59fd commit 4666d56

7 files changed

Lines changed: 193 additions & 73 deletions

File tree

‎dsh-mneme/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ dsh web
433433
| `entitySearchEnabled` | `true` | `entity:` / `attr:` 前缀搜索开关 |
434434
| `trustEpistemicWeighting` | `false` | 记忆可信度加权(v0.4.5,opt-in 默认关):记忆按来源分级 `observation`> `inferred` > `subjective`,开启后检索排序优先高可信记忆、注入对 observation 标注 `[verified]`、dream merge/conflict 偏向高可信一方;关闭时 `epistemic_status` 仅随保存落库、不参与行为 |
435435
| `evalPersistTestResults` | `false` | 检索评估落库(v0.4.5,opt-in 默认关):开启后 `evaluateRetrieval` 把 precision/recall/mrr 快照写入 `recall_evals`;默认关时仅返回调用方不落库。生产 `searchMemories` 审计始终走 `recall_runs`,无条件不触碰 `recall_evals` |
436-
| `autoReindexOnBoot` | `true` | 存量记忆缺 embedding 时,向量已配置则启动后延迟后台按批次限速自动回填重建(设为 `false` 仅手动重建) |
436+
| `autoReindexOnBoot` | `true` | 存量记忆缺 embedding 时,向量已配置则启动后延迟后台按批次限速自动回填重建(设为 `false` 仅手动重建)。只补**活跃**记忆(归档/遗忘行不参与召回,#128);模型指纹一致时照常补缺失行、不再短路(#128) |
437437
| `hybridInject` | `true` | 注入语义召回优先(v0.4.6,Bug4):`injectCandidates` 带非空 query 时先走向量索引语义召回候选,规则筛选补足/去重;空 query / 无向量回退旧逻辑 |
438438
| `bm25SearchEnabled` | `true` | BM25 稀疏第三路召回(v0.5.0):ASCII 词元 + CJK bigram,IDF 加权,散词/ID/代码片段查询不再依赖子串命中 |
439439
| `adaptiveThresholdEnabled` | `true` | 自适应相似度阈值(v0.5.0):按查询形态动态截断(前缀 0.5 / 短查询 0.7 / 长查询 0.6 / 头部分差大放宽 0.5),显式传 `threshold` 走旧行为 |

‎dsh-mneme/lib/index.js‎

Lines changed: 49 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,44 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
8787
};
8888
}
8989

90+
/**
91+
* Issue #128: bounded backfill of rows still missing an embedding (active rows
92+
* only — needsEmbedding filters archived/forgotten). Exported for tests.
93+
*
94+
* Runs regardless of the model fingerprint: the old call-site gate returned
95+
* early when vector_meta already held the embedder's hash, permanently
96+
* orphaning rows whose embed failed at write time (embedder not ready /
97+
* provider rate limit) — one successful embed was enough to never backfill
98+
* again. markModel is idempotent when the fingerprint already matches, so
99+
* re-running costs nothing beyond the actually-missing rows.
100+
*/
101+
export async function backfillMissingEmbeddings({
102+
store, embedder, vectorIndex, logger,
103+
maxTotal = 500, batchSize = 10, rateLimitMs = 200
104+
}) {
105+
let indexed = 0;
106+
for (let done = 0; done < maxTotal;) {
107+
const rows = store.needsEmbedding(batchSize);
108+
if (!rows.length) break;
109+
for (const row of rows) {
110+
try {
111+
const text = [row.title, row.content].filter(Boolean).join("\n");
112+
const vector = await embedder.embedSingle(text);
113+
if (vector?.length) {
114+
store.setEmbedding(row.id, vector);
115+
indexed++;
116+
}
117+
} catch { /* skip the bad row */ }
118+
}
119+
done += rows.length;
120+
// Rate limit: space out batches so the provider is not hammered.
121+
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
122+
}
123+
if (indexed > 0 && embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
124+
logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
125+
return indexed;
126+
}
127+
90128
export const apply = (ctx, config) => {
91129
const rawCfg = Config(config);
92130

@@ -303,10 +341,10 @@ export const apply = (ctx, config) => {
303341

304342
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
305343
// is configured and rows still lack an embedding (e.g. written before vector
306-
// search was enabled) AND the vector_meta fingerprint is absent or stale, the
307-
// index is rebuilt in the background after a short delay. Gated on
308-
// cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
309-
// never floods the provider. Failures degrade silently — search stays keyword.
344+
// search was enabled), the backfill runs in the background after a short
345+
// delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
346+
// large backlog never floods the provider. Failures degrade silently —
347+
// search stays keyword.
310348
function scheduleAutoReindex() {
311349
if (cfg.autoReindexOnBoot === false) return;
312350
const attempt = (tries) => {
@@ -319,36 +357,13 @@ export const apply = (ctx, config) => {
319357
return;
320358
}
321359
if (!store.needsEmbedding(1).length) return; // nothing to backfill
322-
// Model fingerprint gate: vectors already produced by the same model
323-
// mean there is no drift and no rebuild needed.
324-
const current = embedder.modelHash;
325-
if (current && vectorIndex.modelHash?.() === current) return;
326-
const BATCH = 10;
327-
const MAX_TOTAL = 500; // bound boot-time work
328-
(async () => {
329-
let indexed = 0;
330-
for (let done = 0; done < MAX_TOTAL;) {
331-
const rows = store.needsEmbedding(BATCH);
332-
if (!rows.length) break;
333-
for (const row of rows) {
334-
try {
335-
const text = [row.title, row.content].filter(Boolean).join("\n");
336-
const vector = await embedder.embedSingle(text);
337-
if (vector?.length) {
338-
store.setEmbedding(row.id, vector);
339-
indexed++;
340-
}
341-
} catch { /* skip the bad row */ }
342-
}
343-
done += rows.length;
344-
// Rate limit: space out batches so the provider is not hammered.
345-
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
346-
}
347-
if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
348-
ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
349-
})().catch((error) => {
350-
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
351-
});
360+
// Issue #128: no fingerprint gate here anymore — a matching fingerprint
361+
// used to return early and permanently orphan rows whose embed failed
362+
// at write time. See backfillMissingEmbeddings().
363+
backfillMissingEmbeddings({ store, embedder, vectorIndex, logger: ctx.logger })
364+
.catch((error) => {
365+
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
366+
});
352367
} catch (error) {
353368
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
354369
}

‎dsh-mneme/lib/store.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1031,11 +1031,18 @@ export function createStore(path) {
10311031
).get().c;
10321032
}
10331033

1034-
/** Candidate rows still missing an embedding, for incremental re-indexing. */
1034+
/**
1035+
* Candidate rows still missing an embedding, for incremental re-indexing.
1036+
* Issue #128: active rows only — archived/forgotten rows never participate in
1037+
* recall or dream clustering, so backfill quota must not be eaten by them
1038+
* (report measured 25/50 reindex slots landing on archived rows while 88
1039+
* active rows stayed un-embedded). Same 口径 as embeddedCount()/count().
1040+
*/
10351041
function needsEmbedding(limit = 50) {
10361042
return db.prepare(
10371043
`SELECT id, title, content FROM memories
1038-
WHERE embedding IS NULL OR embedding = ''
1044+
WHERE (embedding IS NULL OR embedding = '')
1045+
AND forgotten = 0 AND archived = 0
10391046
ORDER BY updated_at DESC LIMIT ?`
10401047
).all(limit);
10411048
}

‎dsh-mneme/src/index.js‎

Lines changed: 49 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,44 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
8787
};
8888
}
8989

90+
/**
91+
* Issue #128: bounded backfill of rows still missing an embedding (active rows
92+
* only — needsEmbedding filters archived/forgotten). Exported for tests.
93+
*
94+
* Runs regardless of the model fingerprint: the old call-site gate returned
95+
* early when vector_meta already held the embedder's hash, permanently
96+
* orphaning rows whose embed failed at write time (embedder not ready /
97+
* provider rate limit) — one successful embed was enough to never backfill
98+
* again. markModel is idempotent when the fingerprint already matches, so
99+
* re-running costs nothing beyond the actually-missing rows.
100+
*/
101+
export async function backfillMissingEmbeddings({
102+
store, embedder, vectorIndex, logger,
103+
maxTotal = 500, batchSize = 10, rateLimitMs = 200
104+
}) {
105+
let indexed = 0;
106+
for (let done = 0; done < maxTotal;) {
107+
const rows = store.needsEmbedding(batchSize);
108+
if (!rows.length) break;
109+
for (const row of rows) {
110+
try {
111+
const text = [row.title, row.content].filter(Boolean).join("\n");
112+
const vector = await embedder.embedSingle(text);
113+
if (vector?.length) {
114+
store.setEmbedding(row.id, vector);
115+
indexed++;
116+
}
117+
} catch { /* skip the bad row */ }
118+
}
119+
done += rows.length;
120+
// Rate limit: space out batches so the provider is not hammered.
121+
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
122+
}
123+
if (indexed > 0 && embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
124+
logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
125+
return indexed;
126+
}
127+
90128
export const apply = (ctx, config) => {
91129
const rawCfg = Config(config);
92130

@@ -303,10 +341,10 @@ export const apply = (ctx, config) => {
303341

304342
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
305343
// is configured and rows still lack an embedding (e.g. written before vector
306-
// search was enabled) AND the vector_meta fingerprint is absent or stale, the
307-
// index is rebuilt in the background after a short delay. Gated on
308-
// cfg.autoReindexOnBoot; rate-limited in small batches so a large backlog
309-
// never floods the provider. Failures degrade silently — search stays keyword.
344+
// search was enabled), the backfill runs in the background after a short
345+
// delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
346+
// large backlog never floods the provider. Failures degrade silently —
347+
// search stays keyword.
310348
function scheduleAutoReindex() {
311349
if (cfg.autoReindexOnBoot === false) return;
312350
const attempt = (tries) => {
@@ -319,36 +357,13 @@ export const apply = (ctx, config) => {
319357
return;
320358
}
321359
if (!store.needsEmbedding(1).length) return; // nothing to backfill
322-
// Model fingerprint gate: vectors already produced by the same model
323-
// mean there is no drift and no rebuild needed.
324-
const current = embedder.modelHash;
325-
if (current && vectorIndex.modelHash?.() === current) return;
326-
const BATCH = 10;
327-
const MAX_TOTAL = 500; // bound boot-time work
328-
(async () => {
329-
let indexed = 0;
330-
for (let done = 0; done < MAX_TOTAL;) {
331-
const rows = store.needsEmbedding(BATCH);
332-
if (!rows.length) break;
333-
for (const row of rows) {
334-
try {
335-
const text = [row.title, row.content].filter(Boolean).join("\n");
336-
const vector = await embedder.embedSingle(text);
337-
if (vector?.length) {
338-
store.setEmbedding(row.id, vector);
339-
indexed++;
340-
}
341-
} catch { /* skip the bad row */ }
342-
}
343-
done += rows.length;
344-
// Rate limit: space out batches so the provider is not hammered.
345-
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, 200));
346-
}
347-
if (indexed > 0 && current) vectorIndex.markModel?.(current, embedder.dimension);
348-
ctx.logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
349-
})().catch((error) => {
350-
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
351-
});
360+
// Issue #128: no fingerprint gate here anymore — a matching fingerprint
361+
// used to return early and permanently orphan rows whose embed failed
362+
// at write time. See backfillMissingEmbeddings().
363+
backfillMissingEmbeddings({ store, embedder, vectorIndex, logger: ctx.logger })
364+
.catch((error) => {
365+
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
366+
});
352367
} catch (error) {
353368
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
354369
}

‎dsh-mneme/src/store.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1031,11 +1031,18 @@ export function createStore(path) {
10311031
).get().c;
10321032
}
10331033

1034-
/** Candidate rows still missing an embedding, for incremental re-indexing. */
1034+
/**
1035+
* Candidate rows still missing an embedding, for incremental re-indexing.
1036+
* Issue #128: active rows only — archived/forgotten rows never participate in
1037+
* recall or dream clustering, so backfill quota must not be eaten by them
1038+
* (report measured 25/50 reindex slots landing on archived rows while 88
1039+
* active rows stayed un-embedded). Same 口径 as embeddedCount()/count().
1040+
*/
10351041
function needsEmbedding(limit = 50) {
10361042
return db.prepare(
10371043
`SELECT id, title, content FROM memories
1038-
WHERE embedding IS NULL OR embedding = ''
1044+
WHERE (embedding IS NULL OR embedding = '')
1045+
AND forgotten = 0 AND archived = 0
10391046
ORDER BY updated_at DESC LIMIT ?`
10401047
).all(limit);
10411048
}
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import test from "node:test";
2+
import assert from "node:assert/strict";
3+
import { backfillMissingEmbeddings } from "../src/index.js";
4+
import { createStore } from "../src/store.js";
5+
import { createService } from "../src/service.js";
6+
import { createVectorIndex } from "../src/vector-index.js";
7+
8+
// Issue #128:boot 回填曾被模型指纹门短路——任意一条嵌入成功写入 vector_meta
9+
// 指纹后,指纹匹配的每次启动都在门上直接 return,写入时嵌入失败(embedder 未
10+
// 就绪 / 服务商限流)的活跃行从此永远补不上(报告实测 88 条活跃记忆长期缺失,
11+
// /vector-reindex 的配额还被归档死行吃掉)。
12+
13+
test("Issue #128: fingerprint match no longer blocks backfill; archived rows are skipped", async () => {
14+
const store = createStore(":memory:");
15+
const service = createService({ store, mirror: null, config: {} });
16+
const active = service.saveWithDedupe({ type: "preference", title: "活跃", content: "内容" }).memory;
17+
const dead = service.saveWithDedupe({ type: "preference", title: "死行", content: "内容" }).memory;
18+
store.setArchived(dead.id, true);
19+
20+
// 指纹已与 embedder 一致 —— 旧的早退条件在此成立,旧实现会整场不回填。
21+
const vectorIndex = createVectorIndex({ store });
22+
vectorIndex.markModel("mock-model#abc", 3);
23+
const embedded = [];
24+
const embedder = {
25+
embedSingle: async (text) => { embedded.push(text); return [1, 0, 0]; },
26+
modelHash: "mock-model#abc",
27+
dimension: 3
28+
};
29+
30+
const logs = [];
31+
const indexed = await backfillMissingEmbeddings({
32+
store, embedder, vectorIndex,
33+
logger: { info: (m) => logs.push(m) },
34+
rateLimitMs: 0
35+
});
36+
37+
assert.equal(indexed, 1, "the active row is backfilled despite the matching fingerprint");
38+
assert.equal(store.getEmbeddings([active.id]).get(active.id)?.length, 3, "active row now carries a vector");
39+
assert.equal(store.getEmbeddings([dead.id]).get(dead.id), undefined, "archived row is skipped");
40+
assert.equal(vectorIndex.modelHash(), "mock-model#abc", "fingerprint unchanged (markModel idempotent)");
41+
assert.ok(logs.some((m) => m.includes("backfilled 1")), "boot log records the backfill");
42+
store.close();
43+
});
44+
45+
test("Issue #128: backfill is bounded by maxTotal; the rest stays queued", async () => {
46+
const store = createStore(":memory:");
47+
const service = createService({ store, mirror: null, config: {} });
48+
for (let i = 0; i < 5; i++) {
49+
service.saveWithDedupe({ type: "preference", title: `记忆${i}`, content: `内容${i}` });
50+
}
51+
const embedder = { embedSingle: async () => [1, 0, 0], modelHash: "h#1", dimension: 3 };
52+
const vectorIndex = createVectorIndex({ store });
53+
54+
const indexed = await backfillMissingEmbeddings({
55+
store, embedder, vectorIndex,
56+
logger: { info() {} },
57+
maxTotal: 4, batchSize: 2, rateLimitMs: 0
58+
});
59+
assert.equal(indexed, 4, "stops at the maxTotal bound");
60+
assert.equal(store.needsEmbedding(10).length, 1, "remaining rows stay queued for the next boot");
61+
store.close();
62+
});

‎dsh-mneme/test/store.test.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,3 +367,17 @@ test("conflict_pending table persists across store reopen", () => {
367367
assert.equal(pending[0].reason, "x");
368368
s2.close();
369369
});
370+
371+
// Issue #128:回填候选只取活跃行——归档/遗忘行不参与召回与聚类,不该吃掉
372+
// 回填配额(报告实测 50 个 reindex 名额全部落在归档行,88 条活跃记忆永远轮不到)。
373+
test("Issue #128: needsEmbedding skips archived and forgotten rows", () => {
374+
const store = openMemory();
375+
const active = store.save({ type: "preference", title: "活跃", content: "内容" });
376+
const archived = store.save({ type: "preference", title: "归档", content: "内容" });
377+
const forgotten = store.save({ type: "preference", title: "遗忘", content: "内容" });
378+
store.setArchived(archived.id, true);
379+
store.setForget(forgotten.id, true);
380+
const ids = store.needsEmbedding(50).map((r) => r.id);
381+
assert.deepEqual(ids, [active.id], "only the active row is a backfill candidate");
382+
store.close();
383+
});

0 commit comments

Comments
 (0)