Skip to content

Commit c71228d

Browse files
committed
fix(dedupe): candidates were the type's top-100 by importance, not the same-title rows
`saveWithDedupe` looked up merge targets with `store.list({ type, limit: 100 })`, whose default order is `importance DESC, updated_at DESC, id`. That window is "the 100 most important rows of this type", which has nothing to do with "find the row with the same title". Once a type held more than 100 rows with importance above the incoming row's, both the incoming row and its same-title row fell outside the window, the `find(...)` never matched, and the write silently took the create branch: no error, no warning, merge semantics degrading as the library grows. Reported on a 2200-row library where one type held 1047 rows (180 of them importance=5): writing the same title twice at importance=3 returned `created`. Add `store.activeByTypeAndTitle(type, title)` — a directed query on `type = ? AND TRIM(title) = ?` with the same active-row filter as `list()`'s defaults (`archived = 0 AND forgotten = 0`). TRIM is required: `title` is stored verbatim (see `save()`), while the comparison in `saveWithDedupe` is `m.title.trim()`, so an exact match would miss stored titles carrying surrounding whitespace and reproduce the same silent degradation. The dream and narrative summary branches key on `source` (and `tags[0]`) rather than title — their dedupe key deliberately excludes the title because it changes with `memory.language`. Pointing those at same-title candidates would pin them to a permanent miss, so they keep a type-scoped candidate set; it is now unbounded instead of capped at 100, which is what those branches needed all along (they must scan the whole type to find the same-source row). Tests in test/dedupe-candidate-window.test.js reproduce the window overflow, and fail 3/5 on the pre-fix code. Signed-off-by: modusensus <[email protected]>
1 parent cecd2b5 commit c71228d

5 files changed

Lines changed: 244 additions & 6 deletions

File tree

‎dsh-mneme/lib/service.js‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1368,7 +1368,24 @@ export function createService({ store, mirror, config, onWrite, logger, document
13681368
// dream 总览按 source 身份去重,而非本地化标题:memory.language 切换后
13691369
// 标题变化,若仍按 (type, title) 匹配会出现两个活跃的 summary 同时注入。
13701370
// 其余类型维持 (type, title) 匹配不变。
1371-
const candidates = store.list({ type: memory.type, limit: 100 });
1371+
//
1372+
// Issue #505:候选集此前是 store.list({ type, limit: 100 }) —— 那是「该类型
1373+
// 最重要的 100 行」(order 默认 importance),与「找出同标题的那一行」无关。
1374+
// 某类型里 importance 高于待写行的行数一旦超过 100,同标题行与待写行双双落在
1375+
// 窗口外,下面的 find 恒不命中,于是静默走创建分支:不报错、不告警,合并语义
1376+
// 随库增长而失效(实测 1047 行类型、importance=5 有 180 行时稳定复现)。
1377+
// 改为按同标题定向取(store.activeByTypeAndTitle,无窗口)。
1378+
const candidates = store.activeByTypeAndTitle(memory.type, memory.title);
1379+
// dream/narrative 两条分支按 source(及 tags[0])匹配,而不是按标题:它们的
1380+
// 去重键本来就不含标题(语言切换后标题会变)。定向取同标题会把这两条路径
1381+
// 打回「永远不命中」——所以这两类仍走「该类型活跃行」的候选集,只是窗口换成
1382+
// 了无界且不带 importance 排序。类型内行数由 summary 的生产方(autoDream /
1383+
// generateNarratives)决定,量级远小于普通记忆,且这两条分支本来就要求扫全
1384+
// 类型才能找到同 source 的那一行——原来的 100 行窗口对它们同样是错的。
1385+
const sourceScopedCandidates = (memory.type === "summary" && (memory.source === "dream" || memory.source === "narrative"))
1386+
? store.list({ type: memory.type, limit: null })
1387+
: candidates;
1388+
const dreamCandidates = sourceScopedCandidates;
13721389
// v0.8.0 A1(issue #17):去重键由 (type, title) 扩展为
13731390
// (type, title, agent_scope, workspace_scope, sensitivity)——不同作用域的同
13741391
// 标题记忆绝不互相物理合并(旧逻辑跨 agent/workspace 并行,事后无法拆分)。
@@ -1390,13 +1407,13 @@ export function createService({ store, mirror, config, onWrite, logger, document
13901407
const existing = explicitTarget
13911408
? (scopeMatches(explicitTarget) ? explicitTarget : undefined)
13921409
: (memory.type === "summary" && memory.source === "dream")
1393-
? (candidates.find((m) => m.source === "dream" && scopeMatches(m))
1410+
? (dreamCandidates.find((m) => m.source === "dream" && scopeMatches(m))
13941411
?? candidates.find((m) => scopeMatches(m) && m.title.trim() === String(memory.title).trim()))
13951412
: (memory.type === "summary" && memory.source === "narrative")
13961413
// 叙述条(#164)按主题键去重而非本地化标题:标题随语言变
13971414
// (叙述:X / Narrative: X),title 匹配在语言切换后会残留同 tag 双行。
13981415
// tag 存于 tags[0](generateNarratives 落库),跨语言稳定 → 原地刷新。
1399-
? (candidates.find((m) => m.source === "narrative" && m.tags?.[0] === memory.tags?.[0] && scopeMatches(m))
1416+
? (dreamCandidates.find((m) => m.source === "narrative" && m.tags?.[0] === memory.tags?.[0] && scopeMatches(m))
14001417
?? candidates.find((m) => scopeMatches(m) && m.title.trim() === String(memory.title).trim()))
14011418
: candidates.find((m) => scopeMatches(m) && m.title.trim() === String(memory.title).trim());
14021419
if (existing) {

‎dsh-mneme/lib/store.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,6 +1645,30 @@ export function createStore(path) {
16451645
return rows.map(toRow);
16461646
}
16471647

1648+
// Issue #505:写路径的去重候选必须「按同标题定向取」,不能「按 importance 取
1649+
// 前 N 条」。后者是「该类型最重要的 100 行」,与「找出同标题的那一行」无关:
1650+
// 某类型里 importance 高于待写行的行数一旦超过窗口,同标题行与待写行双双落在
1651+
// 窗口外,saveWithDedupe 的 find 恒不命中,于是静默走创建分支——不报错、不告
1652+
// 警,合并语义随库增长而失效(报告实测 1047 行类型、importance=5 有 180 行时
1653+
// 稳定复现)。
1654+
//
1655+
// 口径刻意与 saveWithDedupe 的比较式对齐:那边用 m.title.trim() 比
1656+
// String(memory.title).trim(),而 title 入库时并不 trim(见 save() 的
1657+
// `memory.title` 直写)⇒ 这里必须 TRIM(title),否则存量里带首尾空白的标题
1658+
// 会被精确匹配漏掉,退回同一种静默降级。
1659+
//
1660+
// archived/forgotten 的过滤与 list() 默认口径一致(两者都为 0):合并目标必须
1661+
// 是活跃行,否则会把内容并进一条已经归档/遗忘的记忆里。
1662+
// 无 LIMIT:同 (type, title) 的活跃行本就极少,加窗口等于把 #505 重新引进来。
1663+
function activeByTypeAndTitle(type, title) {
1664+
const rows = db.prepare(
1665+
`SELECT ${memoryColumnList} FROM memories
1666+
WHERE type = ? AND TRIM(title) = ?
1667+
AND forgotten = 0 AND archived = 0`
1668+
).all(type, String(title ?? "").trim());
1669+
return rows.map(toRow);
1670+
}
1671+
16481672
// Issue #430:写路径要的「最近若干条正文」。质量过滤的近重复判定只读 content
16491673
// 这一列 + 前 20 行,此前写成 all().slice(0, 20) —— 全表逐行 toRow(27 字段、
16501674
// 三次 JSON 解析)之后只截前 20 条正文;5k 行实测 100ms 级 → 0.06ms 级。
@@ -3355,6 +3379,9 @@ export function createStore(path) {
33553379
getUnrecalledSince,
33563380
list,
33573381
all,
3382+
// Issue #505:写路径去重的定向候选(按 type + TRIM(title) 精确取活跃行),
3383+
// 替代「importance 前 N 条」那个会随库增长失效的窗口。
3384+
activeByTypeAndTitle,
33583385
// #430:有界单列读,写路径(质量过滤)用。
33593386
recentContents,
33603387
search,

‎dsh-mneme/src/service.js‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1368,7 +1368,24 @@ export function createService({ store, mirror, config, onWrite, logger, document
13681368
// dream 总览按 source 身份去重,而非本地化标题:memory.language 切换后
13691369
// 标题变化,若仍按 (type, title) 匹配会出现两个活跃的 summary 同时注入。
13701370
// 其余类型维持 (type, title) 匹配不变。
1371-
const candidates = store.list({ type: memory.type, limit: 100 });
1371+
//
1372+
// Issue #505:候选集此前是 store.list({ type, limit: 100 }) —— 那是「该类型
1373+
// 最重要的 100 行」(order 默认 importance),与「找出同标题的那一行」无关。
1374+
// 某类型里 importance 高于待写行的行数一旦超过 100,同标题行与待写行双双落在
1375+
// 窗口外,下面的 find 恒不命中,于是静默走创建分支:不报错、不告警,合并语义
1376+
// 随库增长而失效(实测 1047 行类型、importance=5 有 180 行时稳定复现)。
1377+
// 改为按同标题定向取(store.activeByTypeAndTitle,无窗口)。
1378+
const candidates = store.activeByTypeAndTitle(memory.type, memory.title);
1379+
// dream/narrative 两条分支按 source(及 tags[0])匹配,而不是按标题:它们的
1380+
// 去重键本来就不含标题(语言切换后标题会变)。定向取同标题会把这两条路径
1381+
// 打回「永远不命中」——所以这两类仍走「该类型活跃行」的候选集,只是窗口换成
1382+
// 了无界且不带 importance 排序。类型内行数由 summary 的生产方(autoDream /
1383+
// generateNarratives)决定,量级远小于普通记忆,且这两条分支本来就要求扫全
1384+
// 类型才能找到同 source 的那一行——原来的 100 行窗口对它们同样是错的。
1385+
const sourceScopedCandidates = (memory.type === "summary" && (memory.source === "dream" || memory.source === "narrative"))
1386+
? store.list({ type: memory.type, limit: null })
1387+
: candidates;
1388+
const dreamCandidates = sourceScopedCandidates;
13721389
// v0.8.0 A1(issue #17):去重键由 (type, title) 扩展为
13731390
// (type, title, agent_scope, workspace_scope, sensitivity)——不同作用域的同
13741391
// 标题记忆绝不互相物理合并(旧逻辑跨 agent/workspace 并行,事后无法拆分)。
@@ -1390,13 +1407,13 @@ export function createService({ store, mirror, config, onWrite, logger, document
13901407
const existing = explicitTarget
13911408
? (scopeMatches(explicitTarget) ? explicitTarget : undefined)
13921409
: (memory.type === "summary" && memory.source === "dream")
1393-
? (candidates.find((m) => m.source === "dream" && scopeMatches(m))
1410+
? (dreamCandidates.find((m) => m.source === "dream" && scopeMatches(m))
13941411
?? candidates.find((m) => scopeMatches(m) && m.title.trim() === String(memory.title).trim()))
13951412
: (memory.type === "summary" && memory.source === "narrative")
13961413
// 叙述条(#164)按主题键去重而非本地化标题:标题随语言变
13971414
// (叙述:X / Narrative: X),title 匹配在语言切换后会残留同 tag 双行。
13981415
// tag 存于 tags[0](generateNarratives 落库),跨语言稳定 → 原地刷新。
1399-
? (candidates.find((m) => m.source === "narrative" && m.tags?.[0] === memory.tags?.[0] && scopeMatches(m))
1416+
? (dreamCandidates.find((m) => m.source === "narrative" && m.tags?.[0] === memory.tags?.[0] && scopeMatches(m))
14001417
?? candidates.find((m) => scopeMatches(m) && m.title.trim() === String(memory.title).trim()))
14011418
: candidates.find((m) => scopeMatches(m) && m.title.trim() === String(memory.title).trim());
14021419
if (existing) {

‎dsh-mneme/src/store.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,6 +1645,30 @@ export function createStore(path) {
16451645
return rows.map(toRow);
16461646
}
16471647

1648+
// Issue #505:写路径的去重候选必须「按同标题定向取」,不能「按 importance 取
1649+
// 前 N 条」。后者是「该类型最重要的 100 行」,与「找出同标题的那一行」无关:
1650+
// 某类型里 importance 高于待写行的行数一旦超过窗口,同标题行与待写行双双落在
1651+
// 窗口外,saveWithDedupe 的 find 恒不命中,于是静默走创建分支——不报错、不告
1652+
// 警,合并语义随库增长而失效(报告实测 1047 行类型、importance=5 有 180 行时
1653+
// 稳定复现)。
1654+
//
1655+
// 口径刻意与 saveWithDedupe 的比较式对齐:那边用 m.title.trim() 比
1656+
// String(memory.title).trim(),而 title 入库时并不 trim(见 save() 的
1657+
// `memory.title` 直写)⇒ 这里必须 TRIM(title),否则存量里带首尾空白的标题
1658+
// 会被精确匹配漏掉,退回同一种静默降级。
1659+
//
1660+
// archived/forgotten 的过滤与 list() 默认口径一致(两者都为 0):合并目标必须
1661+
// 是活跃行,否则会把内容并进一条已经归档/遗忘的记忆里。
1662+
// 无 LIMIT:同 (type, title) 的活跃行本就极少,加窗口等于把 #505 重新引进来。
1663+
function activeByTypeAndTitle(type, title) {
1664+
const rows = db.prepare(
1665+
`SELECT ${memoryColumnList} FROM memories
1666+
WHERE type = ? AND TRIM(title) = ?
1667+
AND forgotten = 0 AND archived = 0`
1668+
).all(type, String(title ?? "").trim());
1669+
return rows.map(toRow);
1670+
}
1671+
16481672
// Issue #430:写路径要的「最近若干条正文」。质量过滤的近重复判定只读 content
16491673
// 这一列 + 前 20 行,此前写成 all().slice(0, 20) —— 全表逐行 toRow(27 字段、
16501674
// 三次 JSON 解析)之后只截前 20 条正文;5k 行实测 100ms 级 → 0.06ms 级。
@@ -3355,6 +3379,9 @@ export function createStore(path) {
33553379
getUnrecalledSince,
33563380
list,
33573381
all,
3382+
// Issue #505:写路径去重的定向候选(按 type + TRIM(title) 精确取活跃行),
3383+
// 替代「importance 前 N 条」那个会随库增长失效的窗口。
3384+
activeByTypeAndTitle,
33583385
// #430:有界单列读,写路径(质量过滤)用。
33593386
recentContents,
33603387
search,
Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
// Issue #505:saveWithDedupe 的去重候选窗口固定 100 条,库一变大就静默不再合并。
2+
//
3+
// 根因:候选集是 store.list({ type, limit: 100 }),order 默认 importance
4+
// ⇒「该类型最重要的 100 行」,与「找出同标题的那一行」无关。某类型里 importance
5+
// 高于待写行的行数一旦超过窗口,同标题行与待写行双双落在窗口外,find 恒不命中,
6+
// 于是走创建分支:不报错、不告警,合并语义随库增长而失效。
7+
//
8+
// 这组用例刻意复刻报告里的实测形状(类型 1047 行、其中 importance=5 有 180 行),
9+
// 但用更小的数字达成同样的窗口溢出,避免测试本身变慢。
10+
import test from "node:test";
11+
import assert from "node:assert/strict";
12+
import { createStore } from "../src/store.js";
13+
import { createService } from "../src/service.js";
14+
15+
function freshService() {
16+
const store = createStore(":memory:");
17+
const service = createService({ store, mirror: null, config: {} });
18+
return { store, service };
19+
}
20+
21+
// 往某个类型里灌 n 条 importance=5 的行,把窗口挤满。
22+
function crowdType(store, type, n) {
23+
for (let i = 0; i < n; i++) {
24+
store.save({
25+
type,
26+
title: `crowd-${type}-${i}`,
27+
content: `padding row ${i}`,
28+
importance: 5
29+
});
30+
}
31+
}
32+
33+
test("#505: 同类型 importance 更高的行超过窗口后,同标题第二次写入仍然合并", () => {
34+
const { store, service } = freshService();
35+
// 窗口是 100。灌 120 条 importance=5,保证待写行(importance=3)与它的同标题
36+
// 行都排不进前 100。
37+
crowdType(store, "project", 120);
38+
39+
const payload = {
40+
type: "project",
41+
title: "dedupe-target",
42+
content: "first content",
43+
importance: 3
44+
};
45+
const first = service.saveWithDedupe(payload);
46+
assert.equal(first.action, "created", "第一次写入应当是新建");
47+
48+
const second = service.saveWithDedupe({ ...payload, content: "second content" });
49+
assert.equal(
50+
second.action,
51+
"merged",
52+
"第二次同标题写入必须并入已有行;返回 created 说明候选窗口把同标题行挡在窗口外(#505 静默降级)"
53+
);
54+
assert.equal(second.memory.id, first.memory.id, "合并必须落在同一行");
55+
56+
const rows = store.list({ type: "project", limit: null })
57+
.filter((m) => m.title.trim() === "dedupe-target");
58+
assert.equal(rows.length, 1, "同标题只应存在一行");
59+
});
60+
61+
test("#505: 定向候选不受 importance 分布影响(低 importance 的同标题行也能命中)", () => {
62+
const { store, service } = freshService();
63+
// 先写一条 importance=1 的目标行,再用大量 importance=5 的行把它挤出窗口。
64+
const first = service.saveWithDedupe({
65+
type: "pattern",
66+
title: "low-importance-target",
67+
content: "seed",
68+
importance: 1
69+
});
70+
assert.equal(first.action, "created");
71+
72+
crowdType(store, "pattern", 120);
73+
74+
const second = service.saveWithDedupe({
75+
type: "pattern",
76+
title: "low-importance-target",
77+
content: "merged later",
78+
importance: 1
79+
});
80+
assert.equal(second.action, "merged", "低 importance 的同标题行同样必须能被找到");
81+
assert.equal(second.memory.id, first.memory.id);
82+
});
83+
84+
test("#505: 标题首尾空白不影响去重命中(入库不 trim,比较 trim)", () => {
85+
const { store, service } = freshService();
86+
crowdType(store, "constraint", 120);
87+
88+
const first = service.saveWithDedupe({
89+
type: "constraint",
90+
title: " spaced title ",
91+
content: "seed",
92+
importance: 3
93+
});
94+
assert.equal(first.action, "created");
95+
96+
// 同一个标题、首尾空白写法不同——按 trim 口径应当命中同一行。
97+
const second = service.saveWithDedupe({
98+
type: "constraint",
99+
title: "spaced title",
100+
content: "merged",
101+
importance: 3
102+
});
103+
assert.equal(second.action, "merged", "trim 后同标题必须合并,否则存量带空白的标题会退回静默降级");
104+
assert.equal(second.memory.id, first.memory.id);
105+
});
106+
107+
test("#505: 定向候选不跨类型命中", () => {
108+
const { store, service } = freshService();
109+
crowdType(store, "project", 120);
110+
111+
const a = service.saveWithDedupe({
112+
type: "project",
113+
title: "same-title-different-type",
114+
content: "in project",
115+
importance: 3
116+
});
117+
const b = service.saveWithDedupe({
118+
type: "preference",
119+
title: "same-title-different-type",
120+
content: "in preference",
121+
importance: 3
122+
});
123+
assert.equal(a.action, "created");
124+
assert.equal(b.action, "created", "不同 type 的同标题行不得合并");
125+
assert.notEqual(a.memory.id, b.memory.id);
126+
});
127+
128+
test("#505: 归档行不作为合并目标(口径与 list 默认一致)", () => {
129+
const { store, service } = freshService();
130+
crowdType(store, "decision", 120);
131+
132+
const first = service.saveWithDedupe({
133+
type: "decision",
134+
title: "archived-target",
135+
content: "seed",
136+
importance: 3
137+
});
138+
// 归档必须走 store.setArchived——store.update 的 SET 子句里没有 archived 列,
139+
// 用 update({archived:true}) 根本不会归档,测试会假绿。
140+
store.setArchived(first.memory.id, true);
141+
142+
const second = service.saveWithDedupe({
143+
type: "decision",
144+
title: "archived-target",
145+
content: "after archive",
146+
importance: 3
147+
});
148+
assert.equal(second.action, "created", "已归档行不该被当作活跃合并目标");
149+
assert.notEqual(second.memory.id, first.memory.id);
150+
});

0 commit comments

Comments
 (0)