Skip to content

Commit b62bf13

Browse files
modusensusmodusensus
committed
test: supersede 旧值回归集——anti-update 探针进 recall benchmark（E6/#280 链）
- TEST_CASES 支持可选 forbidden 字段，legacy/fused/三融合配方全配置判定 forbiddenHit，CLI 报告加 LEAK 标记 - seedService 复刻 applySupersede 落库状态（loser archived=1 + superseded-by 注记） - 新增 test/recall-anti-update.test.js 6 例：四路默认检索对归档行是排除不是降权；取代是归档不是删除；includeArchived 显式口子仍在 - 依据 StatemenBench anti-update probes；测试 1198 → 1204，全量 npm test 绿
1 parent b64e924 commit b62bf13

3 files changed

Lines changed: 141 additions & 7 deletions

File tree

‎dsh-mneme/CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
- **全工具矩阵的「DTO 键集 ⊆ output schema」系统性断言（issue #195）**：#184（memory_get 内联 schema 漏声明 v0.8.1 的 scope 来源三键 → 任何被标注过的行都过不了 in-process 校验）此前只有单点回归护住 `memory_get` 一个工具，换一个工具、换一个键，同类事故可以原样重演。新增 `test/tools-dto-schema-matrix.test.js`，四层断言各管一段：① 9 个工具每个可安全触达分支的**真实 execute 返回值**过生产同款校验器 `validateJsonSchemaValue`（不写手抄期望值）；② DTO 唯一产地 `toApiList` 在全形态（极简 / 敏感度 / 事件时间 / 单维与全量 scope 标注）下的输出 ⊆ `MEMORY_ITEM_SCHEMA`，并**反向**要求声明里的每个键都被至少一种形态真实产出（死声明会在下次增键时暴露）；③ 全部工具 schema 的结构不变量（闭合、required ⊆ properties、每项带 type——否则前两层会因校验器形同虚设而静默失效）；④ 负例锁：注入未声明键**必须**报错。护栏自证：两次变异测试（删共享 schema 一个键 / 给 memory_get 塞手抄小副本）分别让 2 条与 3 条断言转红。`memory_runtime` 的 provision（联网下载）与 verify 命中载荷（真实加载模型）不在单测内驱动，由 ③ 兜底声明合规。
2626

27+
- **supersede 旧值回归集（anti-update 探针，E6/#280 链）**：`scripts/benchmark-recall.js` 的 TEST_CASES 支持可选 `forbidden` 字段——forbidden id 出现在 top-K 即记 `forbiddenHit`，legacy/fused 与三融合配方全配置判定，CLI 报告加 LEAK 标记；seed 复刻 applySupersede 的落库状态（loser `archived=1` + 正文 superseded-by 注记，无专属取代列）。新增 `test/recall-anti-update.test.js` 6 例，把三件事钉成回归锁：四路默认检索对归档行是**排除**不是降权（谁把排除改成降权立刻红）、取代是归档不是删除（行还在、未遗忘、注记在）、`includeArchived` 显式口子仍在（排除发生在检索层而非行消失）。依据 StatemenBench anti-update probes。测试 1198 → **1204**。
28+
2729
## 🏗️ 工程
2830

2931
- 致谢：heptaspirit（#247 注入命中留痕）、davidekingsss（#248 审计记账修复 + #253 审计边界测试）。

‎dsh-mneme/scripts/benchmark-recall.js‎

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ const SEED = [
5252
// Standard query set: each case is a query plus the ids that MUST appear in
5353
// the top-K for the case to count as a hit. Covers the three recall paths —
5454
// multi-term lexical (BM25's home turf), identifier lookup, and semantic.
55+
// Optional per case: `forbidden` — ids that must NOT appear in the top-K.
56+
// Anti-update probes (E6, #280 链；StatemenBench anti-update probes)：锁的
57+
// 回归类是「supersede 后旧值复现」——被取代旧条靠 archived=1 从四路检索里
58+
// **排除**（不是降权）。未来谁把任一默认检索路径改成 archived 宽容的降权、
59+
// 或把归档行重新放回候选池，forbiddenHit 就会 >0，这里先红。
5560
export const TEST_CASES = [
5661
{ query: "rust 异步", expected: ["mem_async_pattern", "mem_rust_switch"], note: "scattered terms — BM25 territory" },
5762
{ query: "ZFS-4421 checksum", expected: ["mem_zfs_bug"], note: "identifier + keyword" },
@@ -67,7 +72,12 @@ export const TEST_CASES = [
6772
// leading token with a cross-topic memory that must rank below it) and an
6873
// exact-token case that leans on the BM25 path.
6974
{ query: "zfs 磁盘 替换", expected: ["mem_zfs_bug"], note: "shared-token distractor" },
70-
{ query: "tokio spawn channel", expected: ["mem_async_pattern"], note: "exact async tokens" }
75+
{ query: "tokio spawn channel", expected: ["mem_async_pattern"], note: "exact async tokens" },
76+
// Anti-update probes（配 seedService 里的 supersede pair）：第二条 query 的
77+
// 措辞只与被取代的旧值（Go 选型）重叠——归档旧条若经任一默认路径复现，
78+
// forbiddenHit > 0；同时新值必须照常命中（取代后话题有活答案）。
79+
{ query: "编译模块 语言 选型", expected: ["mem_rust_switch"], forbidden: ["mem_rust_switch_old"], note: "anti-update: winner must answer the topic" },
80+
{ query: "Go 部署 简单", expected: ["mem_rust_switch"], forbidden: ["mem_rust_switch_old"], note: "stale-value bait: wording matches only the superseded value" }
7181
];
7282

7383
export function seedService(overrides = {}) {
@@ -95,6 +105,15 @@ export function seedService(overrides = {}) {
95105
const row = store.save({ id: m.id, type: m.type, title: m.title, content: m.content, tags: m.tags, importance: m.importance, source: "seed" });
96106
store.setEmbedding(row.id, hashVec(`${m.title} ${m.content}`));
97107
}
108+
// Supersede pair（anti-update 探针的种子）：复刻 dream/decisions.js
109+
// applySupersede 留下的确切落库状态——没有独立的取代列，取代关系完全由
110+
// loser 的 archived=1 + 正文 (superseded by: …) 注记表达。winner 就是
111+
// 上面的 mem_rust_switch 种子行，让「编译语言」话题始终有活答案，
112+
// 归档的 Go 旧决策必须保持不可见。
113+
const loserContent = "项目编译模块使用 Go 语言，选型理由是部署简单\n\n(superseded by: 语言迁移决策)";
114+
const loser = store.save({ id: "mem_rust_switch_old", type: "decision", title: "编译模块语言（旧）", content: loserContent, importance: 4, tags: ["rust"], source: "seed" });
115+
store.setEmbedding(loser.id, hashVec(`编译模块语言（旧） ${loserContent}`));
116+
store.setArchived(loser.id, true);
98117
return service;
99118
}
100119

@@ -109,18 +128,31 @@ export async function runBenchmark({ topK = 5, mode = "auto" } = {}) {
109128
const rows = [];
110129
let hits = 0;
111130
let mrrSum = 0;
131+
let leaks = 0;
112132
for (const tc of TEST_CASES) {
113133
const results = await service.searchMemories(tc.query, { mode, topK, useRerank: false });
114134
const ids = results.map((r) => r.id);
115135
const metrics = service.computeRetrievalMetrics(ids, tc.expected);
116136
if (metrics.recall === 1) hits++;
117137
mrrSum += metrics.mrr;
118-
rows.push({ query: tc.query, note: tc.note, expected: tc.expected, got: ids, ...metrics });
138+
// forbidden 命中数：anti-update 探针的判定（见 TEST_CASES 头注释）。
139+
const forbiddenHit = (tc.forbidden ?? []).filter((id) => ids.includes(id)).length;
140+
if (forbiddenHit > 0) leaks++;
141+
rows.push({
142+
query: tc.query,
143+
note: tc.note,
144+
expected: tc.expected,
145+
...(tc.forbidden ? { forbidden: tc.forbidden } : {}),
146+
forbiddenHit,
147+
got: ids,
148+
...metrics
149+
});
119150
}
120151
runs.push({
121152
config: cfg.name,
122153
recallAtK: +(hits / TEST_CASES.length).toFixed(3),
123154
avgMrr: +(mrrSum / TEST_CASES.length).toFixed(3),
155+
forbiddenHits: leaks,
124156
rows
125157
});
126158
}
@@ -141,18 +173,30 @@ export async function runFusionBenchmark({ topK = 5, mode = "auto" } = {}) {
141173
const rows = [];
142174
let hits = 0;
143175
let mrrSum = 0;
176+
let leaks = 0;
144177
for (const tc of TEST_CASES) {
145178
const results = await service.searchMemories(tc.query, { mode, topK, useRerank: false });
146179
const ids = results.map((r) => r.id);
147180
const metrics = service.computeRetrievalMetrics(ids, tc.expected);
148181
if (metrics.recall === 1) hits++;
149182
mrrSum += metrics.mrr;
150-
rows.push({ query: tc.query, note: tc.note, expected: tc.expected, got: ids, ...metrics });
183+
const forbiddenHit = (tc.forbidden ?? []).filter((id) => ids.includes(id)).length;
184+
if (forbiddenHit > 0) leaks++;
185+
rows.push({
186+
query: tc.query,
187+
note: tc.note,
188+
expected: tc.expected,
189+
...(tc.forbidden ? { forbidden: tc.forbidden } : {}),
190+
forbiddenHit,
191+
got: ids,
192+
...metrics
193+
});
151194
}
152195
runs.push({
153196
config: recipe,
154197
recallAtK: +(hits / TEST_CASES.length).toFixed(3),
155198
avgMrr: +(mrrSum / TEST_CASES.length).toFixed(3),
199+
forbiddenHits: leaks,
156200
rows
157201
});
158202
}
@@ -164,10 +208,13 @@ function printFusionReport(report) {
164208
console.log(`\n=== ${run.config} (topK=${report.topK}, mode=${report.mode}) ===`);
165209
for (const r of run.rows) {
166210
const ok = r.recall === 1 ? "PASS" : "MISS";
167-
console.log(` [${ok}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
211+
const leak = r.forbiddenHit > 0 ? " LEAK" : "";
212+
console.log(` [${ok}${leak}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
168213
if (r.recall < 1) console.log(` expected ⊇ ${r.expected.join(", ")} got: ${r.got.join(", ") || "—"}`);
214+
if (r.forbiddenHit > 0) console.log(` forbidden hit ×${r.forbiddenHit}: ${r.got.filter((id) => r.forbidden.includes(id)).join(", ")}`);
169215
}
170-
console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}`);
216+
console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}` +
217+
(run.forbiddenHits > 0 ? ` forbiddenHits: ${run.forbiddenHits}` : ""));
171218
}
172219
const summary = report.runs.map((r) => `${r.config}=${(r.recallAtK * 100).toFixed(1)}%`).join(" ");
173220
console.log(`\n融合配方 A/B (Recall@${report.topK}, ${report.mode}): ${summary}`);
@@ -178,10 +225,13 @@ function printReport(report) {
178225
console.log(`\n=== ${run.config} (topK=${report.topK}, mode=${report.mode}) ===`);
179226
for (const r of run.rows) {
180227
const ok = r.recall === 1 ? "PASS" : "MISS";
181-
console.log(` [${ok}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
228+
const leak = r.forbiddenHit > 0 ? " LEAK" : "";
229+
console.log(` [${ok}${leak}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
182230
if (r.recall < 1) console.log(` expected ⊇ ${r.expected.join(", ")} got: ${r.got.join(", ") || "—"}`);
231+
if (r.forbiddenHit > 0) console.log(` forbidden hit ×${r.forbiddenHit}: ${r.got.filter((id) => r.forbidden.includes(id)).join(", ")}`);
183232
}
184-
console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}`);
233+
console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}` +
234+
(run.forbiddenHits > 0 ? ` forbiddenHits: ${run.forbiddenHits}` : ""));
185235
}
186236
const [legacy, fused] = report.runs;
187237
const lift = ((fused.recallAtK - legacy.recallAtK) * 100).toFixed(1);

‎dsh-mneme/test/recall-anti-update.test.js‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Anti-update 回归集（E6，#280 链；StatemenBench anti-update probes）。
2+
//
3+
// 锁的回归类：**supersede 后旧值复现**。取代关系没有专属列——落库状态完全
4+
// 由 applySupersede（dream/decisions.js）的产物表达：loser 的 archived=1 +
5+
// 正文 (superseded by: …) 注记。四条默认检索路径各自过滤 archived（keyword
6+
// store.js:1528、vector store.js:1564、BM25 池 service.js bm25Recall、entity
7+
// service.js:470），本文件把它们钉成「排除，而非降权」：
8+
// - 任何一路让归档旧条回到候选集 → 红；
9+
// - 归档行被误删/误遗忘（取代是归档不是删除）→ 红；
10+
// - includeArchived 显式口子仍在（排除发生在检索层，不是行消失）→ 红。
11+
// seedService 已种好取代对（winner = mem_rust_switch，loser = mem_rust_switch_old）。
12+
import test from "node:test";
13+
import assert from "node:assert/strict";
14+
import { runBenchmark, runFusionBenchmark, seedService } from "../scripts/benchmark-recall.js";
15+
16+
const WINNER = "mem_rust_switch";
17+
const LOSER = "mem_rust_switch_old";
18+
19+
test("benchmark 里带 forbidden 的 anti-update 用例在 legacy/fused 两配置下都零泄露", async () => {
20+
const report = await runBenchmark({ topK: 5 });
21+
for (const run of report.runs) {
22+
const probes = run.rows.filter((r) => r.forbidden?.length);
23+
assert.ok(probes.length >= 2, `${run.config}: anti-update probes present`);
24+
for (const r of probes) {
25+
assert.equal(r.forbiddenHit, 0, `${run.config} "${r.query}": superseded old row leaked`);
26+
assert.equal(r.recall, 1, `${run.config} "${r.query}": winner must still answer the topic`);
27+
}
28+
}
29+
});
30+
31+
test("fusion 三个配方同样零泄露（排除发生在融合上游，与配方无关）", async () => {
32+
const report = await runFusionBenchmark({ topK: 5 });
33+
for (const run of report.runs) {
34+
for (const r of run.rows) {
35+
if (!r.forbidden?.length) continue;
36+
assert.equal(r.forbiddenHit, 0, `${run.config} "${r.query}": leaked via fusion recipe`);
37+
}
38+
}
39+
});
40+
41+
test("mode=keyword / vector / auto 三条路径都不再返回被取代旧值", async () => {
42+
const svc = seedService();
43+
// 查询用 winner 标题「语言迁移决策」：toy 哈希嵌入按连续 CJK 段整段分词，
44+
// 只有这个串在 keyword（LIKE 子串）与 vector（token 精确重叠）两条路上都
45+
// 能命中 winner。注意 loser 的 superseded-by 注记里恰好也含这个串——
46+
// 归档排除一旦失效，注记自己就会把旧值钓回来，本用例立即红。
47+
for (const mode of ["keyword", "vector", "auto"]) {
48+
const rows = await svc.searchMemories("语言迁移决策", { mode, topK: 10, useRerank: false });
49+
const ids = rows.map((r) => r.id);
50+
assert.ok(!ids.includes(LOSER), `mode=${mode}: archived loser resurfaced`);
51+
assert.ok(ids.includes(WINNER), `mode=${mode}: winner missing from default recall`);
52+
}
53+
});
54+
55+
test("BM25 的候选池（store.list）不含归档行——降权化改动在这里最先红", async () => {
56+
const svc = seedService();
57+
// bm25Recall 的取数源就是 store.list({limit, includeForgotten:false}) 再
58+
// filter !m.archived；list 层面归档行本就不回来，这里锁「池子是干净的」。
59+
const pool = svc.list({ limit: 500, includeForgotten: false });
60+
assert.ok(pool.length > 0, "pool non-empty");
61+
assert.ok(!pool.some((m) => m.id === LOSER), "archived loser must not enter the bm25 pool");
62+
});
63+
64+
test("取代是归档不是删除：旧行还在、未遗忘、带 superseded-by 注记", async () => {
65+
const svc = seedService();
66+
const loser = svc.getById(LOSER);
67+
assert.ok(loser, "loser row must still exist");
68+
assert.equal(loser.archived, true, "loser must be archived (supersede semantics)");
69+
assert.equal(loser.forgotten, false, "supersede must not forget the row (heat/forget 是另一条线)");
70+
assert.ok(loser.content.includes("(superseded by: 语言迁移决策)"), "content carries the superseded-by note");
71+
});
72+
73+
test("includeArchived 显式口子仍在：排除发生在检索层，不是行消失", async () => {
74+
const svc = seedService();
75+
// store.search 是整串 LIKE 子串扫描（不切词），所以这里用 loser 正文的
76+
// 连续子串作探针；被取代旧条不在默认结果里、但在 includeArchived 下回来。
77+
const probe = "选型理由是部署简单";
78+
const without = svc.search(probe, { limit: 50 });
79+
const withArchived = svc.search(probe, { limit: 50, includeArchived: true });
80+
assert.ok(!without.some((m) => m.id === LOSER), "default search excludes the archived loser");
81+
assert.ok(withArchived.some((m) => m.id === LOSER), "includeArchived must still surface it (row exists)");
82+
});

0 commit comments

Comments
 (0)