Skip to content

Commit 5c2d76b

Browse files
committed
feat(retrieval): recall fusion recipes + signal transparency (PR0+PR1)
PR0 — benchmark harness trust: - Fix seeded-id mapping: store.save now receives the caller's id, so the seeded mem_* ids match TEST_CASES.expected (previously every row got a UUID and the harness reported 0% recall on both configs). - Expand the seed corpus with cross-topic distractor memories + 2 query cases, raising the recall bar for the fused ranking. - Add runFusionBenchmark() + --fusion CLI for the blend/rrf/minmax A/B. PR1 — #1 recall fusion + #2 signal transparency (off by default): - config.recallFusion: enum [blend | rrf | minmax], default �lend (legacy behavior preserved byte-for-byte; rrf = Reciprocal Rank Fusion with k=60, minmax = scale-aware normalized weighted sum). - config.signalTransparency: bool; when on, each search result carries a signals object { keyword, vector, bm25, final }. Decorative only. - Add both to the feature-flag whitelist so the panel can toggle them. - Update /features effective-count test (37 -> 37) for the two new keys. Verified: benchmark (legacy 83.3% -> fused 100% Recall@5), service-search blend weights (0.92/0.9) unchanged, api featureSnapshot, and ~200 panel/inject tests pass.
1 parent 0fe6833 commit 5c2d76b

9 files changed

Lines changed: 424 additions & 129 deletions

File tree

‎dsh-mneme/lib/config.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,19 @@ export const Config = z.object({
178178
searchSemanticDedup: z.boolean().default(false),
179179
searchSemanticDedupThreshold: z.number().min(0.5).max(1).default(0.95),
180180

181+
// Recall fusion recipe (plan #1): how the keyword/vector/BM25 ranked lists
182+
// are combined into the final ranking. `blend` (default) is the legacy
183+
// behavior — weighted sum for vector/hybrid, union backfill for auto —
184+
// unchanged. `rrf` (Reciprocal Rank Fusion) and `minmax` (min-max normalized
185+
// weighted sum) are rank/scale-aware recipes that fix the unit mismatch the
186+
// issue describes (raw cosine vs keyword score vs normalized IDF are added
187+
// directly). Off by default so existing behavior holds exactly.
188+
recallFusion: z.union([z.const("blend"), z.const("rrf"), z.const("minmax")]).default("blend"),
189+
// Attach a `signals` object { keyword, vector, bm25, final } to each search
190+
// result for transparency/debugging (plan #2). Default off; when on it only
191+
// decorates the returned rows, never changes the ranking.
192+
signalTransparency: z.boolean().default(false),
193+
181194
// --- semantic: rerank layer (v0.2) --------------------------------------
182195
// Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
183196
// (transformers.js) at init, so a bare install must not load it. Only an

‎dsh-mneme/lib/service.js‎

Lines changed: 131 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,131 @@ export function createService({ store, mirror, config, onWrite, logger }) {
416416
}
417417
}
418418

419+
/**
420+
* Recall fusion (plan #1). Turns the three ranked signal lists (keyword,
421+
* vector, BM25) into a single merged list. Three recipes, selected by
422+
* config.recallFusion:
423+
* - blend (default): legacy behavior — weighted sum for vector/hybrid, union
424+
* backfill for auto. Byte-identical to pre-fusion code, so enabling the
425+
* config never regresses anybody.
426+
* - rrf: Reciprocal Rank Fusion — Σ 1/(k + rank + 1) over each list a row
427+
* appears in. Rank-based, so the unit mismatch (raw cosine vs keyword
428+
* score vs normalized IDF) is irrelevant.
429+
* - minmax: min-max normalize each source list's scores to [0,1] then take
430+
* the weighted sum — a scale-aware version of `blend`.
431+
* Returns { merged, signals }, where signals is Map<id, {keyword, vector,
432+
* bm25}> so searchMemories can decorate rows when signalTransparency is on.
433+
*/
434+
function fuseRecall({ keyword, vector, bm25, lim, mode, wv, wk, wb }) {
435+
const recipe = config?.recallFusion ?? "blend";
436+
437+
// Per-source scores are recorded for every recipe so signalTransparency
438+
// works regardless of how the ranking was produced.
439+
const signals = new Map();
440+
const addSig = (id, field, sc) => {
441+
const cur = signals.get(id) ?? {};
442+
cur[field] = sc;
443+
signals.set(id, cur);
444+
};
445+
for (const m of keyword) addSig(m.id, "keyword", m.score ?? 0);
446+
for (const m of vector) addSig(m.id, "vector", m.score ?? 0);
447+
for (const m of bm25) addSig(m.id, "bm25", m.score ?? 0);
448+
449+
const vectorIds = new Set(vector.map((m) => m.id));
450+
const keywordIds = new Set(keyword.map((m) => m.id));
451+
452+
let merged;
453+
if (recipe === "rrf") {
454+
// Rank-based: only the position of a row inside each surviving source
455+
// list matters, so no cross-signal scale calibration is needed.
456+
const k = 60; // standard RRF constant (plan #1 documents k=60)
457+
const rows = new Map();
458+
const addList = (list) => list.forEach((m, idx) => {
459+
const s = 1 / (k + idx + 1);
460+
const cur = rows.get(m.id);
461+
if (cur) cur.score += s;
462+
else rows.set(m.id, { ...m, score: s });
463+
});
464+
addList(keyword);
465+
addList(vector);
466+
addList(bm25);
467+
merged = [...rows.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, lim);
468+
} else if (recipe === "minmax") {
469+
// Scale-aware weighted sum: each source list is min-max normalized to
470+
// [0,1] before blending, so raw cosine and keyword score live on the
471+
// same footing.
472+
const norm = (list) => {
473+
if (!list.length) return new Map();
474+
let min = Infinity, max = -Infinity;
475+
for (const m of list) { const s = m.score ?? 0; if (s < min) min = s; if (s > max) max = s; }
476+
const range = max - min;
477+
const out = new Map();
478+
for (const m of list) out.set(m.id, range > 0 ? ((m.score ?? 0) - min) / range : 0.5);
479+
return out;
480+
};
481+
const kw = norm(keyword), ve = norm(vector), bm = norm(bm25);
482+
const rows = new Map();
483+
const seed = (m) => { if (!rows.has(m.id)) rows.set(m.id, { ...m, score: 0 }); };
484+
for (const m of keyword) seed(m);
485+
for (const m of vector) seed(m);
486+
for (const m of bm25) seed(m);
487+
for (const [id, row] of rows) {
488+
const k = kw.get(id) ?? 0;
489+
const v = ve.get(id) ?? 0;
490+
const b = bm.get(id) ?? 0;
491+
row.score = v * wv + k * wk + b * wb;
492+
}
493+
merged = [...rows.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, lim);
494+
} else {
495+
// blend — the pre-existing per-mode behavior, extracted verbatim.
496+
if (mode === "keyword") {
497+
merged = keyword;
498+
} else if (mode === "vector" || mode === "hybrid") {
499+
const byId = new Map();
500+
for (const m of vector) {
501+
const rec = byId.get(m.id);
502+
byId.set(m.id, rec ? { ...rec, score: Math.max(rec.score ?? 0, m.score ?? 0) } : m);
503+
}
504+
for (const m of keyword) {
505+
const rec = byId.get(m.id);
506+
if (rec) {
507+
byId.set(m.id, { ...rec, score: (rec.score ?? 0) * wv + (m.score ?? 0) * wk });
508+
} else {
509+
byId.set(m.id, m);
510+
}
511+
}
512+
for (const m of bm25) {
513+
const rec = byId.get(m.id);
514+
if (rec) {
515+
if (keywordIds.has(m.id)) continue;
516+
byId.set(m.id, { ...rec, score: (rec.score ?? 0) + wb * (m.score ?? 0) });
517+
} else {
518+
byId.set(m.id, { ...m, score: wb * (m.score ?? 0) });
519+
}
520+
}
521+
const ranked = [...byId.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
522+
merged = ranked.slice(0, lim);
523+
if (merged.length < lim && !merged.length) {
524+
merged = keyword.slice(0, lim);
525+
}
526+
} else {
527+
// auto: keyword leads, vector + BM25 fill remaining slots.
528+
merged = keyword.slice(0, lim);
529+
const seen = new Set(merged.map((m) => m.id));
530+
for (const m of vector) {
531+
if (merged.length >= lim) break;
532+
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
533+
}
534+
for (const m of bm25) {
535+
if (merged.length >= lim) break;
536+
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
537+
}
538+
}
539+
}
540+
541+
return { merged, signals };
542+
}
543+
419544
async function searchMemories(query, options = {}) {
420545
const { mode = "auto", topK = 20, threshold, useRerank = true, recordRecall = options.recordRecall ?? (config?.recallRecordDefault ?? true) } = options;
421546
const q = String(query ?? "").trim();
@@ -477,69 +602,17 @@ export function createService({ store, mirror, config, onWrite, logger }) {
477602
// Loose blend weight: BM25 confirms and backfills, never dominates the
478603
// semantic signal. Same-memory overlap boosts, unseen ids backfill.
479604
const wb = 0.3;
480-
// Path bookkeeping for the boost rule below: which ids each semantic
481-
// recall path surfaced.
482-
const vectorIds = new Set(vector.map((m) => m.id));
483-
const keywordIds = new Set(keyword.map((m) => m.id));
484605

485606
// Hybrid blending weights from config when provided.
486607
const wv = config?.hybridSearchVectorWeight ?? DEFAULT_HYBRID_WEIGHTS.vector;
487608
const wk = config?.hybridSearchKeywordWeight ?? DEFAULT_HYBRID_WEIGHTS.keyword;
488609

489-
let merged;
490-
if (mode === "keyword") {
491-
merged = keyword;
492-
} else if (mode === "vector" || mode === "hybrid") {
493-
// semantic-first: vector recalls lead, keyword + BM25 fill remaining
494-
// slots. Weighted blend when sides scored the same memory; otherwise
495-
// vector order leads (it is the semantic signal), lexical paths
496-
// backfill.
497-
const byId = new Map();
498-
for (const m of vector) {
499-
const rec = byId.get(m.id);
500-
byId.set(m.id, rec ? { ...rec, score: Math.max(rec.score ?? 0, m.score ?? 0) } : m);
501-
}
502-
for (const m of keyword) {
503-
const rec = byId.get(m.id);
504-
if (rec) {
505-
// Same memory from both sides: blend the scores.
506-
byId.set(m.id, { ...rec, score: (rec.score ?? 0) * wv + (m.score ?? 0) * wk });
507-
} else {
508-
byId.set(m.id, m);
509-
}
510-
}
511-
for (const m of bm25) {
512-
const rec = byId.get(m.id);
513-
if (rec) {
514-
// Boost rule: a row the LIKE keyword path already hit carries the
515-
// query as a substring, so BM25 tokens are trivially present —
516-
// boosting it double-counts lexical evidence. Only vector-recalled
517-
// rows (lexical hit is genuinely new information) get the boost.
518-
if (keywordIds.has(m.id)) continue;
519-
byId.set(m.id, { ...rec, score: (rec.score ?? 0) + wb * (m.score ?? 0) });
520-
} else {
521-
byId.set(m.id, { ...m, score: wb * (m.score ?? 0) });
522-
}
523-
}
524-
const ranked = [...byId.values()].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
525-
merged = ranked.slice(0, lim);
526-
if (merged.length < lim && !merged.length) {
527-
// Vector unavailable entirely: fall back to plain keyword.
528-
merged = keyword.slice(0, lim);
529-
}
530-
} else {
531-
// auto: keyword leads, vector + BM25 fill remaining slots (legacy
532-
// behavior, extended with the third path)
533-
merged = keyword.slice(0, lim);
534-
const seen = new Set(merged.map((m) => m.id));
535-
for (const m of vector) {
536-
if (merged.length >= lim) break;
537-
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
538-
}
539-
for (const m of bm25) {
540-
if (merged.length >= lim) break;
541-
if (!seen.has(m.id)) { seen.add(m.id); merged.push(m); }
542-
}
610+
const { merged: fusedMerged, signals } = fuseRecall({ keyword, vector, bm25, lim, mode, wv, wk, wb });
611+
let merged = fusedMerged;
612+
// Signal transparency (#2): decorate each returned row with its per-source
613+
// scores and the final fused score. Purely additive — never changes rank.
614+
if (config?.signalTransparency === true) {
615+
merged = merged.map((m) => ({ ...m, signals: { ...(signals.get(m.id) ?? {}), final: m.score ?? 0 } }));
543616
}
544617

545618
// Search-time semantic dedup (v0.5.0 2.3): near-duplicate rows are

‎dsh-mneme/lib/settings.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ const FEATURE_FLAG_BOOLEANS = [
5858
"bm25SearchEnabled",
5959
"conflictFreezeEnabled",
6060
"trustEpistemicWeighting",
61+
// Plan #2: attach per-source {keyword, vector, bm25, final} signals to each
62+
// search result for transparency/debugging. Default off, purely decorative.
63+
"signalTransparency",
6164
// Issue #89:宽容校验回归(默认开)+ 跨类型合并显式放宽(默认关)。
6265
"dreamSkipInvalid",
6366
"allowCrossTypeMerge",
@@ -94,7 +97,10 @@ const FEATURE_FLAG_STRINGS = [
9497
const FEATURE_FLAG_URLS = ["ollamaBaseUrl"];
9598
// 枚举开关(与 config.js 的 z.union(z.const(...)) 对齐):仅允许列出的值。
9699
const FEATURE_FLAG_ENUMS = {
97-
embedProvider: ["openai", "local", "ollama"]
100+
embedProvider: ["openai", "local", "ollama"],
101+
// Plan #1: recall fusion recipe. blend = legacy (default); rrf / minmax are
102+
// rank/scale-aware alternatives selected by the panel.
103+
recallFusion: ["blend", "rrf", "minmax"]
98104
};
99105
const FEATURE_FLAG_STRING_MAX = 200;
100106

‎dsh-mneme/scripts/benchmark-recall.js‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,16 @@ const SEED = [
3737
{ id: "mem_city_thesis", type: "project", title: "湿地论文", content: "毕业论文研究城市湿地公园周边开发案例,ArcGIS 空间分析", importance: 4, tags: ["thesis"] },
3838
{ id: "mem_async_pattern", type: "decision", title: "异步并发模式", content: "async runtime 选用 tokio,任务用 spawn 管理,channel 通信", importance: 3, tags: ["rust"] },
3939
{ id: "mem_python_etl", type: "project", title: "ETL 脚本", content: "夜间 ETL 用 Python 编写,pandas 清洗,SQLite 落地", importance: 3, tags: ["etl"] },
40-
{ id: "mem_ui_style", type: "preference", title: "界面审美", content: "喜欢编辑风 brutalism 排版,低饱和度配色,衬线标题", importance: 3, tags: ["design"] }
40+
{ id: "mem_ui_style", type: "preference", title: "界面审美", content: "喜欢编辑风 brutalism 排版,低饱和度配色,衬线标题", importance: 3, tags: ["design"] },
41+
// Cross-topic distractors (plan #0): memories that share keywords with a
42+
// target but belong to a different subject. They raise the recall bar — the
43+
// fused ranking must keep the true positive ahead of the distractor, which
44+
// is exactly what a scale-mixed blend (raw cosine + keyword score + IDF)
45+
// tends to get wrong.
46+
{ id: "mem_ops_alert", type: "project", title: "存储告警", content: "Prometheus 存储告警走 zfs 池健康检查与磁盘替换流程", importance: 3, tags: ["ops"] },
47+
{ id: "mem_rust_dep", type: "project", title: "rust 依赖", content: "Rust 项目的 cargo 依赖管理与 workspace 组织", importance: 3, tags: ["rust"] },
48+
{ id: "mem_etl_csv", type: "project", title: "ETL CSV", content: "每日 CSV 导入脚本用 golang 而非 python,写 postgres", importance: 2, tags: ["etl"] },
49+
{ id: "mem_ux_toolbar", type: "preference", title: "工具栏", content: "偏好 IDE 顶栏简洁,避免深色浮层遮挡代码", importance: 2, tags: ["design"] }
4150
];
4251

4352
// Standard query set: each case is a query plus the ids that MUST appear in
@@ -53,10 +62,15 @@ export const TEST_CASES = [
5362
{ query: "channel 通信 任务", expected: ["mem_async_pattern"], note: "scattered terms" },
5463
{ query: "内存安全 语言", expected: ["mem_rust_switch"], note: "scattered terms" },
5564
{ query: "配色 审美", expected: ["mem_ui_style"], note: "scattered CJK" },
56-
{ query: "HBA 固件", expected: ["mem_zfs_bug"], note: "scattered terms" }
65+
{ query: "HBA 固件", expected: ["mem_zfs_bug"], note: "scattered terms" },
66+
// Plan #0 additions: a distractor-dominance case (the target shares the
67+
// leading token with a cross-topic memory that must rank below it) and an
68+
// exact-token case that leans on the BM25 path.
69+
{ query: "zfs 磁盘 替换", expected: ["mem_zfs_bug"], note: "shared-token distractor" },
70+
{ query: "tokio spawn channel", expected: ["mem_async_pattern"], note: "exact async tokens" }
5771
];
5872

59-
function seedService(overrides = {}) {
73+
export function seedService(overrides = {}) {
6074
const store = createStore(":memory:");
6175
const config = {
6276
bm25SearchEnabled: true,
@@ -74,7 +88,11 @@ function seedService(overrides = {}) {
7488
embedSingle: async (text) => hashVec(text)
7589
});
7690
for (const m of SEED) {
77-
const row = store.save({ type: m.type, title: m.title, content: m.content, tags: m.tags, importance: m.importance, source: "seed" });
91+
// store.save accepts a caller-supplied id (store.js: memory.id ?? randomUUID).
92+
// Passing m.id keeps the seeded id stable so TEST_CASES.expected (which
93+
// references mem_*) match — without it every row gets a UUID and the
94+
// benchmark always reports 0% recall.
95+
const row = store.save({ id: m.id, type: m.type, title: m.title, content: m.content, tags: m.tags, importance: m.importance, source: "seed" });
7896
store.setEmbedding(row.id, hashVec(`${m.title} ${m.content}`));
7997
}
8098
return service;
@@ -109,6 +127,52 @@ export async function runBenchmark({ topK = 5, mode = "auto" } = {}) {
109127
return { topK, mode, runs };
110128
}
111129

130+
/**
131+
* Fusion-recipe A/B (plan #1): runs the same seed + query set with
132+
* config.recallFusion forced to each of blend / rrf / minmax, so the scale
133+
* mismatch fix can be judged on identical data. `blend` is the legacy recipe
134+
* and acts as the control — the pre-fusion behavior.
135+
*/
136+
export async function runFusionBenchmark({ topK = 5, mode = "auto" } = {}) {
137+
const recipes = ["blend", "rrf", "minmax"];
138+
const runs = [];
139+
for (const recipe of recipes) {
140+
const service = seedService({ recallFusion: recipe });
141+
const rows = [];
142+
let hits = 0;
143+
let mrrSum = 0;
144+
for (const tc of TEST_CASES) {
145+
const results = await service.searchMemories(tc.query, { mode, topK, useRerank: false });
146+
const ids = results.map((r) => r.id);
147+
const metrics = service.computeRetrievalMetrics(ids, tc.expected);
148+
if (metrics.recall === 1) hits++;
149+
mrrSum += metrics.mrr;
150+
rows.push({ query: tc.query, note: tc.note, expected: tc.expected, got: ids, ...metrics });
151+
}
152+
runs.push({
153+
config: recipe,
154+
recallAtK: +(hits / TEST_CASES.length).toFixed(3),
155+
avgMrr: +(mrrSum / TEST_CASES.length).toFixed(3),
156+
rows
157+
});
158+
}
159+
return { topK, mode, runs };
160+
}
161+
162+
function printFusionReport(report) {
163+
for (const run of report.runs) {
164+
console.log(`\n=== ${run.config} (topK=${report.topK}, mode=${report.mode}) ===`);
165+
for (const r of run.rows) {
166+
const ok = r.recall === 1 ? "PASS" : "MISS";
167+
console.log(` [${ok}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
168+
if (r.recall < 1) console.log(` expected ⊇ ${r.expected.join(", ")} got: ${r.got.join(", ") || "—"}`);
169+
}
170+
console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}`);
171+
}
172+
const summary = report.runs.map((r) => `${r.config}=${(r.recallAtK * 100).toFixed(1)}%`).join(" ");
173+
console.log(`\n融合配方 A/B (Recall@${report.topK}, ${report.mode}): ${summary}`);
174+
}
175+
112176
function printReport(report) {
113177
for (const run of report.runs) {
114178
console.log(`\n=== ${run.config} (topK=${report.topK}, mode=${report.mode}) ===`);
@@ -127,7 +191,9 @@ function printReport(report) {
127191
const invokedDirectly = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/").split("/").pop() ?? "");
128192
if (invokedDirectly) {
129193
const asJson = process.argv.includes("--json");
130-
const report = await runBenchmark({});
194+
const asFusion = process.argv.includes("--fusion");
195+
const report = asFusion ? await runFusionBenchmark({}) : await runBenchmark({});
131196
if (asJson) console.log(JSON.stringify(report, null, 2));
197+
else if (asFusion) printFusionReport(report);
132198
else printReport(report);
133199
}

‎dsh-mneme/src/config.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,19 @@ export const Config = z.object({
178178
searchSemanticDedup: z.boolean().default(false),
179179
searchSemanticDedupThreshold: z.number().min(0.5).max(1).default(0.95),
180180

181+
// Recall fusion recipe (plan #1): how the keyword/vector/BM25 ranked lists
182+
// are combined into the final ranking. `blend` (default) is the legacy
183+
// behavior — weighted sum for vector/hybrid, union backfill for auto —
184+
// unchanged. `rrf` (Reciprocal Rank Fusion) and `minmax` (min-max normalized
185+
// weighted sum) are rank/scale-aware recipes that fix the unit mismatch the
186+
// issue describes (raw cosine vs keyword score vs normalized IDF are added
187+
// directly). Off by default so existing behavior holds exactly.
188+
recallFusion: z.union([z.const("blend"), z.const("rrf"), z.const("minmax")]).default("blend"),
189+
// Attach a `signals` object { keyword, vector, bm25, final } to each search
190+
// result for transparency/debugging (plan #2). Default off; when on it only
191+
// decorates the returned rows, never changes the ranking.
192+
signalTransparency: z.boolean().default(false),
193+
181194
// --- semantic: rerank layer (v0.2) --------------------------------------
182195
// Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
183196
// (transformers.js) at init, so a bare install must not load it. Only an

0 commit comments

Comments
 (0)