Skip to content

Commit 029dd5a

Browse files
committed
feat(admission): 内容哈希计量与归档行纳入去重候选集(#254)
维护者 2026-09-24 拍板的两条实现侧信号,仍只计量不拦截: 1. 计量信号用内容哈希,不用相似度:新增 content_hash 派生列(归一化后 sha256), 幂等迁移、索引、存量回填,五个写入口全部重算;写入准入的 metadata.dup 记下 命中行的 id 与归档状态。 2. 归档行纳入去重候选集:findContentHashMatches 的候选集含归档行(scope 三维按 IS 比较)。归档出口止体积不止重复,剩下的穿透口靠它兜住。 归一化口径(折叠了什么、为什么命中不等于内容相同)写在 src/content-hash.js 文件头; 索引列序 hash 打头由 EXPLAIN QUERY PLAN 用例锁住,否则存量回填每次开库退化成全表扫。
1 parent 269f37d commit 029dd5a

8 files changed

Lines changed: 713 additions & 38 deletions

File tree

‎dsh-mneme/lib/content-hash.js‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// #254 写入准入的确定性计量锚:内容归一化哈希(exact duplicate 判定)。
2+
//
3+
// 为什么用哈希而不是相似度:计量阶段要的是一个零成本、可复算、不留争议的判据。
4+
// 相似度阈值两边都是误判——同一件事换个说法就掉到线下,不同的事共享话题词又顶到
5+
// 线上,阈值往哪边挪都只是换一种错法(#254 拍板:计量信号用内容哈希)。哈希只认
6+
// 「归一化后逐字节相同」,算不出来就只有一种解释。
7+
//
8+
// 但它不等于「原文逐字节相同」:归一化会折掉格式,也会折掉标点,于是 `版本 3.5` 与
9+
// `版本 35`、`a-b` 与 `ab` 落同一个键。折叠范围(大小写 / 空白 / 标点)是口径本身
10+
// 定的,#254 已拍板——误报面在计量阶段可接受(只标记、不拦截),但「命中即精确重复」
11+
// 这句话别升级成「命中即内容相同」。要更窄的判据就得改归一化口径,而那要带存量重算。
12+
//
13+
// 归一化口径一次定死(NFKC → 小写 → 去标点 → 空白折叠 → trim),动机是让「只差
14+
// 格式」的两次写入落在同一个键上:全角/半角、大小写、行内换行与缩进、句末标点都
15+
// 不该让同一件事变成两条。反过来它不折叠同义词、不排序词序——那些只能靠相似度,
16+
// 而相似度已判定不可用于本信号。
17+
//
18+
// 改口径必须带存量重算策略:memories.content_hash 是派生列,口径一变,库里已有的
19+
// 值全部按旧口径算——补 NULL 修不好,只能整列重算(UPDATE 全表 + 重建索引)。
20+
// 所以口径只能在这里改,且改要单独成批,不要混进别的改动里。
21+
//
22+
// 空内容(标题与正文归一后都为空)返回 null:这类写入没有可比内容,落 NULL 让它们
23+
// 不互相匹配成「全是重复」。
24+
//
25+
// 另有一处形状相近、口径不同的哈希:service.js 里镜像人工编辑的 digest 校验(两处)
26+
// 用 sha256(title\0content),那里**不**归一化(要逐字节保真)。两把哈希用途不同,别互换。
27+
import { createHash } from "node:crypto";
28+
29+
/**
30+
* 归一化一段文本用于内容比对(口径见文件头)。
31+
* @param {unknown} value
32+
* @returns {string}
33+
*/
34+
export function normalizeForHash(value) {
35+
return String(value ?? "")
36+
.normalize("NFKC")
37+
.toLowerCase()
38+
.replace(/\p{P}/gu, "")
39+
.replace(/\s+/g, " ")
40+
.trim();
41+
}
42+
43+
/**
44+
* 一行记忆的内容锚:标题与正文分别归一后用 NUL 拼接再取 sha256(十六进制全串)。
45+
* NUL 分隔是防拼接歧义——('ab','c') 与 ('a','bc') 不能落到同一个键上。
46+
* @param {{title?: string, content?: string}|null|undefined} memory
47+
* @returns {string|null} 归一后无内容时返回 null
48+
*/
49+
export function contentHashOf(memory) {
50+
const title = normalizeForHash(memory?.title);
51+
const content = normalizeForHash(memory?.content);
52+
if (!title && !content) return null;
53+
return createHash("sha256").update(`${title}\u0000${content}`).digest("hex");
54+
}

‎dsh-mneme/lib/store.js‎

Lines changed: 96 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { DatabaseSync } from "node:sqlite";
22
import { randomUUID } from "node:crypto";
33
import { statSync } from "node:fs";
4+
import { contentHashOf } from "./content-hash.js";
45

56
const SCHEMA = `
67
CREATE TABLE IF NOT EXISTS memories (
@@ -14,6 +15,7 @@ CREATE TABLE IF NOT EXISTS memories (
1415
archived INTEGER NOT NULL DEFAULT 0,
1516
source TEXT,
1617
content_history TEXT,
18+
content_hash TEXT,
1719
embedding TEXT,
1820
epistemic_status TEXT NOT NULL DEFAULT 'subjective',
1921
last_accessed_at TEXT,
@@ -407,6 +409,7 @@ function toRow(row) {
407409
type: row.type,
408410
title: row.title,
409411
content: row.content,
412+
content_hash: row.content_hash ?? undefined,
410413
tags: parseTags(row.tags),
411414
importance: row.importance,
412415
forgotten: row.forgotten === 1,
@@ -715,6 +718,34 @@ export function createStore(path) {
715718
addColumn("memories", "workspace_scope_source", "ALTER TABLE memories ADD COLUMN workspace_scope_source TEXT");
716719
addColumn("memories", "scope_decided_at", "ALTER TABLE memories ADD COLUMN scope_decided_at TEXT");
717720

721+
// #254 计量信号:内容归一化哈希(口径与动机写在 content-hash.js 的文件头)。派生
722+
// 列,不参与任何判定——只给「同一内容又被写了一次」当等值锚。索引建在加列之后:
723+
// 老库打开时 SCHEMA 的 CREATE TABLE 对既有表不生效,列还不存在,把索引写进 SCHEMA
724+
// 会直接报 no such column(与下方 llm_audit_logs.session_key 同理)。
725+
addColumn("memories", "content_hash", "ALTER TABLE memories ADD COLUMN content_hash TEXT");
726+
// 列序是 content_hash 打头:哈希几乎唯一,等值 seek 就已收窄到候选行,type 只当同
727+
// 一次 seek 里的第二列过滤;反过来(type 打头)下面的存量回填就得全表扫——每次打开
728+
// 都要把全部正文读一遍。
729+
db.exec("CREATE INDEX IF NOT EXISTS idx_memories_content_hash ON memories(content_hash, type)");
730+
// 存量回填:老库(含归档区)的行都没有值,而「归档行纳入去重候选集」这条口径正是
731+
// 要靠既有归档行出数据。只算 NULL 行、幂等;改口径不是补 NULL 能修好的,要整列重算
732+
// (见 content-hash.js 的文件头)。空标题空正文的行算不出锚,跳过不写——否则每次
733+
// 打开都要为这些行再跑一遍 UPDATE。
734+
{
735+
const updates = db
736+
.prepare("SELECT id, title, content FROM memories WHERE content_hash IS NULL")
737+
.all()
738+
.map((row) => [contentHashOf(row), row.id])
739+
.filter(([hash]) => hash);
740+
if (updates.length) {
741+
const stmt = db.prepare("UPDATE memories SET content_hash = ? WHERE id = ? AND content_hash IS NULL");
742+
// 一个事务包住整批:逐条自动提交是每行一次 WAL 提交,几千行的存量库上纯属浪费。
743+
runAtomically(() => {
744+
for (const [hash, id] of updates) stmt.run(hash, id);
745+
});
746+
}
747+
}
748+
718749
// Legacy dream_runs without policy_epoch → backfill with the default epoch.
719750
addColumn("dream_runs", "policy_epoch", "ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
720751
addColumn("dream_runs", "run_type", "ALTER TABLE dream_runs ADD COLUMN run_type TEXT NOT NULL DEFAULT 'auto'");
@@ -953,6 +984,8 @@ export function createStore(path) {
953984
const importance = Number.isInteger(memory.importance) ? memory.importance : 3;
954985
const evidence = Array.isArray(memory.evidence) ? JSON.stringify(memory.evidence) : null;
955986
const docPath = typeof memory.doc_path === "string" && memory.doc_path.trim() ? memory.doc_path : null;
987+
// #254 计量锚:由本行的 title/content 派生,调用方传什么都以这里算出的为准。
988+
const contentHash = contentHashOf(memory);
956989
const embedding = Array.isArray(memory.embedding) && memory.embedding.length
957990
? JSON.stringify(memory.embedding)
958991
: null;
@@ -963,8 +996,8 @@ export function createStore(path) {
963996
: inferEpistemicStatus(memory);
964997
runAtomically(() => {
965998
db.prepare(
966-
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, agent_scope, workspace_scope, agent_scope_source, workspace_scope_source, scope_decided_at, sensitivity, occurred_at, evidence, doc_path, created_at, updated_at)
967-
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
999+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, agent_scope, workspace_scope, agent_scope_source, workspace_scope_source, scope_decided_at, sensitivity, occurred_at, evidence, doc_path, content_hash, created_at, updated_at)
1000+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
9681001
).run(
9691002
id,
9701003
type,
@@ -987,6 +1020,7 @@ export function createStore(path) {
9871020
normalizeOccurredAt(memory.occurred_at),
9881021
evidence,
9891022
docPath,
1023+
contentHash,
9901024
now,
9911025
now
9921026
);
@@ -999,6 +1033,41 @@ export function createStore(path) {
9991033
return getById(id);
10001034
}
10011035

1036+
/**
1037+
* #254 计量:与本次写入「归一化内容哈希」相同的既有行(去重候选集)。
1038+
* 结构化锚先收窄成本:按哈希等值 seek(走 idx_memories_content_hash,哈希几乎唯
1039+
* 一,落在候选集上的行只有几条),再按 type 与 scope 三维过滤——不是全表扫、也不
1040+
* 逐行比哈希。scope 用 IS 比(NULL 安全):未标注行互相匹配、与已标注行不匹配,
1041+
* 与 service.js 去重键(scopeKeyOf)同口径;入参先按 store 自己的 normalizeScopeText
1042+
* 归一,免得调用方传 " foo " 就漏配。
1043+
* 归档行与已遗忘行都在集内:#275 的分界是「出口止体积、不止重复」——被质量闸归档
1044+
* 的同一个事实必须仍能判成重复,否则同样的内容再写一次又是一条新行(#254 拍板:
1045+
* 归档行纳入去重候选集)。
1046+
* 排序即取舍:活跃行排在归档/遗忘行之前,再按 updated_at 倒序。LIMIT 先于调用方的
1047+
* 「活区优先」判断执行,而归档动作本身会顶 updated_at——纯按时间倒序时,同键命中一
1048+
* 旦超过窗口宽度,活跃行就被归档行挤出候选集,调用方只能看到归档命中,把「活跃重复」
1049+
* 误报成「归档重复」,恰好把这条信号要分流的两类弄反。
1050+
* @returns {Array<{id: string, archived: boolean, forgotten: boolean}>} 活跃行优先,其后按最近写入
1051+
*/
1052+
function findContentHashMatches({ type, hash, agent_scope: agentScope, workspace_scope: workspaceScope, sensitivity, limit = 10 } = {}) {
1053+
if (!hash || !type) return [];
1054+
const lim = Number.isInteger(limit) && limit > 0 ? Math.min(limit, 50) : 10;
1055+
const rows = db.prepare(
1056+
`SELECT id, archived, forgotten FROM memories
1057+
WHERE type = ? AND content_hash = ?
1058+
AND agent_scope IS ? AND workspace_scope IS ? AND sensitivity IS ?
1059+
ORDER BY archived ASC, forgotten ASC, updated_at DESC, id LIMIT ?`
1060+
).all(
1061+
type,
1062+
hash,
1063+
normalizeScopeText(agentScope),
1064+
normalizeScopeText(workspaceScope),
1065+
normalizeScopeText(sensitivity),
1066+
lim
1067+
);
1068+
return rows.map((row) => ({ id: row.id, archived: row.archived === 1, forgotten: row.forgotten === 1 }));
1069+
}
1070+
10021071
function update(id, patch) {
10031072
const existing = getById(id);
10041073
if (!existing) throw new Error(`memory not found: ${id}`);
@@ -1045,13 +1114,18 @@ export function createStore(path) {
10451114
const nextDecidedAt = patch.scope_decided_at !== undefined
10461115
? normalizeOccurredAt(patch.scope_decided_at)
10471116
: (existing.scope_decided_at ?? null);
1117+
// #254:标题或正文变了,锚跟着重算——它是当前内容的派生物,留旧值等于让「这行
1118+
// 现在装的什么」和标记对不上。
1119+
const nextTitle = patch.title ?? existing.title;
1120+
const nextContent = patch.content ?? existing.content;
1121+
const contentHash = contentHashOf({ title: nextTitle, content: nextContent });
10481122
runAtomically(() => {
10491123
db.prepare(
1050-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, agent_scope=?, workspace_scope=?, agent_scope_source=?, workspace_scope_source=?, scope_decided_at=?, evidence=?, updated_at=? WHERE id=?`
1124+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, agent_scope=?, workspace_scope=?, agent_scope_source=?, workspace_scope_source=?, scope_decided_at=?, evidence=?, content_hash=?, updated_at=? WHERE id=?`
10511125
).run(
10521126
type,
1053-
patch.title ?? existing.title,
1054-
patch.content ?? existing.content,
1127+
nextTitle,
1128+
nextContent,
10551129
JSON.stringify(patch.tags ?? existing.tags),
10561130
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
10571131
patch.source !== undefined ? patch.source : (existing.source ?? null),
@@ -1065,6 +1139,7 @@ export function createStore(path) {
10651139
nextWorkspaceSource,
10661140
nextDecidedAt,
10671141
evidence,
1142+
contentHash,
10681143
now,
10691144
id
10701145
);
@@ -1126,22 +1201,27 @@ export function createStore(path) {
11261201
// dirty == false — recoverMirror sees no debt and the mirror stays stale.
11271202
// Wrapping both in one transaction means a CAS miss rolls back cleanly too
11281203
// (no write, no generation bump).
1204+
// 同 update:#254 的锚随标题/正文重算(CAS 路径也改这两列)。
1205+
const nextTitle = patch.title ?? existing.title;
1206+
const nextContent = patch.content ?? existing.content;
1207+
const contentHash = contentHashOf({ title: nextTitle, content: nextContent });
11291208
let applied = false;
11301209
runAtomically(() => {
11311210
const result = db.prepare(
1132-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, updated_at=?
1211+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, content_hash=?, updated_at=?
11331212
WHERE id=? AND updated_at=?`
11341213
).run(
11351214
type,
1136-
patch.title ?? existing.title,
1137-
patch.content ?? existing.content,
1215+
nextTitle,
1216+
nextContent,
11381217
JSON.stringify(patch.tags ?? existing.tags),
11391218
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
11401219
patch.source !== undefined ? patch.source : (existing.source ?? null),
11411220
contentHistory,
11421221
qualityScore,
11431222
embedding,
11441223
epistemicStatus,
1224+
contentHash,
11451225
now,
11461226
id,
11471227
expectedUpdatedAt
@@ -1193,15 +1273,15 @@ export function createStore(path) {
11931273
function demoteToSummary(id, summary, { minRefTimeMs } = {}) {
11941274
let changed = false;
11951275
runAtomically(() => {
1196-
const row = db.prepare("SELECT last_accessed_at, content, _full_content FROM memories WHERE id = ?").get(id);
1276+
const row = db.prepare("SELECT title, last_accessed_at, content, _full_content FROM memories WHERE id = ?").get(id);
11971277
if (!row || row._full_content) return;
11981278
if (minRefTimeMs !== undefined && row.last_accessed_at) {
11991279
const lastMs = Date.parse(row.last_accessed_at);
12001280
if (lastMs >= minRefTimeMs) return; // touched after snapshot — still hot
12011281
}
12021282
db.prepare(
1203-
"UPDATE memories SET content = ?, _full_content = ?, updated_at = ? WHERE id = ?"
1204-
).run(summary, row.content, nowIso(), id);
1283+
"UPDATE memories SET content = ?, _full_content = ?, content_hash = ?, updated_at = ? WHERE id = ?"
1284+
).run(summary, row.content, contentHashOf({ title: row.title, content: summary }), nowIso(), id);
12051285
incrementGeneration();
12061286
changed = true;
12071287
});
@@ -1212,11 +1292,11 @@ export function createStore(path) {
12121292
function restoreContent(id) {
12131293
let changed = false;
12141294
runAtomically(() => {
1215-
const row = db.prepare("SELECT content, _full_content FROM memories WHERE id = ?").get(id);
1295+
const row = db.prepare("SELECT title, content, _full_content FROM memories WHERE id = ?").get(id);
12161296
if (!row || !row._full_content) return;
12171297
db.prepare(
1218-
"UPDATE memories SET content = ?, _full_content = NULL, updated_at = ? WHERE id = ?"
1219-
).run(row._full_content, nowIso(), id);
1298+
"UPDATE memories SET content = ?, _full_content = NULL, content_hash = ?, updated_at = ? WHERE id = ?"
1299+
).run(row._full_content, contentHashOf({ title: row.title, content: row._full_content }), nowIso(), id);
12201300
incrementGeneration();
12211301
changed = true;
12221302
});
@@ -2606,6 +2686,8 @@ export function createStore(path) {
26062686
saveDocument,
26072687
update,
26082688
compareAndUpdate,
2689+
// #254:写入准入的内容哈希候选集(只读)。
2690+
findContentHashMatches,
26092691
remove,
26102692
setForget,
26112693
setArchived,

0 commit comments

Comments
 (0)