11import { DatabaseSync } from "node:sqlite" ;
22import { randomUUID } from "node:crypto" ;
33import { statSync } from "node:fs" ;
4+ import { contentHashOf } from "./content-hash.js" ;
45
56const SCHEMA = `
67CREATE TABLE IF NOT EXISTS memories (
@@ -14,6 +15,7 @@ CREATE TABLE IF NOT EXISTS memories (
1415 archived INTEGER NOT NULL DEFAULT 0,
1516 source TEXT,
1617 content_history TEXT,
18+ content_hash TEXT,
1719 embedding TEXT,
1820 epistemic_status TEXT NOT NULL DEFAULT 'subjective',
1921 last_accessed_at TEXT,
@@ -407,6 +409,7 @@ function toRow(row) {
407409 type : row . type ,
408410 title : row . title ,
409411 content : row . content ,
412+ content_hash : row . content_hash ?? undefined ,
410413 tags : parseTags ( row . tags ) ,
411414 importance : row . importance ,
412415 forgotten : row . forgotten === 1 ,
@@ -715,6 +718,34 @@ export function createStore(path) {
715718 addColumn ( "memories" , "workspace_scope_source" , "ALTER TABLE memories ADD COLUMN workspace_scope_source TEXT" ) ;
716719 addColumn ( "memories" , "scope_decided_at" , "ALTER TABLE memories ADD COLUMN scope_decided_at TEXT" ) ;
717720
721+ // #254 计量信号:内容归一化哈希(口径与动机写在 content-hash.js 的文件头)。派生
722+ // 列,不参与任何判定——只给「同一内容又被写了一次」当等值锚。索引建在加列之后:
723+ // 老库打开时 SCHEMA 的 CREATE TABLE 对既有表不生效,列还不存在,把索引写进 SCHEMA
724+ // 会直接报 no such column(与下方 llm_audit_logs.session_key 同理)。
725+ addColumn ( "memories" , "content_hash" , "ALTER TABLE memories ADD COLUMN content_hash TEXT" ) ;
726+ // 列序是 content_hash 打头:哈希几乎唯一,等值 seek 就已收窄到候选行,type 只当同
727+ // 一次 seek 里的第二列过滤;反过来(type 打头)下面的存量回填就得全表扫——每次打开
728+ // 都要把全部正文读一遍。
729+ db . exec ( "CREATE INDEX IF NOT EXISTS idx_memories_content_hash ON memories(content_hash, type)" ) ;
730+ // 存量回填:老库(含归档区)的行都没有值,而「归档行纳入去重候选集」这条口径正是
731+ // 要靠既有归档行出数据。只算 NULL 行、幂等;改口径不是补 NULL 能修好的,要整列重算
732+ // (见 content-hash.js 的文件头)。空标题空正文的行算不出锚,跳过不写——否则每次
733+ // 打开都要为这些行再跑一遍 UPDATE。
734+ {
735+ const updates = db
736+ . prepare ( "SELECT id, title, content FROM memories WHERE content_hash IS NULL" )
737+ . all ( )
738+ . map ( ( row ) => [ contentHashOf ( row ) , row . id ] )
739+ . filter ( ( [ hash ] ) => hash ) ;
740+ if ( updates . length ) {
741+ const stmt = db . prepare ( "UPDATE memories SET content_hash = ? WHERE id = ? AND content_hash IS NULL" ) ;
742+ // 一个事务包住整批:逐条自动提交是每行一次 WAL 提交,几千行的存量库上纯属浪费。
743+ runAtomically ( ( ) => {
744+ for ( const [ hash , id ] of updates ) stmt . run ( hash , id ) ;
745+ } ) ;
746+ }
747+ }
748+
718749 // Legacy dream_runs without policy_epoch → backfill with the default epoch.
719750 addColumn ( "dream_runs" , "policy_epoch" , "ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0" ) ;
720751 addColumn ( "dream_runs" , "run_type" , "ALTER TABLE dream_runs ADD COLUMN run_type TEXT NOT NULL DEFAULT 'auto'" ) ;
@@ -953,6 +984,8 @@ export function createStore(path) {
953984 const importance = Number . isInteger ( memory . importance ) ? memory . importance : 3 ;
954985 const evidence = Array . isArray ( memory . evidence ) ? JSON . stringify ( memory . evidence ) : null ;
955986 const docPath = typeof memory . doc_path === "string" && memory . doc_path . trim ( ) ? memory . doc_path : null ;
987+ // #254 计量锚:由本行的 title/content 派生,调用方传什么都以这里算出的为准。
988+ const contentHash = contentHashOf ( memory ) ;
956989 const embedding = Array . isArray ( memory . embedding ) && memory . embedding . length
957990 ? JSON . stringify ( memory . embedding )
958991 : null ;
@@ -963,8 +996,8 @@ export function createStore(path) {
963996 : inferEpistemicStatus ( memory ) ;
964997 runAtomically ( ( ) => {
965998 db . prepare (
966- `INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, agent_scope, workspace_scope, agent_scope_source, workspace_scope_source, scope_decided_at, sensitivity, occurred_at, evidence, doc_path, created_at, updated_at)
967- VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
999+ `INSERT INTO memories (id, type, title, content, tags, importance, forgotten, archived, source, content_history, quality_score, embedding, epistemic_status, agent_scope, workspace_scope, agent_scope_source, workspace_scope_source, scope_decided_at, sensitivity, occurred_at, evidence, doc_path, content_hash, created_at, updated_at)
1000+ VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )`
9681001 ) . run (
9691002 id ,
9701003 type ,
@@ -987,6 +1020,7 @@ export function createStore(path) {
9871020 normalizeOccurredAt ( memory . occurred_at ) ,
9881021 evidence ,
9891022 docPath ,
1023+ contentHash ,
9901024 now ,
9911025 now
9921026 ) ;
@@ -999,6 +1033,41 @@ export function createStore(path) {
9991033 return getById ( id ) ;
10001034 }
10011035
1036+ /**
1037+ * #254 计量:与本次写入「归一化内容哈希」相同的既有行(去重候选集)。
1038+ * 结构化锚先收窄成本:按哈希等值 seek(走 idx_memories_content_hash,哈希几乎唯
1039+ * 一,落在候选集上的行只有几条),再按 type 与 scope 三维过滤——不是全表扫、也不
1040+ * 逐行比哈希。scope 用 IS 比(NULL 安全):未标注行互相匹配、与已标注行不匹配,
1041+ * 与 service.js 去重键(scopeKeyOf)同口径;入参先按 store 自己的 normalizeScopeText
1042+ * 归一,免得调用方传 " foo " 就漏配。
1043+ * 归档行与已遗忘行都在集内:#275 的分界是「出口止体积、不止重复」——被质量闸归档
1044+ * 的同一个事实必须仍能判成重复,否则同样的内容再写一次又是一条新行(#254 拍板:
1045+ * 归档行纳入去重候选集)。
1046+ * 排序即取舍:活跃行排在归档/遗忘行之前,再按 updated_at 倒序。LIMIT 先于调用方的
1047+ * 「活区优先」判断执行,而归档动作本身会顶 updated_at——纯按时间倒序时,同键命中一
1048+ * 旦超过窗口宽度,活跃行就被归档行挤出候选集,调用方只能看到归档命中,把「活跃重复」
1049+ * 误报成「归档重复」,恰好把这条信号要分流的两类弄反。
1050+ * @returns {Array<{id: string, archived: boolean, forgotten: boolean}> } 活跃行优先,其后按最近写入
1051+ */
1052+ function findContentHashMatches ( { type, hash, agent_scope : agentScope , workspace_scope : workspaceScope , sensitivity, limit = 10 } = { } ) {
1053+ if ( ! hash || ! type ) return [ ] ;
1054+ const lim = Number . isInteger ( limit ) && limit > 0 ? Math . min ( limit , 50 ) : 10 ;
1055+ const rows = db . prepare (
1056+ `SELECT id, archived, forgotten FROM memories
1057+ WHERE type = ? AND content_hash = ?
1058+ AND agent_scope IS ? AND workspace_scope IS ? AND sensitivity IS ?
1059+ ORDER BY archived ASC, forgotten ASC, updated_at DESC, id LIMIT ?`
1060+ ) . all (
1061+ type ,
1062+ hash ,
1063+ normalizeScopeText ( agentScope ) ,
1064+ normalizeScopeText ( workspaceScope ) ,
1065+ normalizeScopeText ( sensitivity ) ,
1066+ lim
1067+ ) ;
1068+ return rows . map ( ( row ) => ( { id : row . id , archived : row . archived === 1 , forgotten : row . forgotten === 1 } ) ) ;
1069+ }
1070+
10021071 function update ( id , patch ) {
10031072 const existing = getById ( id ) ;
10041073 if ( ! existing ) throw new Error ( `memory not found: ${ id } ` ) ;
@@ -1045,13 +1114,18 @@ export function createStore(path) {
10451114 const nextDecidedAt = patch . scope_decided_at !== undefined
10461115 ? normalizeOccurredAt ( patch . scope_decided_at )
10471116 : ( existing . scope_decided_at ?? null ) ;
1117+ // #254:标题或正文变了,锚跟着重算——它是当前内容的派生物,留旧值等于让「这行
1118+ // 现在装的什么」和标记对不上。
1119+ const nextTitle = patch . title ?? existing . title ;
1120+ const nextContent = patch . content ?? existing . content ;
1121+ const contentHash = contentHashOf ( { title : nextTitle , content : nextContent } ) ;
10481122 runAtomically ( ( ) => {
10491123 db . prepare (
1050- `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, agent_scope=?, workspace_scope=?, agent_scope_source=?, workspace_scope_source=?, scope_decided_at=?, evidence=?, updated_at=? WHERE id=?`
1124+ `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, agent_scope=?, workspace_scope=?, agent_scope_source=?, workspace_scope_source=?, scope_decided_at=?, evidence=?, content_hash=?, updated_at=? WHERE id=?`
10511125 ) . run (
10521126 type ,
1053- patch . title ?? existing . title ,
1054- patch . content ?? existing . content ,
1127+ nextTitle ,
1128+ nextContent ,
10551129 JSON . stringify ( patch . tags ?? existing . tags ) ,
10561130 Number . isInteger ( patch . importance ) ? patch . importance : existing . importance ,
10571131 patch . source !== undefined ? patch . source : ( existing . source ?? null ) ,
@@ -1065,6 +1139,7 @@ export function createStore(path) {
10651139 nextWorkspaceSource ,
10661140 nextDecidedAt ,
10671141 evidence ,
1142+ contentHash ,
10681143 now ,
10691144 id
10701145 ) ;
@@ -1126,22 +1201,27 @@ export function createStore(path) {
11261201 // dirty == false — recoverMirror sees no debt and the mirror stays stale.
11271202 // Wrapping both in one transaction means a CAS miss rolls back cleanly too
11281203 // (no write, no generation bump).
1204+ // 同 update:#254 的锚随标题/正文重算(CAS 路径也改这两列)。
1205+ const nextTitle = patch . title ?? existing . title ;
1206+ const nextContent = patch . content ?? existing . content ;
1207+ const contentHash = contentHashOf ( { title : nextTitle , content : nextContent } ) ;
11291208 let applied = false ;
11301209 runAtomically ( ( ) => {
11311210 const result = db . prepare (
1132- `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, updated_at=?
1211+ `UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, content_history=?, quality_score=?, embedding=?, epistemic_status=?, content_hash=?, updated_at=?
11331212 WHERE id=? AND updated_at=?`
11341213 ) . run (
11351214 type ,
1136- patch . title ?? existing . title ,
1137- patch . content ?? existing . content ,
1215+ nextTitle ,
1216+ nextContent ,
11381217 JSON . stringify ( patch . tags ?? existing . tags ) ,
11391218 Number . isInteger ( patch . importance ) ? patch . importance : existing . importance ,
11401219 patch . source !== undefined ? patch . source : ( existing . source ?? null ) ,
11411220 contentHistory ,
11421221 qualityScore ,
11431222 embedding ,
11441223 epistemicStatus ,
1224+ contentHash ,
11451225 now ,
11461226 id ,
11471227 expectedUpdatedAt
@@ -1193,15 +1273,15 @@ export function createStore(path) {
11931273 function demoteToSummary ( id , summary , { minRefTimeMs } = { } ) {
11941274 let changed = false ;
11951275 runAtomically ( ( ) => {
1196- const row = db . prepare ( "SELECT last_accessed_at, content, _full_content FROM memories WHERE id = ?" ) . get ( id ) ;
1276+ const row = db . prepare ( "SELECT title, last_accessed_at, content, _full_content FROM memories WHERE id = ?" ) . get ( id ) ;
11971277 if ( ! row || row . _full_content ) return ;
11981278 if ( minRefTimeMs !== undefined && row . last_accessed_at ) {
11991279 const lastMs = Date . parse ( row . last_accessed_at ) ;
12001280 if ( lastMs >= minRefTimeMs ) return ; // touched after snapshot — still hot
12011281 }
12021282 db . prepare (
1203- "UPDATE memories SET content = ?, _full_content = ?, updated_at = ? WHERE id = ?"
1204- ) . run ( summary , row . content , nowIso ( ) , id ) ;
1283+ "UPDATE memories SET content = ?, _full_content = ?, content_hash = ?, updated_at = ? WHERE id = ?"
1284+ ) . run ( summary , row . content , contentHashOf ( { title : row . title , content : summary } ) , nowIso ( ) , id ) ;
12051285 incrementGeneration ( ) ;
12061286 changed = true ;
12071287 } ) ;
@@ -1212,11 +1292,11 @@ export function createStore(path) {
12121292 function restoreContent ( id ) {
12131293 let changed = false ;
12141294 runAtomically ( ( ) => {
1215- const row = db . prepare ( "SELECT content, _full_content FROM memories WHERE id = ?" ) . get ( id ) ;
1295+ const row = db . prepare ( "SELECT title, content, _full_content FROM memories WHERE id = ?" ) . get ( id ) ;
12161296 if ( ! row || ! row . _full_content ) return ;
12171297 db . prepare (
1218- "UPDATE memories SET content = ?, _full_content = NULL, updated_at = ? WHERE id = ?"
1219- ) . run ( row . _full_content , nowIso ( ) , id ) ;
1298+ "UPDATE memories SET content = ?, _full_content = NULL, content_hash = ?, updated_at = ? WHERE id = ?"
1299+ ) . run ( row . _full_content , contentHashOf ( { title : row . title , content : row . _full_content } ) , nowIso ( ) , id ) ;
12201300 incrementGeneration ( ) ;
12211301 changed = true ;
12221302 } ) ;
@@ -2606,6 +2686,8 @@ export function createStore(path) {
26062686 saveDocument,
26072687 update,
26082688 compareAndUpdate,
2689+ // #254:写入准入的内容哈希候选集(只读)。
2690+ findContentHashMatches,
26092691 remove,
26102692 setForget,
26112693 setArchived,
0 commit comments