|
| 1 | +// dsh-mneme/src/organize.js |
| 2 | +// agent 主动整理接口(issue #231,口径基线 = #164 的「第三腿触发」):dryRun |
| 3 | +// 出比对报告 → agent 判断 → apply 落库;筛除项进归档不删;调用全程留审计回执。 |
| 4 | +// |
| 5 | +// 只做功能本体(维护者口径,issue #231):落在 service 层并带测试,**不进工具 |
| 6 | +// 列表、不加独立 opt-in 开关**——#249 到位时只差「注册工具 + 注入指引」一步。 |
| 7 | +// |
| 8 | +// 独立成模块(AGENTS.md 尺寸约定,同 document.js / recall-stats.js 先例): |
| 9 | +// service.js 只做依赖注入 + barrel 出口,调用方零改动。 |
| 10 | +// |
| 11 | +// 三条硬规则,都是「宁可什么都不做」的形态: |
| 12 | +// 1. dryRun 不写记忆表:它只读 + 写一行审计(dream_runs)。比对是报告,不是动作。 |
| 13 | +// 2. apply 必须引用一次真实 dryRun(run_id 且 run_type='organize')——没有比对过 |
| 14 | +// 的候选一律不落库,堵死「跳过报告直接写」这条绕过路径。 |
| 15 | +// 3. 筛除 = 归档(store.setArchived),绝不物理删除:归档可恢复,删除不可逆。 |
| 16 | +// |
| 17 | +// 宽容形态(仓库红线 4,同 issue #89):单条非法候选/决策跳过 + 应用合法子集 + |
| 18 | +// run 记 degraded,不整单拒绝。审计口径复用 dream_runs 的 receipt 语义 |
| 19 | +// (buildReceipt / parseReceipt 同一格式),不新建审计面。 |
| 20 | + |
| 21 | +import { createHash, randomUUID } from "node:crypto"; |
| 22 | +import { scopeKeyOf } from "./scope.js"; |
| 23 | +import { cosineSimilarity } from "./dream/clustering.js"; |
| 24 | +import { buildReceipt } from "./dream.js"; |
| 25 | + |
| 26 | +// 与 document 的 C2 档(#230)和 findSessionDuplicate 的 vector 档同源:同一份 |
| 27 | +// 「近重复」语义只有一个阈值,不在第三个地方另发明一个。 |
| 28 | +const MIN_SIM = 0.92; |
| 29 | +// 向量比对对象的上限(document.js 同口径:精确层全量、向量层有界)。精确层 |
| 30 | +// 不能截断——同标题漏检会让报告给出错误的 verdict。 |
| 31 | +const CANDIDATE_LIMIT = 200; |
| 32 | +// 单次 dryRun 的候选硬上限:主场景是「长段工作收尾整理」,不是批量导入;不设界 |
| 33 | +// 的话一次调用会把整库拉进比对。 |
| 34 | +const MAX_ITEMS = 50; |
| 35 | + |
| 36 | +/** |
| 37 | + * Build the organizer. |
| 38 | + * |
| 39 | + * Injected deps are service.js closure members (store / embedQuery / saveWithDedupe |
| 40 | + * / transaction) so this module stays free of service-internal wiring; `saveWithDedupe` |
| 41 | + * is the normal write path, so its own epilogue (mirror sync / notify / re-embed) |
| 42 | + * is never duplicated here. |
| 43 | + * |
| 44 | + * @returns {{ organize: (payload: object) => Promise<object> }} |
| 45 | + * `organize({ mode: "dryRun"|"apply", ... })` — 一个入口带 mode 参数(维护者 |
| 46 | + * 倾向的形态),内部两个具名函数各自可测。 |
| 47 | + */ |
| 48 | +export function createOrganizer({ store, embedQuery, saveWithDedupe, transaction }) { |
| 49 | + const hashOf = (value) => createHash("sha256").update(JSON.stringify(value ?? null)).digest("hex"); |
| 50 | + |
| 51 | + /** |
| 52 | + * 候选规范化。返回 `{ candidate }` 或 `{ error }`——错误在 dryRun 里进 skipped, |
| 53 | + * 不抛整单(红线 4)。document 刻意在此拦下:它的唯一铸造口是 registerDocument |
| 54 | + * (#230),service 的通用 save 路径也会拒绝,这里提前拦截只是为了报告不撒谎。 |
| 55 | + */ |
| 56 | + function normalizeCandidate(raw) { |
| 57 | + const type = String(raw?.type ?? "").trim(); |
| 58 | + const title = String(raw?.title ?? "").trim(); |
| 59 | + const content = String(raw?.content ?? "").trim(); |
| 60 | + if (!type || !title || !content) return { error: "type/title/content are required" }; |
| 61 | + if (type === "document") return { error: "document rows are minted only via registerDocument (#230)" }; |
| 62 | + const candidate = { |
| 63 | + type, |
| 64 | + title, |
| 65 | + content, |
| 66 | + importance: Number.isInteger(raw?.importance) ? Math.min(5, Math.max(1, raw.importance)) : 3, |
| 67 | + tags: [...new Set((Array.isArray(raw?.tags) ? raw.tags : []).map((t) => String(t ?? "").trim()).filter(Boolean))], |
| 68 | + source: String(raw?.source ?? "organize") |
| 69 | + }; |
| 70 | + // scope 标注只做搬运(判定在 store / service 侧),省得整理接口成为第二条 |
| 71 | + // 绕过 scope 治理的写入通道。 |
| 72 | + for (const key of ["sensitivity", "agent_scope", "workspace_scope", "agent_scope_source", "workspace_scope_source"]) { |
| 73 | + if (raw?.[key] !== undefined) candidate[key] = raw[key]; |
| 74 | + } |
| 75 | + return { candidate }; |
| 76 | + } |
| 77 | + |
| 78 | + /** 标题归一(同 findSessionDuplicate 的 title 档):大小写与标点不构成「不同标题」。 */ |
| 79 | + const normalizeTitle = (s) => String(s ?? "").toLowerCase().replace(/[\s\p{P}]+/gu, ""); |
| 80 | + |
| 81 | + /** 向量档最佳命中;拿不到向量/嵌入器时返回 null(无信号 = 不判定,不是报错)。 */ |
| 82 | + async function bestNear(candidate, rows, vecs) { |
| 83 | + if (!vecs?.size) return null; |
| 84 | + let probe = null; |
| 85 | + try { |
| 86 | + probe = await embedQuery([candidate.title, candidate.content].filter(Boolean).join("\n")); |
| 87 | + } catch { |
| 88 | + return null; // 嵌入器故障只降级比对,不阻塞报告 |
| 89 | + } |
| 90 | + if (!probe) return null; |
| 91 | + let best = null; |
| 92 | + for (const m of rows) { |
| 93 | + const v = vecs.get(m.id); |
| 94 | + if (!v) continue; // 无向量的既有行不参与比对 |
| 95 | + const sim = cosineSimilarity(probe, v); |
| 96 | + if (sim >= MIN_SIM && (!best || sim > best.sim)) best = { memory: m, sim }; |
| 97 | + } |
| 98 | + return best; |
| 99 | + } |
| 100 | + |
| 101 | + /** |
| 102 | + * 比对报告。**不写记忆表**;只写一行 dream_runs 审计(run_type='organize')。 |
| 103 | + * |
| 104 | + * @param {object} payload |
| 105 | + * `{ candidates: Array<{type,title,content,importance?,tags?}>, agent_scope?, |
| 106 | + * workspace_scope?, sensitivity? }` |
| 107 | + * @returns {Promise<{run_id, created_at, status, counts, items, skipped}>} |
| 108 | + * items[i] = `{ index, type, title, verdict: "exact"|"near"|"new", match }` |
| 109 | + */ |
| 110 | + async function dryRun(payload = {}) { |
| 111 | + const raws = Array.isArray(payload.candidates) ? payload.candidates : null; |
| 112 | + if (!raws) throw new Error("organize.dryRun: candidates must be an array"); |
| 113 | + const bounded = raws.slice(0, MAX_ITEMS); |
| 114 | + |
| 115 | + const skipped = []; |
| 116 | + const accepted = []; |
| 117 | + bounded.forEach((raw, index) => { |
| 118 | + const { candidate, error } = normalizeCandidate(raw); |
| 119 | + if (error) skipped.push({ index, error }); |
| 120 | + else accepted.push({ index, candidate }); |
| 121 | + }); |
| 122 | + |
| 123 | + // scope 同等才比对(跨 scope 永不互判——防泄漏,与 saveWithDedupe 的去重键 |
| 124 | + // 「type+title+scope」同口径)。行扫描与向量读取按 type 缓存:一批候选通常 |
| 125 | + // 同类型,避免逐条全表扫。 |
| 126 | + const scopeMatches = (m) => |
| 127 | + scopeKeyOf(m.agent_scope) === scopeKeyOf(payload?.agent_scope) && |
| 128 | + scopeKeyOf(m.workspace_scope) === scopeKeyOf(payload?.workspace_scope) && |
| 129 | + scopeKeyOf(m.sensitivity) === scopeKeyOf(payload?.sensitivity); |
| 130 | + const rowsByType = new Map(); |
| 131 | + const rowsFor = (type) => { |
| 132 | + if (!rowsByType.has(type)) { |
| 133 | + rowsByType.set(type, store.list({ type, limit: null }).filter((m) => !m.archived && scopeMatches(m))); |
| 134 | + } |
| 135 | + return rowsByType.get(type); |
| 136 | + }; |
| 137 | + const vecsByType = new Map(); |
| 138 | + const vecsFor = (type, rows) => { |
| 139 | + if (!vecsByType.has(type)) { |
| 140 | + vecsByType.set(type, store.getEmbeddings(rows.slice(0, CANDIDATE_LIMIT).map((m) => m.id))); |
| 141 | + } |
| 142 | + return vecsByType.get(type); |
| 143 | + }; |
| 144 | + |
| 145 | + const items = []; |
| 146 | + for (const { index, candidate } of accepted) { |
| 147 | + const rows = rowsFor(candidate.type); |
| 148 | + let item = { index, type: candidate.type, title: candidate.title, verdict: "new", match: null }; |
| 149 | + const exact = rows.find((m) => normalizeTitle(m.title) === normalizeTitle(candidate.title)); |
| 150 | + if (exact) { |
| 151 | + item = { ...item, verdict: "exact", match: { id: exact.id, title: exact.title, sim: null } }; |
| 152 | + } else { |
| 153 | + const near = await bestNear(candidate, rows, vecsFor(candidate.type, rows)); |
| 154 | + if (near) { |
| 155 | + item = { ...item, verdict: "near", match: { id: near.memory.id, title: near.memory.title, sim: near.sim } }; |
| 156 | + } |
| 157 | + } |
| 158 | + items.push(item); |
| 159 | + } |
| 160 | + |
| 161 | + const counts = { |
| 162 | + total: items.length, |
| 163 | + exact: items.filter((i) => i.verdict === "exact").length, |
| 164 | + near: items.filter((i) => i.verdict === "near").length, |
| 165 | + new: items.filter((i) => i.verdict === "new").length, |
| 166 | + skipped: skipped.length, |
| 167 | + truncated: raws.length - bounded.length |
| 168 | + }; |
| 169 | + // 有被拒的候选 → degraded(如实标记),而不是 ok 掩盖过去;一条都没报 → noop。 |
| 170 | + const status = skipped.length ? "degraded" : items.length ? "ok" : "noop"; |
| 171 | + const runId = randomUUID(); |
| 172 | + const createdAt = new Date().toISOString(); |
| 173 | + const snapshotHash = hashOf(bounded); |
| 174 | + store.saveDreamRun({ |
| 175 | + id: runId, |
| 176 | + created_at: createdAt, |
| 177 | + status, |
| 178 | + provider: null, |
| 179 | + model: null, |
| 180 | + snapshot_hash: snapshotHash, |
| 181 | + input_count: bounded.length, |
| 182 | + input: bounded, |
| 183 | + decisions: items, |
| 184 | + applied: 0, |
| 185 | + summary_stored: 0, |
| 186 | + receipt: buildReceipt({ runId, status, snapshotHash, inputCount: bounded.length, applied: 0, summaryStored: false }), |
| 187 | + policy_epoch: 0, |
| 188 | + run_type: "organize", |
| 189 | + ...(skipped.length ? { skipped } : {}) |
| 190 | + }); |
| 191 | + return { run_id: runId, created_at: createdAt, status, counts, items, skipped }; |
| 192 | + } |
| 193 | + |
| 194 | + /** |
| 195 | + * 按 agent 的判断落库。decisions 逐条: |
| 196 | + * `{ action: "save", index }` — 写候选(走 saveWithDedupe:同键自动合并) |
| 197 | + * `{ action: "discard", index }` — 不写(只进回执,不删任何东西) |
| 198 | + * `{ action: "archive", id }` — 库内被取代的行 → 归档(可恢复,不删) |
| 199 | + * |
| 200 | + * @param {object} payload `{ run_id, decisions }` |
| 201 | + * @param {object} [opts] `{ hiddenIds }` — 调用者 scope 看不见的行按「不存在」 |
| 202 | + * 处理(#170 复核项 4:无存在性泄漏),与 document 的 hiddenEvidenceIds 同形。 |
| 203 | + * @returns {{run_id, dry_run_id, status, saved, discarded, archived, memory_ids, skipped, degraded}} |
| 204 | + */ |
| 205 | + function apply(payload = {}, { hiddenIds = [] } = {}) { |
| 206 | + const dryRunId = String(payload?.run_id ?? "").trim(); |
| 207 | + if (!dryRunId) { |
| 208 | + throw new Error("organize.apply: run_id is required (apply must reference a dryRun report)"); |
| 209 | + } |
| 210 | + const report = store.getDreamRun(dryRunId); |
| 211 | + if (!report || report.run_type !== "organize") { |
| 212 | + throw new Error(`organize.apply: unknown organize run "${dryRunId}"`); |
| 213 | + } |
| 214 | + const decisions = Array.isArray(payload?.decisions) ? payload.decisions : null; |
| 215 | + if (!decisions) throw new Error("organize.apply: decisions must be an array"); |
| 216 | + |
| 217 | + const input = Array.isArray(report.input) ? report.input : []; |
| 218 | + const hidden = new Set((Array.isArray(hiddenIds) ? hiddenIds : []).map((id) => String(id))); |
| 219 | + const skipped = []; |
| 220 | + const byId = {}; |
| 221 | + const memoryIds = []; |
| 222 | + let saved = 0; |
| 223 | + let discarded = 0; |
| 224 | + let archived = 0; |
| 225 | + |
| 226 | + const runDecisions = () => { |
| 227 | + decisions.forEach((decision, i) => { |
| 228 | + const action = String(decision?.action ?? "").trim(); |
| 229 | + if (action === "save") { |
| 230 | + const index = Number(decision?.index); |
| 231 | + const candidate = Number.isInteger(index) ? input[index] : undefined; |
| 232 | + if (!candidate) { |
| 233 | + skipped.push({ index: i, action, error: "candidate index is out of range" }); |
| 234 | + return; |
| 235 | + } |
| 236 | + try { |
| 237 | + const result = saveWithDedupe(candidate); |
| 238 | + saved++; |
| 239 | + memoryIds.push(result.memory.id); |
| 240 | + byId[result.memory.id] = action; |
| 241 | + } catch (e) { |
| 242 | + // 单条失败不拖垮整批(红线 4);原因进 skipped 明细,run 记 degraded。 |
| 243 | + skipped.push({ index: i, action, error: String(e?.message ?? e) }); |
| 244 | + } |
| 245 | + return; |
| 246 | + } |
| 247 | + if (action === "discard") { |
| 248 | + discarded++; |
| 249 | + return; |
| 250 | + } |
| 251 | + if (action === "archive") { |
| 252 | + const id = String(decision?.id ?? "").trim(); |
| 253 | + const row = !id || hidden.has(id) ? null : store.getById(id); |
| 254 | + if (!row) { |
| 255 | + skipped.push({ index: i, action, ids: id ? [id] : [], error: "unknown, archived or out of scope" }); |
| 256 | + return; |
| 257 | + } |
| 258 | + if (row.archived) { |
| 259 | + byId[id] = "keep"; // 已归档 = 目标状态已达,不重复计数 |
| 260 | + return; |
| 261 | + } |
| 262 | + store.setArchived(id, true); |
| 263 | + archived++; |
| 264 | + byId[id] = action; |
| 265 | + return; |
| 266 | + } |
| 267 | + skipped.push({ index: i, action, error: `unknown action "${action}"` }); |
| 268 | + }); |
| 269 | + }; |
| 270 | + |
| 271 | + // 批量写在一个事务里:要么整批可见,要么全不可见(不留半批)。单条业务失败 |
| 272 | + // 已在上面吞掉,所以这里抛出的都是基础设施级错误 → failed + 原样上抛。 |
| 273 | + const applyRunId = randomUUID(); |
| 274 | + const createdAt = new Date().toISOString(); |
| 275 | + const snapshotHash = report.snapshot_hash; |
| 276 | + const record = (status, error) => store.saveDreamRun({ |
| 277 | + id: applyRunId, |
| 278 | + created_at: createdAt, |
| 279 | + status, |
| 280 | + error, |
| 281 | + provider: null, |
| 282 | + model: null, |
| 283 | + snapshot_hash: snapshotHash, |
| 284 | + input_count: input.length, |
| 285 | + input, |
| 286 | + decisions, |
| 287 | + // dry_run_id 是这条回执与它所指报告的连接键:审计能还原「报告 → 判断 → 落地」 |
| 288 | + // 三步,而不用新建审计面。 |
| 289 | + outcome: { dry_run_id: dryRunId, byId }, |
| 290 | + applied: saved + archived, |
| 291 | + summary_stored: 0, |
| 292 | + receipt: buildReceipt({ |
| 293 | + runId: applyRunId, |
| 294 | + status, |
| 295 | + snapshotHash, |
| 296 | + inputCount: input.length, |
| 297 | + applied: saved + archived, |
| 298 | + summaryStored: false |
| 299 | + }), |
| 300 | + policy_epoch: 0, |
| 301 | + run_type: "organize", |
| 302 | + ...(skipped.length ? { skipped } : {}) |
| 303 | + }); |
| 304 | + |
| 305 | + try { |
| 306 | + transaction(runDecisions); |
| 307 | + } catch (e) { |
| 308 | + record("failed", String(e?.message ?? e)); |
| 309 | + throw e; |
| 310 | + } |
| 311 | + |
| 312 | + // 只有 discard 的批次什么都没改 → noop(绝不虚报 ok)。 |
| 313 | + const status = skipped.length ? "degraded" : saved + archived > 0 ? "ok" : "noop"; |
| 314 | + record(status, null); |
| 315 | + return { |
| 316 | + run_id: applyRunId, |
| 317 | + dry_run_id: dryRunId, |
| 318 | + status, |
| 319 | + saved, |
| 320 | + discarded, |
| 321 | + archived, |
| 322 | + memory_ids: memoryIds, |
| 323 | + skipped, |
| 324 | + degraded: skipped.length > 0 |
| 325 | + }; |
| 326 | + } |
| 327 | + |
| 328 | + async function organize(payload = {}) { |
| 329 | + const mode = String(payload?.mode ?? "dryRun").trim(); |
| 330 | + if (mode === "dryRun") return dryRun(payload); |
| 331 | + if (mode === "apply") return apply(payload); |
| 332 | + throw new Error(`organize: unknown mode "${mode}" (dryRun | apply)`); |
| 333 | + } |
| 334 | + |
| 335 | + return { organize }; |
| 336 | +} |
0 commit comments