Skip to content

Commit 95e9f62

Browse files
authored
Merge branch 'main' into fix/embed-reasons
2 parents 78523cc + 2e45040 commit 95e9f62

7 files changed

Lines changed: 179 additions & 0 deletions

File tree

‎dsh-mneme/lib/api.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,25 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
10801080
const configured = embedder ? (typeof embedder.configured === "boolean" ? embedder.configured : true) : null;
10811081
const embedded = Number(stats?.embeddedCount ?? 0);
10821082
const total = Number(stats?.totalCount ?? 0);
1083+
// Per-model / per-dimension provenance of stored vectors. "embedded"
1084+
// counts rows with any vector; rows whose dimension differs from the
1085+
// current embedder output are skipped by searchVector, so the honest
1086+
// retrievable number is the current-dimension slice. modelBreakdown
1087+
// groups by row-level embedding_model ('' = unknown origin).
1088+
const dimensionBreakdown = typeof service.embeddingDimensionBreakdown === "function"
1089+
? service.embeddingDimensionBreakdown()
1090+
: null;
1091+
const currentDim = embedder?.dimension != null ? String(embedder.dimension) : null;
1092+
const byDimension = dimensionBreakdown
1093+
? [...dimensionBreakdown.entries()].map(([dim, c]) => ({ dimension: dim === "" ? null : Number(dim), count: c }))
1094+
: [];
1095+
const unsearchable = (currentDim && dimensionBreakdown)
1096+
? [...dimensionBreakdown.entries()]
1097+
.filter(([dim]) => dim !== currentDim)
1098+
.reduce((s, [, c]) => s + c, 0)
1099+
: 0; const modelBreakdown = typeof service.embeddingModelBreakdown === "function"
1100+
? service.embeddingModelBreakdown().map((r) => ({ model: r.model === "" ? null : r.model, count: r.c }))
1101+
: [];
10831102
sendJson(res, 200, {
10841103
ready,
10851104
configured,
@@ -1090,6 +1109,16 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
10901109
: null,
10911110
// 空库时 ratio 必须是 null 而非 0 —— 0 会被读成「一条都没嵌上」。
10921111
coverage: { embedded, total, ratio: total > 0 ? embedded / total : null },
1112+
// distill-tx-embed-gap 后续:向量「存在」≠「可检索」。unsearchable =
1113+
// 维度与当前 embedder 输出不同的行数(searchVector 会跳过它们),
1114+
// retrievable = 与当前维度同维、真正参与召回的行数。两者差值非零时
1115+
// 面板必须能说出「有 N 条向量不可检索」,而不是只给一个虚高的 coverage。
1116+
vectors: {
1117+
byDimension,
1118+
byModel: modelBreakdown,
1119+
unsearchable,
1120+
retrievable: currentDim ? (dimensionBreakdown.get(currentDim) ?? 0) : null,
1121+
},
10931122
// 配了、库里有东西可嵌、却一条都没嵌上 = 真的降级(本机此前长期
10941123
// 处于 ready:true + embeddedCount:0,面板却显示正常)。
10951124
degraded: configured === true && total > 0 && embedded === 0,

‎dsh-mneme/lib/service.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2664,6 +2664,11 @@ export function createService({ store, mirror, config, onWrite, logger, document
26642664
search: (q, o) => store.search(q, o),
26652665
searchVector: (v, o) => store.searchVector(v, o),
26662666
embeddedCount: () => store.embeddedCount(),
2667+
// 向量出身的两个分组读数:状态端点 /semantic 用它们区分「有向量」和
2668+
// 「可检索」(byModel 按行级 embedding_model,byDimension 按向量维数)。
2669+
// passthrough 形状与 embeddedCount 同款,api 层不直接摸 store。
2670+
embeddingModelBreakdown: () => store.embeddingModelBreakdown(),
2671+
embeddingDimensionBreakdown: () => store.embeddingDimensionBreakdown(),
26672672
list: (o) => store.list(o),
26682673
all: () => store.all(),
26692674
count: (type, opts) => store.count(type, opts),

‎dsh-mneme/lib/store.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1775,6 +1775,32 @@ export function createStore(path) {
17751775
).all();
17761776
}
17771777

1778+
/**
1779+
* Per-dimension counts over active embedded rows ('' = vector that failed to
1780+
* parse as an array — legacy garbage rows, not a real dimension).
1781+
*
1782+
* Why per-dimension: embeddedCount() alone says "has a vector", but vectors
1783+
* written by a different model/dimension are not comparable with the current
1784+
* query embedding — searchVector skips them, so they are "present" without
1785+
* being "retrievable". The status card needs this split to stop reporting a
1786+
* coverage number that silently counts unsearchable rows.
1787+
*/
1788+
function embeddingDimensionBreakdown() {
1789+
return db.prepare(
1790+
`SELECT id, embedding FROM memories
1791+
WHERE embedding IS NOT NULL AND embedding != ''
1792+
AND forgotten = 0 AND archived = 0`
1793+
).all().reduce((acc, row) => {
1794+
let dim = "";
1795+
try {
1796+
const parsed = JSON.parse(row.embedding);
1797+
dim = Array.isArray(parsed) ? String(parsed.length) : "";
1798+
} catch { /* unparseable vector counts under "" */ }
1799+
acc.set(dim, (acc.get(dim) ?? 0) + 1);
1800+
return acc;
1801+
}, new Map());
1802+
}
1803+
17781804
function search(query, { limit = 20, includeArchived = false } = {}) {
17791805
const q = String(query).trim();
17801806
if (!q) return [];
@@ -3340,6 +3366,7 @@ export function createStore(path) {
33403366
needsEmbeddingFor,
33413367
hasStaleEmbedding,
33423368
embeddingModelBreakdown,
3369+
embeddingDimensionBreakdown,
33433370
searchVector,
33443371
saveDreamRun,
33453372
lastDreamRunAt,

‎dsh-mneme/src/api.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,25 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
10801080
const configured = embedder ? (typeof embedder.configured === "boolean" ? embedder.configured : true) : null;
10811081
const embedded = Number(stats?.embeddedCount ?? 0);
10821082
const total = Number(stats?.totalCount ?? 0);
1083+
// Per-model / per-dimension provenance of stored vectors. "embedded"
1084+
// counts rows with any vector; rows whose dimension differs from the
1085+
// current embedder output are skipped by searchVector, so the honest
1086+
// retrievable number is the current-dimension slice. modelBreakdown
1087+
// groups by row-level embedding_model ('' = unknown origin).
1088+
const dimensionBreakdown = typeof service.embeddingDimensionBreakdown === "function"
1089+
? service.embeddingDimensionBreakdown()
1090+
: null;
1091+
const currentDim = embedder?.dimension != null ? String(embedder.dimension) : null;
1092+
const byDimension = dimensionBreakdown
1093+
? [...dimensionBreakdown.entries()].map(([dim, c]) => ({ dimension: dim === "" ? null : Number(dim), count: c }))
1094+
: [];
1095+
const unsearchable = (currentDim && dimensionBreakdown)
1096+
? [...dimensionBreakdown.entries()]
1097+
.filter(([dim]) => dim !== currentDim)
1098+
.reduce((s, [, c]) => s + c, 0)
1099+
: 0; const modelBreakdown = typeof service.embeddingModelBreakdown === "function"
1100+
? service.embeddingModelBreakdown().map((r) => ({ model: r.model === "" ? null : r.model, count: r.c }))
1101+
: [];
10831102
sendJson(res, 200, {
10841103
ready,
10851104
configured,
@@ -1090,6 +1109,16 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
10901109
: null,
10911110
// 空库时 ratio 必须是 null 而非 0 —— 0 会被读成「一条都没嵌上」。
10921111
coverage: { embedded, total, ratio: total > 0 ? embedded / total : null },
1112+
// distill-tx-embed-gap 后续:向量「存在」≠「可检索」。unsearchable =
1113+
// 维度与当前 embedder 输出不同的行数(searchVector 会跳过它们),
1114+
// retrievable = 与当前维度同维、真正参与召回的行数。两者差值非零时
1115+
// 面板必须能说出「有 N 条向量不可检索」,而不是只给一个虚高的 coverage。
1116+
vectors: {
1117+
byDimension,
1118+
byModel: modelBreakdown,
1119+
unsearchable,
1120+
retrievable: currentDim ? (dimensionBreakdown.get(currentDim) ?? 0) : null,
1121+
},
10931122
// 配了、库里有东西可嵌、却一条都没嵌上 = 真的降级(本机此前长期
10941123
// 处于 ready:true + embeddedCount:0,面板却显示正常)。
10951124
degraded: configured === true && total > 0 && embedded === 0,

‎dsh-mneme/src/service.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2664,6 +2664,11 @@ export function createService({ store, mirror, config, onWrite, logger, document
26642664
search: (q, o) => store.search(q, o),
26652665
searchVector: (v, o) => store.searchVector(v, o),
26662666
embeddedCount: () => store.embeddedCount(),
2667+
// 向量出身的两个分组读数:状态端点 /semantic 用它们区分「有向量」和
2668+
// 「可检索」(byModel 按行级 embedding_model,byDimension 按向量维数)。
2669+
// passthrough 形状与 embeddedCount 同款,api 层不直接摸 store。
2670+
embeddingModelBreakdown: () => store.embeddingModelBreakdown(),
2671+
embeddingDimensionBreakdown: () => store.embeddingDimensionBreakdown(),
26672672
list: (o) => store.list(o),
26682673
all: () => store.all(),
26692674
count: (type, opts) => store.count(type, opts),

‎dsh-mneme/src/store.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1775,6 +1775,32 @@ export function createStore(path) {
17751775
).all();
17761776
}
17771777

1778+
/**
1779+
* Per-dimension counts over active embedded rows ('' = vector that failed to
1780+
* parse as an array — legacy garbage rows, not a real dimension).
1781+
*
1782+
* Why per-dimension: embeddedCount() alone says "has a vector", but vectors
1783+
* written by a different model/dimension are not comparable with the current
1784+
* query embedding — searchVector skips them, so they are "present" without
1785+
* being "retrievable". The status card needs this split to stop reporting a
1786+
* coverage number that silently counts unsearchable rows.
1787+
*/
1788+
function embeddingDimensionBreakdown() {
1789+
return db.prepare(
1790+
`SELECT id, embedding FROM memories
1791+
WHERE embedding IS NOT NULL AND embedding != ''
1792+
AND forgotten = 0 AND archived = 0`
1793+
).all().reduce((acc, row) => {
1794+
let dim = "";
1795+
try {
1796+
const parsed = JSON.parse(row.embedding);
1797+
dim = Array.isArray(parsed) ? String(parsed.length) : "";
1798+
} catch { /* unparseable vector counts under "" */ }
1799+
acc.set(dim, (acc.get(dim) ?? 0) + 1);
1800+
return acc;
1801+
}, new Map());
1802+
}
1803+
17781804
function search(query, { limit = 20, includeArchived = false } = {}) {
17791805
const q = String(query).trim();
17801806
if (!q) return [];
@@ -3340,6 +3366,7 @@ export function createStore(path) {
33403366
needsEmbeddingFor,
33413367
hasStaleEmbedding,
33423368
embeddingModelBreakdown,
3369+
embeddingDimensionBreakdown,
33433370
searchVector,
33443371
saveDreamRun,
33453372
lastDreamRunAt,

‎dsh-mneme/test/api.test.js‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,63 @@ test("GET /api/dsh-mneme/semantic reports configured/coverage/degraded", async (
538538
assert.equal(noIndex.degraded, false);
539539
});
540540

541+
// --- distill-tx-embed-gap 后续:/semantic 区分「有向量」与「可检索」 -----------
542+
543+
test("GET /api/dsh-mneme/semantic reports per-dimension/per-model vector breakdown", async () => {
544+
const { routes, store, service } = setup(undefined);
545+
// 三种状态混进一个库:3 维 2 条、旧 4 维 1 条、不可解析 1 条;
546+
// 模型出身 m1#aaa 一行、m2#bbb 一行、无出身两行。
547+
const a = service.saveWithDedupe({ type: "preference", title: "同维A", content: "x" }).memory;
548+
const b = service.saveWithDedupe({ type: "preference", title: "同维B", content: "x" }).memory;
549+
const c = service.saveWithDedupe({ type: "preference", title: "旧维C", content: "x" }).memory;
550+
const d = service.saveWithDedupe({ type: "preference", title: "坏D", content: "x" }).memory;
551+
store.db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(JSON.stringify([1, 2, 3]), a.id);
552+
store.db.prepare("UPDATE memories SET embedding = ?, embedding_model = ? WHERE id = ?").run(JSON.stringify([1, 2, 3]), "m1#aaa", b.id);
553+
store.db.prepare("UPDATE memories SET embedding = ?, embedding_model = ? WHERE id = ?").run(JSON.stringify([1, 2, 3, 4]), "m2#bbb", c.id);
554+
store.db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run("not-json", d.id);
555+
556+
const sem = routes.find((r) => r.path === "/api/dsh-mneme/semantic");
557+
const res = new FakeRes();
558+
await sem.handler(req("/api/dsh-mneme/semantic"), res);
559+
assert.equal(res.statusCode, 200);
560+
const data = JSON.parse(res.body);
561+
562+
// 无 embedder → 无当前维度可比对:unsearchable 报 0、retrievable 报 null。
563+
assert.equal(data.vectors.unsearchable, 0);
564+
assert.equal(data.vectors.retrievable, null);
565+
const dims = Object.fromEntries(data.vectors.byDimension.map((e) => [e.dimension, e.count]));
566+
assert.equal(dims["3"], 2);
567+
assert.equal(dims["4"], 1);
568+
assert.equal(dims[null], 1, "unparseable vector counted under null");
569+
const models = Object.fromEntries(data.vectors.byModel.map((e) => [e.model, e.count]));
570+
assert.equal(models["m1#aaa"], 1);
571+
assert.equal(models["m2#bbb"], 1);
572+
assert.equal(models[null], 2, "rows without provenance counted under null");
573+
});
574+
575+
test("GET /api/dsh-mneme/semantic counts stale-dimension rows as unsearchable with a matching embedder", async () => {
576+
// 核心回归锁:库里掺一条错维向量,端点必须报出 unsearchable ≥ 1。
577+
// 旧实现无论怎么混都只给 coverage,状态卡说不出「有向量但检索不到」。
578+
const { routes, store, service } = setup({ name: "OpenAI", embed() {}, dimension: 3 });
579+
const a = service.saveWithDedupe({ type: "preference", title: "同维", content: "x" }).memory;
580+
const b = service.saveWithDedupe({ type: "preference", title: "错维", content: "x" }).memory;
581+
store.db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(JSON.stringify([1, 2, 3]), a.id);
582+
store.db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(JSON.stringify([1, 2, 3, 4, 5, 6, 7, 8]), b.id);
583+
const sem = routes.find((r) => r.path === "/api/dsh-mneme/semantic");
584+
const res = new FakeRes();
585+
await sem.handler(req("/api/dsh-mneme/semantic"), res);
586+
assert.equal(res.statusCode, 200);
587+
const data = JSON.parse(res.body);
588+
assert.equal(data.vectors.unsearchable, 1, "the 8-dim row is not searchable against a 3-dim embedder");
589+
assert.equal(data.vectors.retrievable, 1, "the 3-dim row is searchable");
590+
// coverage 走 semantic.vectorIndex.getStats(本测试未挂 → 0),所以「有向量的行」
591+
// 用 byDimension 总和对照:两条都在「有向量」里,但只有一条可检索 —— 这正是
592+
// unsearchable 必须存在的原因。
593+
const embeddedRows = data.vectors.byDimension.reduce((s, e) => s + e.count, 0);
594+
assert.equal(embeddedRows, 2, "both rows have vectors");
595+
assert.equal(data.vectors.unsearchable, 1, "yet only one is retrievable");
596+
});
597+
541598
// --- Bug8: llm-audit API (pagination + stats) --------------------------------
542599

543600
test("GET /api/dsh-mneme/semantic/llm-audit returns paginated rows", async () => {

0 commit comments

Comments
 (0)