Skip to content

Commit d58a11d

Browse files
Afloat16thedotmackclaude
authored
fix(sync): remove replica vectors after remote tombstones (#5647)
* fix(sync): remove replica vectors after remote tombstones * build: synchronize verified runtime artifacts * chore: drop generated bundles (rebuilt at release) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_019FbY8a4P6RjaWFnrGVZXnd --------- Co-authored-by: Alex Newman <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent e7978b0 commit d58a11d

4 files changed

Lines changed: 183 additions & 19 deletions

File tree

‎src/services/sync/SyncApply.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ import { getRedactionConfig, redactJsonStrings, redactSensitive, type RedactionC
191191
import { redactSyncDiagnostic } from './diagnostic-redaction.js';
192192
import { isSharedWorkList } from '../work-view/shared-records.mjs';
193193
import { redactSharedInboundRow, saveSharedRedactionReceipt } from '../work-view/shared-redaction.mjs';
194+
import { scheduleChromaRowCleanup, type IndexedRowKind } from './chroma-row-cleanup.js';
194195

195196
/** sync_state key: replaySetAsideWorkState has run on this database. */
196197
const WORK_STATE_REPLAY_MARKER = 'work_state_set_aside_replayed';
@@ -306,6 +307,7 @@ export interface SyncOp {
306307
* it and Phase 3 can pass the real instance.
307308
*/
308309
export interface ChromaSyncLike {
310+
deleteRowDocuments?(docType: string, sqliteId: number): Promise<unknown>;
309311
syncObservation(
310312
observationId: number,
311313
memorySessionId: string,
@@ -985,9 +987,23 @@ export class SyncApply {
985987
: op.kind === 'work_state'
986988
? 'work_state_entries'
987989
: 'user_prompts';
990+
// The origin ID belongs to another database. Capture this replica's
991+
// row ID before deletion so only its own fragments are removed.
992+
const existing = op.kind === 'work_state' ? undefined : this.findByOrigin(table, op.origin_device, op.origin_id);
988993
this.db.prepare(
989994
`DELETE FROM ${table} WHERE origin_device_id = ? AND origin_local_id = ?`
990995
).run(op.origin_device, op.origin_id);
996+
const chroma = this.chromaSync;
997+
if (existing && chroma?.deleteRowDocuments) {
998+
const kind = op.kind as IndexedRowKind;
999+
const row = { kind, id: existing.id };
1000+
const deleter = { deleteRowDocuments: chroma.deleteRowDocuments.bind(chroma) };
1001+
// The normal post-commit dispatch discards this work on rollback.
1002+
// Follow earlier writes of the same row, then join the cleanup lane
1003+
// shared with local deletes; neither path can resurrect fragments.
1004+
chromaJobs.push(() => this.enqueueChromaWrite(`${kind}:${existing.id}`,
1005+
() => scheduleChromaRowCleanup(deleter, [row])));
1006+
}
9911007
outcome = 'applied';
9921008
} else {
9931009
outcome = this.applyRowOp(op, chromaJobs);
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { logger } from '../../utils/logger.js';
2+
import type { ContentKind } from './CanonicalContent.js';
3+
4+
export type IndexedRowKind = Exclude<ContentKind, 'work_state'>;
5+
6+
const CHROMA_DOC_TYPES: Record<IndexedRowKind, string> = {
7+
observation: 'observation',
8+
summary: 'session_summary',
9+
prompt: 'user_prompt',
10+
};
11+
12+
interface ChromaRowDeleter {
13+
deleteRowDocuments(docType: string, sqliteId: number): Promise<unknown>;
14+
}
15+
16+
// Local deletes and replicated tombstones share the same cleanup lane. A
17+
// burst must not flood chroma-mcp's single stdio channel with parallel reads.
18+
let cleanupTail: Promise<void> = Promise.resolve();
19+
20+
/** Rows must already be committed gone; failures never undo durable deletion. */
21+
export function scheduleChromaRowCleanup(
22+
chroma: ChromaRowDeleter,
23+
rows: ReadonlyArray<{ kind: IndexedRowKind; id: string | number }>,
24+
): Promise<void> {
25+
if (rows.length === 0) return cleanupTail;
26+
const run = async (): Promise<void> => {
27+
for (const row of rows) {
28+
try {
29+
await chroma.deleteRowDocuments(CHROMA_DOC_TYPES[row.kind], Number(row.id));
30+
} catch (error: unknown) {
31+
logger.warn('CHROMA_SYNC', 'Post-delete Chroma cleanup failed', { kind: row.kind, sqlite_id: row.id, error });
32+
}
33+
}
34+
};
35+
cleanupTail = cleanupTail.then(run, run);
36+
return cleanupTail;
37+
}

‎src/services/worker/http/routes/DataRoutes.ts‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,7 @@ import { emitContextInvalidation } from '../../../../shared/context-invalidation
2929
import { parseProjectList } from '../../../../shared/project-list.js';
3030
import { parseStringListField } from '../../../sync/string-list-field.js';
3131
import { projectReadKeysFor } from '../../search/project-where-filter.js';
32-
33-
// Chroma doc_type per content kind. Observation and summary id spaces overlap,
34-
// so the delete-route cleanup must be scoped by doc_type, not sqlite_id alone.
35-
type IndexedRowKind = Exclude<ContentKind, 'work_state'>;
36-
const CHROMA_DOC_TYPES: Record<IndexedRowKind, string> = {
37-
observation: 'observation',
38-
summary: 'session_summary',
39-
prompt: 'user_prompt',
40-
};
32+
import { scheduleChromaRowCleanup, type IndexedRowKind } from '../../../sync/chroma-row-cleanup.js';
4133
import { withObservationMedia } from '../../../media/feed.js';
4234

4335
// The MCP get_observations schema advertises `number | string` ids, so array
@@ -524,16 +516,7 @@ export class DataRoutes extends BaseRouteHandler {
524516
private scheduleChromaRowCleanup(rows: ReadonlyArray<{ kind: IndexedRowKind; id: string }>): void {
525517
const chromaSync = this.dbManager.getChromaSync();
526518
if (!chromaSync || rows.length === 0) return;
527-
const run = async (): Promise<void> => {
528-
for (const row of rows) {
529-
try {
530-
await chromaSync.deleteRowDocuments(CHROMA_DOC_TYPES[row.kind], Number(row.id));
531-
} catch (error: unknown) {
532-
logger.warn('CHROMA_SYNC', 'Post-delete Chroma cleanup failed', { kind: row.kind, sqlite_id: row.id, error });
533-
}
534-
}
535-
};
536-
this.chromaCleanupTail = this.chromaCleanupTail.then(run, run);
519+
this.chromaCleanupTail = scheduleChromaRowCleanup(chromaSync, rows);
537520
}
538521

539522
/** Production deletion surface: safety check and row delete for a single content row. */
Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
import { describe, it } from 'node:test';
2+
import assert from 'node:assert/strict';
3+
import { setImmediate } from 'node:timers/promises';
4+
import { SessionStore } from '../../../src/services/sqlite/SessionStore.js';
5+
import { SyncApply, type ChromaSyncLike, type SyncOp } from '../../../src/services/sync/SyncApply.js';
6+
7+
type IndexedKind = 'observation' | 'summary' | 'prompt';
8+
const TABLES = { observation: 'observations', summary: 'session_summaries', prompt: 'user_prompts' };
9+
const DOC_TYPES = { observation: 'observation', summary: 'session_summary', prompt: 'user_prompt' };
10+
const NOW = 1_750_000_000_000;
11+
12+
function op(seq: number, kind: IndexedKind, originId = '800', deleted = false): SyncOp {
13+
const common = { memory_session_id: 'remote-memory', project: 'remote-project', created_at_epoch: NOW, prompt_number: 1 };
14+
const body = kind === 'observation' ? { ...common, type: 'discovery', title: 'Remote title', narrative: 'Remote memory' }
15+
: kind === 'summary' ? { ...common, request: 'Remote request', completed: 'Remote completion' }
16+
: { ...common, content_session_id: 'remote-session', prompt_text: 'Remote prompt', platform_source: 'claude' };
17+
const rev = deleted ? '2' : '1';
18+
return {
19+
seq: String(seq), kind, origin_device: 'remote-device', origin_id: originId,
20+
rev, body: deleted ? 'null' : JSON.stringify(body), server_ts: NOW,
21+
entity_id: `${kind}:remote-device:${originId}`, entity_rev: rev,
22+
operation_sha256: (deleted ? 'b' : 'a').repeat(64), deleted,
23+
};
24+
}
25+
26+
function fixture(options: { waitForWrite?: Promise<void>; failDelete?: number } = {}) {
27+
const store = new SessionStore(':memory:');
28+
const docs = new Set<string>();
29+
const deletions: Array<[string, number]> = [];
30+
let writing = false;
31+
let deletesRemainingToFail = options.failDelete ?? 0;
32+
const write = async (kind: IndexedKind, id: number) => {
33+
writing = true;
34+
await options.waitForWrite;
35+
docs.add(`${DOC_TYPES[kind]}:${id}`);
36+
};
37+
const chroma = {
38+
syncObservation: async (id: number) => write('observation', id),
39+
syncSummary: async (id: number) => write('summary', id),
40+
syncUserPrompt: async (id: number) => write('prompt', id),
41+
deleteRowDocuments: async (docType: string, id: number) => {
42+
const kind = (Object.keys(DOC_TYPES) as IndexedKind[]).find(key => DOC_TYPES[key] === docType)!;
43+
assert.equal(store.db.query(`SELECT id FROM ${TABLES[kind]} WHERE id = ?`).get(id) ?? null, null,
44+
'cleanup must see the committed deletion');
45+
deletions.push([docType, id]);
46+
if (deletesRemainingToFail-- > 0) throw new Error('temporary vector service failure');
47+
docs.delete(`${docType}:${id}`);
48+
return 1;
49+
},
50+
} as unknown as ChromaSyncLike;
51+
const apply = new SyncApply(store.db, { deviceId: 'local-device', chromaSync: chroma });
52+
const rowId = (kind: IndexedKind, originId = '800') => (store.db.query(
53+
`SELECT id FROM ${TABLES[kind]} WHERE origin_device_id = 'remote-device' AND origin_local_id = ?`,
54+
).get(originId) as { id: number }).id;
55+
return { store, docs, deletions, apply, rowId, writing: () => writing };
56+
}
57+
58+
describe('remote tombstones remove indexed fragments after commit', () => {
59+
for (const kind of ['observation', 'summary', 'prompt'] as const) {
60+
it(`cleans ${kind} fragments using the replica row id and matching document type`, async () => {
61+
const f = fixture();
62+
try {
63+
f.apply.applyOps([op(1, kind)]);
64+
const localId = f.rowId(kind);
65+
assert.notEqual(localId, 800, 'origin ids belong to another database');
66+
await setImmediate();
67+
assert.equal(f.docs.size, 1);
68+
const result = f.apply.applyOps([op(2, kind, '800', true)]);
69+
assert.equal(result.cursor, '2');
70+
await setImmediate();
71+
assert.deepEqual(f.deletions, [[DOC_TYPES[kind], localId]]);
72+
assert.equal(f.docs.size, 0);
73+
// The head ledger makes redelivery inert, including vector cleanup.
74+
f.apply.applyOps([{ ...op(3, kind, '800', true) }]);
75+
await setImmediate();
76+
assert.equal(f.deletions.length, 1);
77+
} finally { f.store.close(); }
78+
});
79+
}
80+
81+
it('waits for an older pending write so it cannot recreate the deleted fragments', async () => {
82+
let release!: () => void;
83+
const waitForWrite = new Promise<void>(resolve => { release = resolve; });
84+
const f = fixture({ waitForWrite });
85+
try {
86+
f.apply.applyOps([op(1, 'observation')]);
87+
await setImmediate();
88+
assert.equal(f.writing(), true);
89+
f.apply.applyOps([op(2, 'observation', '800', true)]);
90+
await setImmediate();
91+
assert.equal(f.deletions.length, 0, 'the same-row write still owns its turn');
92+
release();
93+
await setImmediate();
94+
assert.equal(f.deletions.length, 1);
95+
assert.equal(f.docs.size, 0);
96+
} finally { release(); await setImmediate(); f.store.close(); }
97+
});
98+
99+
it('does not clean vectors for a transaction that rolls back', async () => {
100+
const f = fixture();
101+
try {
102+
f.apply.applyOps([op(1, 'observation')]);
103+
await setImmediate();
104+
assert.throws(() => f.apply.applyOps([
105+
op(2, 'observation', '800', true), op(4, 'observation', '801'),
106+
], { requireContiguous: true }), /sequence gap/);
107+
await setImmediate();
108+
assert.equal(f.apply.getCursor(), '1');
109+
assert.equal(f.docs.size, 1);
110+
assert.equal(f.deletions.length, 0);
111+
assert.ok(f.rowId('observation'));
112+
} finally { f.store.close(); }
113+
});
114+
115+
it('keeps durable sync advancing and later cleanup running after a vector error', async () => {
116+
const f = fixture({ failDelete: 1 });
117+
try {
118+
f.apply.applyOps([op(1, 'observation'), op(2, 'summary')]);
119+
await setImmediate();
120+
const result = f.apply.applyOps([op(3, 'observation', '800', true), op(4, 'summary', '800', true)]);
121+
assert.equal(result.cursor, '4');
122+
await setImmediate();
123+
assert.equal(f.deletions.length, 2);
124+
assert.equal(f.docs.size, 1, 'one failed best-effort cleanup must not block the next row');
125+
assert.equal([...f.docs][0].startsWith('observation:'), true);
126+
} finally { f.store.close(); }
127+
});
128+
});

0 commit comments

Comments
 (0)