|
| 1 | +import { describe, it } from 'node:test'; |
| 2 | +import assert from 'node:assert/strict'; |
| 3 | +import { setImmediate } from 'node:timers/promises'; |
| 4 | +import { SessionStore } from '../../../src/services/sqlite/SessionStore.js'; |
| 5 | +import { SyncApply, type ChromaSyncLike, type SyncOp } from '../../../src/services/sync/SyncApply.js'; |
| 6 | + |
| 7 | +type IndexedKind = 'observation' | 'summary' | 'prompt'; |
| 8 | +const TABLES = { observation: 'observations', summary: 'session_summaries', prompt: 'user_prompts' }; |
| 9 | +const DOC_TYPES = { observation: 'observation', summary: 'session_summary', prompt: 'user_prompt' }; |
| 10 | +const NOW = 1_750_000_000_000; |
| 11 | + |
| 12 | +function op(seq: number, kind: IndexedKind, originId = '800', deleted = false): SyncOp { |
| 13 | + const common = { memory_session_id: 'remote-memory', project: 'remote-project', created_at_epoch: NOW, prompt_number: 1 }; |
| 14 | + const body = kind === 'observation' ? { ...common, type: 'discovery', title: 'Remote title', narrative: 'Remote memory' } |
| 15 | + : kind === 'summary' ? { ...common, request: 'Remote request', completed: 'Remote completion' } |
| 16 | + : { ...common, content_session_id: 'remote-session', prompt_text: 'Remote prompt', platform_source: 'claude' }; |
| 17 | + const rev = deleted ? '2' : '1'; |
| 18 | + return { |
| 19 | + seq: String(seq), kind, origin_device: 'remote-device', origin_id: originId, |
| 20 | + rev, body: deleted ? 'null' : JSON.stringify(body), server_ts: NOW, |
| 21 | + entity_id: `${kind}:remote-device:${originId}`, entity_rev: rev, |
| 22 | + operation_sha256: (deleted ? 'b' : 'a').repeat(64), deleted, |
| 23 | + }; |
| 24 | +} |
| 25 | + |
| 26 | +function fixture(options: { waitForWrite?: Promise<void>; failDelete?: number } = {}) { |
| 27 | + const store = new SessionStore(':memory:'); |
| 28 | + const docs = new Set<string>(); |
| 29 | + const deletions: Array<[string, number]> = []; |
| 30 | + let writing = false; |
| 31 | + let deletesRemainingToFail = options.failDelete ?? 0; |
| 32 | + const write = async (kind: IndexedKind, id: number) => { |
| 33 | + writing = true; |
| 34 | + await options.waitForWrite; |
| 35 | + docs.add(`${DOC_TYPES[kind]}:${id}`); |
| 36 | + }; |
| 37 | + const chroma = { |
| 38 | + syncObservation: async (id: number) => write('observation', id), |
| 39 | + syncSummary: async (id: number) => write('summary', id), |
| 40 | + syncUserPrompt: async (id: number) => write('prompt', id), |
| 41 | + deleteRowDocuments: async (docType: string, id: number) => { |
| 42 | + const kind = (Object.keys(DOC_TYPES) as IndexedKind[]).find(key => DOC_TYPES[key] === docType)!; |
| 43 | + assert.equal(store.db.query(`SELECT id FROM ${TABLES[kind]} WHERE id = ?`).get(id) ?? null, null, |
| 44 | + 'cleanup must see the committed deletion'); |
| 45 | + deletions.push([docType, id]); |
| 46 | + if (deletesRemainingToFail-- > 0) throw new Error('temporary vector service failure'); |
| 47 | + docs.delete(`${docType}:${id}`); |
| 48 | + return 1; |
| 49 | + }, |
| 50 | + } as unknown as ChromaSyncLike; |
| 51 | + const apply = new SyncApply(store.db, { deviceId: 'local-device', chromaSync: chroma }); |
| 52 | + const rowId = (kind: IndexedKind, originId = '800') => (store.db.query( |
| 53 | + `SELECT id FROM ${TABLES[kind]} WHERE origin_device_id = 'remote-device' AND origin_local_id = ?`, |
| 54 | + ).get(originId) as { id: number }).id; |
| 55 | + return { store, docs, deletions, apply, rowId, writing: () => writing }; |
| 56 | +} |
| 57 | + |
| 58 | +describe('remote tombstones remove indexed fragments after commit', () => { |
| 59 | + for (const kind of ['observation', 'summary', 'prompt'] as const) { |
| 60 | + it(`cleans ${kind} fragments using the replica row id and matching document type`, async () => { |
| 61 | + const f = fixture(); |
| 62 | + try { |
| 63 | + f.apply.applyOps([op(1, kind)]); |
| 64 | + const localId = f.rowId(kind); |
| 65 | + assert.notEqual(localId, 800, 'origin ids belong to another database'); |
| 66 | + await setImmediate(); |
| 67 | + assert.equal(f.docs.size, 1); |
| 68 | + const result = f.apply.applyOps([op(2, kind, '800', true)]); |
| 69 | + assert.equal(result.cursor, '2'); |
| 70 | + await setImmediate(); |
| 71 | + assert.deepEqual(f.deletions, [[DOC_TYPES[kind], localId]]); |
| 72 | + assert.equal(f.docs.size, 0); |
| 73 | + // The head ledger makes redelivery inert, including vector cleanup. |
| 74 | + f.apply.applyOps([{ ...op(3, kind, '800', true) }]); |
| 75 | + await setImmediate(); |
| 76 | + assert.equal(f.deletions.length, 1); |
| 77 | + } finally { f.store.close(); } |
| 78 | + }); |
| 79 | + } |
| 80 | + |
| 81 | + it('waits for an older pending write so it cannot recreate the deleted fragments', async () => { |
| 82 | + let release!: () => void; |
| 83 | + const waitForWrite = new Promise<void>(resolve => { release = resolve; }); |
| 84 | + const f = fixture({ waitForWrite }); |
| 85 | + try { |
| 86 | + f.apply.applyOps([op(1, 'observation')]); |
| 87 | + await setImmediate(); |
| 88 | + assert.equal(f.writing(), true); |
| 89 | + f.apply.applyOps([op(2, 'observation', '800', true)]); |
| 90 | + await setImmediate(); |
| 91 | + assert.equal(f.deletions.length, 0, 'the same-row write still owns its turn'); |
| 92 | + release(); |
| 93 | + await setImmediate(); |
| 94 | + assert.equal(f.deletions.length, 1); |
| 95 | + assert.equal(f.docs.size, 0); |
| 96 | + } finally { release(); await setImmediate(); f.store.close(); } |
| 97 | + }); |
| 98 | + |
| 99 | + it('does not clean vectors for a transaction that rolls back', async () => { |
| 100 | + const f = fixture(); |
| 101 | + try { |
| 102 | + f.apply.applyOps([op(1, 'observation')]); |
| 103 | + await setImmediate(); |
| 104 | + assert.throws(() => f.apply.applyOps([ |
| 105 | + op(2, 'observation', '800', true), op(4, 'observation', '801'), |
| 106 | + ], { requireContiguous: true }), /sequence gap/); |
| 107 | + await setImmediate(); |
| 108 | + assert.equal(f.apply.getCursor(), '1'); |
| 109 | + assert.equal(f.docs.size, 1); |
| 110 | + assert.equal(f.deletions.length, 0); |
| 111 | + assert.ok(f.rowId('observation')); |
| 112 | + } finally { f.store.close(); } |
| 113 | + }); |
| 114 | + |
| 115 | + it('keeps durable sync advancing and later cleanup running after a vector error', async () => { |
| 116 | + const f = fixture({ failDelete: 1 }); |
| 117 | + try { |
| 118 | + f.apply.applyOps([op(1, 'observation'), op(2, 'summary')]); |
| 119 | + await setImmediate(); |
| 120 | + const result = f.apply.applyOps([op(3, 'observation', '800', true), op(4, 'summary', '800', true)]); |
| 121 | + assert.equal(result.cursor, '4'); |
| 122 | + await setImmediate(); |
| 123 | + assert.equal(f.deletions.length, 2); |
| 124 | + assert.equal(f.docs.size, 1, 'one failed best-effort cleanup must not block the next row'); |
| 125 | + assert.equal([...f.docs][0].startsWith('observation:'), true); |
| 126 | + } finally { f.store.close(); } |
| 127 | + }); |
| 128 | +}); |
0 commit comments