|
| 1 | +export * as Recall from "./indexer" |
| 2 | + |
| 3 | +import { and, eq, inArray, sql } from "drizzle-orm" |
| 4 | +import { Context, Duration, Effect, Layer, Stream } from "effect" |
| 5 | +import { Database } from "../database/database" |
| 6 | +import { EventV2 } from "../event" |
| 7 | +import { makeGlobalNode } from "../effect/app-node" |
| 8 | +import { Flag } from "../flag/flag" |
| 9 | +import { SessionV1, type PartID } from "../v1/session" |
| 10 | +import type { SessionSchema } from "../session/schema" |
| 11 | + |
| 12 | +type SessionID = SessionSchema.ID |
| 13 | +import { MessageTable, PartTable, SessionTable } from "../session/sql" |
| 14 | +import { RecallChunkTable } from "./sql" |
| 15 | +import { cosine, HashingProvider, textHash, type EmbeddingProvider } from "./provider" |
| 16 | + |
| 17 | +const CHUNK_CHARS = 1200 |
| 18 | +const FLUSH_MILLIS = 2000 |
| 19 | + |
| 20 | +export interface Hit { |
| 21 | + readonly sessionID: string |
| 22 | + readonly messageID: string |
| 23 | + readonly partID: string |
| 24 | + readonly text: string |
| 25 | + readonly score: number |
| 26 | +} |
| 27 | + |
| 28 | +export interface Interface { |
| 29 | + /** Semantic recall over indexed transcript chunks. Empty when the feature flag is off. */ |
| 30 | + readonly search: (input: { query: string; limit?: number }) => Effect.Effect<Hit[]> |
| 31 | +} |
| 32 | + |
| 33 | +export class Service extends Context.Service<Service, Interface>()("@opencode/Recall") {} |
| 34 | + |
| 35 | +interface PartRow { |
| 36 | + id: PartID |
| 37 | + message_id: string |
| 38 | + session_id: string |
| 39 | + data: unknown |
| 40 | +} |
| 41 | + |
| 42 | +function isIndexableText(row: PartRow): row is PartRow & { data: { type: "text"; text: string } } { |
| 43 | + const data = row.data as { type?: string; text?: unknown; synthetic?: boolean; ignored?: boolean } |
| 44 | + return ( |
| 45 | + data.type === "text" && |
| 46 | + !data.synthetic && |
| 47 | + !data.ignored && |
| 48 | + typeof data.text === "string" && |
| 49 | + data.text.trim().length > 0 |
| 50 | + ) |
| 51 | +} |
| 52 | + |
| 53 | +function chunkText(text: string): string[] { |
| 54 | + if (text.length <= CHUNK_CHARS) return [text] |
| 55 | + const pieces: string[] = [] |
| 56 | + for (let i = 0; i < text.length; i += CHUNK_CHARS) pieces.push(text.slice(i, i + CHUNK_CHARS)) |
| 57 | + return pieces |
| 58 | +} |
| 59 | + |
| 60 | +const layer = Layer.effect( |
| 61 | + Service, |
| 62 | + Effect.gen(function* () { |
| 63 | + const events = yield* EventV2.Service |
| 64 | + const { db } = yield* Database.Service |
| 65 | + const provider: EmbeddingProvider = HashingProvider |
| 66 | + // Read at boot; flipping the env var requires a restart, which keeps the |
| 67 | + // index lifecycle deterministic for the process. |
| 68 | + const enabled = Flag.OPENCODE_EXPERIMENTAL_TRANSCRIPT_RECALL |
| 69 | + |
| 70 | + const deletePart = (partID: PartID) => |
| 71 | + db.delete(RecallChunkTable).where(eq(RecallChunkTable.part_id, partID)).run().pipe(Effect.orDie) |
| 72 | + |
| 73 | + const deleteMessage = (input: { sessionID: string; messageID: string }) => |
| 74 | + db |
| 75 | + .delete(RecallChunkTable) |
| 76 | + .where( |
| 77 | + and(eq(RecallChunkTable.session_id, input.sessionID), eq(RecallChunkTable.message_id, input.messageID)), |
| 78 | + ) |
| 79 | + .run() |
| 80 | + .pipe(Effect.orDie) |
| 81 | + |
| 82 | + const deleteSession = (sessionID: string) => |
| 83 | + db.delete(RecallChunkTable).where(eq(RecallChunkTable.session_id, sessionID)).run().pipe(Effect.orDie) |
| 84 | + |
| 85 | + // Re-index the current committed state of the given parts. The event only |
| 86 | + // marks parts dirty; the part table is always the source of truth, which |
| 87 | + // naturally collapses streaming churn into one final write per flush. |
| 88 | + const indexParts = (partIDs: PartID[]) => |
| 89 | + Effect.gen(function* () { |
| 90 | + if (partIDs.length === 0) return |
| 91 | + const rows = (yield* db.select().from(PartTable).where(inArray(PartTable.id, partIDs)).all().pipe( |
| 92 | + Effect.orDie, |
| 93 | + )) as PartRow[] |
| 94 | + const stale = rows.filter((row) => !isIndexableText(row)) |
| 95 | + for (const row of stale) yield* deletePart(row.id) |
| 96 | + const candidates = rows.filter(isIndexableText) |
| 97 | + for (const row of candidates) { |
| 98 | + const existing = yield* db |
| 99 | + .select({ id: RecallChunkTable.id, text_hash: RecallChunkTable.text_hash }) |
| 100 | + .from(RecallChunkTable) |
| 101 | + .where(eq(RecallChunkTable.part_id, row.id)) |
| 102 | + .all() |
| 103 | + .pipe(Effect.orDie) |
| 104 | + const keep = new Set<string>() |
| 105 | + const pieces = chunkText(row.data.text) |
| 106 | + let changed = false |
| 107 | + for (let i = 0; i < pieces.length; i++) { |
| 108 | + const piece = pieces[i] |
| 109 | + const hash = textHash(piece) |
| 110 | + const id = `${row.id}:${i}` |
| 111 | + keep.add(id) |
| 112 | + if (existing.some((entry) => entry.id === id && entry.text_hash === hash)) continue |
| 113 | + changed = true |
| 114 | + const vectors = yield* provider.embed([piece]) |
| 115 | + const values = { |
| 116 | + id, |
| 117 | + session_id: row.session_id, |
| 118 | + message_id: row.message_id, |
| 119 | + part_id: row.id, |
| 120 | + chunk_index: i, |
| 121 | + provider: provider.id, |
| 122 | + dim: provider.dim, |
| 123 | + model_id: provider.modelID, |
| 124 | + text_hash: hash, |
| 125 | + text: piece, |
| 126 | + vec: Buffer.from(vectors[0].buffer, vectors[0].byteOffset, vectors[0].byteLength), |
| 127 | + } |
| 128 | + yield* db |
| 129 | + .insert(RecallChunkTable) |
| 130 | + .values(values) |
| 131 | + .onConflictDoUpdate({ target: RecallChunkTable.id, set: values }) |
| 132 | + .run() |
| 133 | + .pipe(Effect.orDie) |
| 134 | + } |
| 135 | + const removed = existing.filter((entry) => !keep.has(entry.id)).map((entry) => entry.id) |
| 136 | + if (removed.length > 0) { |
| 137 | + yield* db.delete(RecallChunkTable).where(inArray(RecallChunkTable.id, removed)).run().pipe(Effect.orDie) |
| 138 | + } |
| 139 | + if (!changed && removed.length === 0) continue |
| 140 | + } |
| 141 | + }) |
| 142 | + |
| 143 | + const touched = new Set<PartID>() |
| 144 | + const touchedSessions = new Set<SessionID>() |
| 145 | + |
| 146 | + // One anchor chunk per session holding its title plus any compaction |
| 147 | + // summaries. Gives aggregate queries ("what did we decide about X?") a |
| 148 | + // high-precision entry point that plain per-part chunks lack. |
| 149 | + const indexSessionMeta = (sessionIDs: SessionID[]) => |
| 150 | + Effect.gen(function* () { |
| 151 | + for (const sessionID of sessionIDs) { |
| 152 | + const metaID = `meta:${sessionID}` |
| 153 | + const sessionRow = yield* db |
| 154 | + .select({ title: SessionTable.title }) |
| 155 | + .from(SessionTable) |
| 156 | + .where(eq(SessionTable.id, sessionID)) |
| 157 | + .get() |
| 158 | + .pipe(Effect.orDie) |
| 159 | + const summaries = yield* db |
| 160 | + .select({ data: MessageTable.data }) |
| 161 | + .from(MessageTable) |
| 162 | + .where( |
| 163 | + and( |
| 164 | + eq(MessageTable.session_id, sessionID), |
| 165 | + sql`json_extract(${MessageTable.data}, '$.summary.body') IS NOT NULL`, |
| 166 | + ), |
| 167 | + ) |
| 168 | + .all() |
| 169 | + .pipe(Effect.orDie) |
| 170 | + const pieces: string[] = [] |
| 171 | + if (sessionRow?.title) pieces.push(`Session title: ${sessionRow.title}`) |
| 172 | + for (const row of summaries) { |
| 173 | + const summary = (row.data as { summary?: { title?: string; body?: string } }).summary |
| 174 | + if (summary?.title) pieces.push(`Summary title: ${summary.title}`) |
| 175 | + if (summary?.body) pieces.push(summary.body) |
| 176 | + } |
| 177 | + if (pieces.length === 0) { |
| 178 | + yield* db.delete(RecallChunkTable).where(eq(RecallChunkTable.id, metaID)).run().pipe(Effect.orDie) |
| 179 | + continue |
| 180 | + } |
| 181 | + const text = pieces.join("\n\n") |
| 182 | + const hash = textHash(text) |
| 183 | + const existing = yield* db |
| 184 | + .select({ text_hash: RecallChunkTable.text_hash }) |
| 185 | + .from(RecallChunkTable) |
| 186 | + .where(eq(RecallChunkTable.id, metaID)) |
| 187 | + .get() |
| 188 | + .pipe(Effect.orDie) |
| 189 | + if (existing?.text_hash === hash) continue |
| 190 | + const vectors = yield* provider.embed([text]) |
| 191 | + const values = { |
| 192 | + id: metaID, |
| 193 | + session_id: sessionID, |
| 194 | + message_id: "meta", |
| 195 | + part_id: sessionID, |
| 196 | + chunk_index: 0, |
| 197 | + provider: provider.id, |
| 198 | + dim: provider.dim, |
| 199 | + model_id: provider.modelID, |
| 200 | + text_hash: hash, |
| 201 | + text, |
| 202 | + vec: Buffer.from(vectors[0].buffer, vectors[0].byteOffset, vectors[0].byteLength), |
| 203 | + } |
| 204 | + yield* db |
| 205 | + .insert(RecallChunkTable) |
| 206 | + .values(values) |
| 207 | + .onConflictDoUpdate({ target: RecallChunkTable.id, set: values }) |
| 208 | + .run() |
| 209 | + .pipe(Effect.orDie) |
| 210 | + } |
| 211 | + }) |
| 212 | + |
| 213 | + if (enabled) { |
| 214 | + // Backfill: anything in the part table without chunks yet. Chunks are |
| 215 | + // keyed deterministically, so overlap with live indexing is harmless. |
| 216 | + yield* Effect.gen(function* () { |
| 217 | + const indexed = yield* db |
| 218 | + .selectDistinct({ part_id: RecallChunkTable.part_id }) |
| 219 | + .from(RecallChunkTable) |
| 220 | + .all() |
| 221 | + .pipe(Effect.orDie) |
| 222 | + const known = new Set(indexed.map((row) => row.part_id)) |
| 223 | + const parts = yield* db.select({ id: PartTable.id }).from(PartTable).all().pipe(Effect.orDie) |
| 224 | + const missing = parts.map((part) => part.id).filter((id) => !known.has(id)) |
| 225 | + for (let i = 0; i < missing.length; i += 50) { |
| 226 | + yield* indexParts(missing.slice(i, i + 50)) |
| 227 | + } |
| 228 | + const sessions = yield* db.selectDistinct({ id: PartTable.session_id }).from(PartTable).all().pipe(Effect.orDie) |
| 229 | + yield* indexSessionMeta(sessions.map((row) => row.id)) |
| 230 | + }).pipe( |
| 231 | + Effect.catchCause((cause) => |
| 232 | + Effect.logWarning("recall backfill failed", { cause: String(cause) }).pipe(Effect.asVoid), |
| 233 | + ), |
| 234 | + Effect.forkScoped, |
| 235 | + ) |
| 236 | + |
| 237 | + yield* events.subscribe(SessionV1.Event.PartUpdated).pipe( |
| 238 | + Stream.runForEach((event) => |
| 239 | + Effect.sync(() => { |
| 240 | + touched.add(event.data.part.id) |
| 241 | + touchedSessions.add(event.data.part.sessionID) |
| 242 | + }), |
| 243 | + ), |
| 244 | + Effect.forkScoped, |
| 245 | + ) |
| 246 | + yield* events.subscribe(SessionV1.Event.Updated).pipe( |
| 247 | + Stream.runForEach((event) => |
| 248 | + Effect.sync(() => { |
| 249 | + touchedSessions.add(event.data.info.id) |
| 250 | + }), |
| 251 | + ), |
| 252 | + Effect.forkScoped, |
| 253 | + ) |
| 254 | + yield* events.subscribe(SessionV1.Event.PartRemoved).pipe( |
| 255 | + Stream.runForEach((event) => deletePart(event.data.partID)), |
| 256 | + Effect.forkScoped, |
| 257 | + ) |
| 258 | + yield* events.subscribe(SessionV1.Event.MessageRemoved).pipe( |
| 259 | + Stream.runForEach((event) => deleteMessage(event.data)), |
| 260 | + Effect.forkScoped, |
| 261 | + ) |
| 262 | + yield* events.subscribe(SessionV1.Event.Deleted).pipe( |
| 263 | + Stream.runForEach((event) => deleteSession(event.data.sessionID)), |
| 264 | + Effect.forkScoped, |
| 265 | + ) |
| 266 | + yield* Effect.forever( |
| 267 | + Effect.gen(function* () { |
| 268 | + yield* Effect.sleep(Duration.millis(FLUSH_MILLIS)) |
| 269 | + if (touched.size === 0 && touchedSessions.size === 0) return |
| 270 | + const batch = [...touched] |
| 271 | + const sessions = [...touchedSessions] |
| 272 | + touched.clear() |
| 273 | + touchedSessions.clear() |
| 274 | + yield* Effect.all([ |
| 275 | + indexParts(batch).pipe( |
| 276 | + Effect.catchCause((cause) => |
| 277 | + Effect.logWarning("recall index flush failed", { cause: String(cause) }).pipe(Effect.asVoid), |
| 278 | + ), |
| 279 | + ), |
| 280 | + indexSessionMeta(sessions).pipe( |
| 281 | + Effect.catchCause((cause) => |
| 282 | + Effect.logWarning("recall meta flush failed", { cause: String(cause) }).pipe(Effect.asVoid), |
| 283 | + ), |
| 284 | + ), |
| 285 | + ], { discard: true }) |
| 286 | + }), |
| 287 | + ).pipe(Effect.forkScoped) |
| 288 | + } |
| 289 | + |
| 290 | + const search: Interface["search"] = ({ query, limit }) => |
| 291 | + Effect.gen(function* () { |
| 292 | + if (!enabled) return [] |
| 293 | + const trimmed = query.trim() |
| 294 | + if (!trimmed) return [] |
| 295 | + const vectors = yield* provider.embed([trimmed]) |
| 296 | + const rows = yield* db.select().from(RecallChunkTable).all().pipe(Effect.orDie) |
| 297 | + const scored: Hit[] = [] |
| 298 | + for (const row of rows) { |
| 299 | + if (row.dim !== provider.dim || row.provider !== provider.id) continue |
| 300 | + const bytes = new Uint8Array(row.vec) |
| 301 | + const copy = new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength)) |
| 302 | + scored.push({ |
| 303 | + sessionID: row.session_id, |
| 304 | + messageID: row.message_id, |
| 305 | + partID: row.part_id, |
| 306 | + text: row.text, |
| 307 | + score: cosine(vectors[0], copy), |
| 308 | + }) |
| 309 | + } |
| 310 | + scored.sort((a, b) => b.score - a.score) |
| 311 | + return scored.slice(0, limit ?? 5) |
| 312 | + }) |
| 313 | + |
| 314 | + return Service.of({ search }) |
| 315 | + }), |
| 316 | +) |
| 317 | + |
| 318 | +export const node = makeGlobalNode({ name: "transcript-recall", layer, deps: [EventV2.node, Database.node] }) |
0 commit comments