Skip to content

Commit 87ab3b0

Browse files
feat(core): transcript recall index for semantic session history
Adds an opt-in embedding index over transcript chunks with a recall tool. - Durable event subscription (EventV2.PartUpdated/MessageUpdated/PartRemoved) - Per-session anchor chunks (title + compaction summaries) - Deterministic hash provider (bag-of-n-grams, zero deps) - Backfill on startup, debounced flush every 2s - Open-coded cosine similarity over stored vectors Closes anomalyco#41354 Co-authored-by: Copilot App <[email protected]>
1 parent 5d5c35e commit 87ab3b0

11 files changed

Lines changed: 499 additions & 0 deletions

File tree

‎packages/core/src/database/migration.gen.ts‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { Effect } from "effect"
2+
import type { DatabaseMigration } from "../migration"
3+
4+
export default {
5+
id: "20260823000000_add_recall_chunk",
6+
up(tx) {
7+
return Effect.gen(function* () {
8+
yield* tx.run(`
9+
CREATE TABLE \`recall_chunk\` (
10+
\`id\` text PRIMARY KEY,
11+
\`session_id\` text NOT NULL,
12+
\`message_id\` text NOT NULL,
13+
\`part_id\` text NOT NULL,
14+
\`chunk_index\` integer NOT NULL,
15+
\`provider\` text NOT NULL,
16+
\`dim\` integer NOT NULL,
17+
\`model_id\` text NOT NULL,
18+
\`text_hash\` text NOT NULL,
19+
\`text\` text NOT NULL,
20+
\`vec\` blob NOT NULL,
21+
\`time_created\` integer NOT NULL,
22+
\`time_updated\` integer NOT NULL
23+
);
24+
`)
25+
yield* tx.run(`CREATE INDEX \`recall_chunk_session_idx\` ON \`recall_chunk\` (\`session_id\`);`)
26+
yield* tx.run(`CREATE INDEX \`recall_chunk_part_idx\` ON \`recall_chunk\` (\`part_id\`);`)
27+
yield* tx.run(`CREATE INDEX \`recall_chunk_message_idx\` ON \`recall_chunk\` (\`message_id\`);`)
28+
})
29+
},
30+
} satisfies DatabaseMigration.Migration

‎packages/core/src/flag/flag.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export const Flag = {
4848

4949
OPENCODE_WORKSPACE_ID: process.env["OPENCODE_WORKSPACE_ID"],
5050
OPENCODE_EXPERIMENTAL_WORKSPACES: enabledByExperimental("OPENCODE_EXPERIMENTAL_WORKSPACES"),
51+
OPENCODE_EXPERIMENTAL_TRANSCRIPT_RECALL: enabledByExperimental("OPENCODE_EXPERIMENTAL_TRANSCRIPT_RECALL"),
5152

5253
// Evaluated at access time (not module load) because tests, the CLI, and
5354
// external tooling set these env vars at runtime.
Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
export * as Recall from "./indexer"
2+
3+
import { and, eq, inArray, sql } from "drizzle-orm"
4+
import { Context, Duration, Effect, Layer, Stream } from "effect"
5+
import { Database } from "../database/database"
6+
import { EventV2 } from "../event"
7+
import { makeGlobalNode } from "../effect/app-node"
8+
import { Flag } from "../flag/flag"
9+
import { SessionV1, type PartID } from "../v1/session"
10+
import type { SessionSchema } from "../session/schema"
11+
12+
type SessionID = SessionSchema.ID
13+
import { MessageTable, PartTable, SessionTable } from "../session/sql"
14+
import { RecallChunkTable } from "./sql"
15+
import { cosine, HashingProvider, textHash, type EmbeddingProvider } from "./provider"
16+
17+
const CHUNK_CHARS = 1200
18+
const FLUSH_MILLIS = 2000
19+
20+
export interface Hit {
21+
readonly sessionID: string
22+
readonly messageID: string
23+
readonly partID: string
24+
readonly text: string
25+
readonly score: number
26+
}
27+
28+
export interface Interface {
29+
/** Semantic recall over indexed transcript chunks. Empty when the feature flag is off. */
30+
readonly search: (input: { query: string; limit?: number }) => Effect.Effect<Hit[]>
31+
}
32+
33+
export class Service extends Context.Service<Service, Interface>()("@opencode/Recall") {}
34+
35+
interface PartRow {
36+
id: PartID
37+
message_id: string
38+
session_id: string
39+
data: unknown
40+
}
41+
42+
function isIndexableText(row: PartRow): row is PartRow & { data: { type: "text"; text: string } } {
43+
const data = row.data as { type?: string; text?: unknown; synthetic?: boolean; ignored?: boolean }
44+
return (
45+
data.type === "text" &&
46+
!data.synthetic &&
47+
!data.ignored &&
48+
typeof data.text === "string" &&
49+
data.text.trim().length > 0
50+
)
51+
}
52+
53+
function chunkText(text: string): string[] {
54+
if (text.length <= CHUNK_CHARS) return [text]
55+
const pieces: string[] = []
56+
for (let i = 0; i < text.length; i += CHUNK_CHARS) pieces.push(text.slice(i, i + CHUNK_CHARS))
57+
return pieces
58+
}
59+
60+
const layer = Layer.effect(
61+
Service,
62+
Effect.gen(function* () {
63+
const events = yield* EventV2.Service
64+
const { db } = yield* Database.Service
65+
const provider: EmbeddingProvider = HashingProvider
66+
// Read at boot; flipping the env var requires a restart, which keeps the
67+
// index lifecycle deterministic for the process.
68+
const enabled = Flag.OPENCODE_EXPERIMENTAL_TRANSCRIPT_RECALL
69+
70+
const deletePart = (partID: PartID) =>
71+
db.delete(RecallChunkTable).where(eq(RecallChunkTable.part_id, partID)).run().pipe(Effect.orDie)
72+
73+
const deleteMessage = (input: { sessionID: string; messageID: string }) =>
74+
db
75+
.delete(RecallChunkTable)
76+
.where(
77+
and(eq(RecallChunkTable.session_id, input.sessionID), eq(RecallChunkTable.message_id, input.messageID)),
78+
)
79+
.run()
80+
.pipe(Effect.orDie)
81+
82+
const deleteSession = (sessionID: string) =>
83+
db.delete(RecallChunkTable).where(eq(RecallChunkTable.session_id, sessionID)).run().pipe(Effect.orDie)
84+
85+
// Re-index the current committed state of the given parts. The event only
86+
// marks parts dirty; the part table is always the source of truth, which
87+
// naturally collapses streaming churn into one final write per flush.
88+
const indexParts = (partIDs: PartID[]) =>
89+
Effect.gen(function* () {
90+
if (partIDs.length === 0) return
91+
const rows = (yield* db.select().from(PartTable).where(inArray(PartTable.id, partIDs)).all().pipe(
92+
Effect.orDie,
93+
)) as PartRow[]
94+
const stale = rows.filter((row) => !isIndexableText(row))
95+
for (const row of stale) yield* deletePart(row.id)
96+
const candidates = rows.filter(isIndexableText)
97+
for (const row of candidates) {
98+
const existing = yield* db
99+
.select({ id: RecallChunkTable.id, text_hash: RecallChunkTable.text_hash })
100+
.from(RecallChunkTable)
101+
.where(eq(RecallChunkTable.part_id, row.id))
102+
.all()
103+
.pipe(Effect.orDie)
104+
const keep = new Set<string>()
105+
const pieces = chunkText(row.data.text)
106+
let changed = false
107+
for (let i = 0; i < pieces.length; i++) {
108+
const piece = pieces[i]
109+
const hash = textHash(piece)
110+
const id = `${row.id}:${i}`
111+
keep.add(id)
112+
if (existing.some((entry) => entry.id === id && entry.text_hash === hash)) continue
113+
changed = true
114+
const vectors = yield* provider.embed([piece])
115+
const values = {
116+
id,
117+
session_id: row.session_id,
118+
message_id: row.message_id,
119+
part_id: row.id,
120+
chunk_index: i,
121+
provider: provider.id,
122+
dim: provider.dim,
123+
model_id: provider.modelID,
124+
text_hash: hash,
125+
text: piece,
126+
vec: Buffer.from(vectors[0].buffer, vectors[0].byteOffset, vectors[0].byteLength),
127+
}
128+
yield* db
129+
.insert(RecallChunkTable)
130+
.values(values)
131+
.onConflictDoUpdate({ target: RecallChunkTable.id, set: values })
132+
.run()
133+
.pipe(Effect.orDie)
134+
}
135+
const removed = existing.filter((entry) => !keep.has(entry.id)).map((entry) => entry.id)
136+
if (removed.length > 0) {
137+
yield* db.delete(RecallChunkTable).where(inArray(RecallChunkTable.id, removed)).run().pipe(Effect.orDie)
138+
}
139+
if (!changed && removed.length === 0) continue
140+
}
141+
})
142+
143+
const touched = new Set<PartID>()
144+
const touchedSessions = new Set<SessionID>()
145+
146+
// One anchor chunk per session holding its title plus any compaction
147+
// summaries. Gives aggregate queries ("what did we decide about X?") a
148+
// high-precision entry point that plain per-part chunks lack.
149+
const indexSessionMeta = (sessionIDs: SessionID[]) =>
150+
Effect.gen(function* () {
151+
for (const sessionID of sessionIDs) {
152+
const metaID = `meta:${sessionID}`
153+
const sessionRow = yield* db
154+
.select({ title: SessionTable.title })
155+
.from(SessionTable)
156+
.where(eq(SessionTable.id, sessionID))
157+
.get()
158+
.pipe(Effect.orDie)
159+
const summaries = yield* db
160+
.select({ data: MessageTable.data })
161+
.from(MessageTable)
162+
.where(
163+
and(
164+
eq(MessageTable.session_id, sessionID),
165+
sql`json_extract(${MessageTable.data}, '$.summary.body') IS NOT NULL`,
166+
),
167+
)
168+
.all()
169+
.pipe(Effect.orDie)
170+
const pieces: string[] = []
171+
if (sessionRow?.title) pieces.push(`Session title: ${sessionRow.title}`)
172+
for (const row of summaries) {
173+
const summary = (row.data as { summary?: { title?: string; body?: string } }).summary
174+
if (summary?.title) pieces.push(`Summary title: ${summary.title}`)
175+
if (summary?.body) pieces.push(summary.body)
176+
}
177+
if (pieces.length === 0) {
178+
yield* db.delete(RecallChunkTable).where(eq(RecallChunkTable.id, metaID)).run().pipe(Effect.orDie)
179+
continue
180+
}
181+
const text = pieces.join("\n\n")
182+
const hash = textHash(text)
183+
const existing = yield* db
184+
.select({ text_hash: RecallChunkTable.text_hash })
185+
.from(RecallChunkTable)
186+
.where(eq(RecallChunkTable.id, metaID))
187+
.get()
188+
.pipe(Effect.orDie)
189+
if (existing?.text_hash === hash) continue
190+
const vectors = yield* provider.embed([text])
191+
const values = {
192+
id: metaID,
193+
session_id: sessionID,
194+
message_id: "meta",
195+
part_id: sessionID,
196+
chunk_index: 0,
197+
provider: provider.id,
198+
dim: provider.dim,
199+
model_id: provider.modelID,
200+
text_hash: hash,
201+
text,
202+
vec: Buffer.from(vectors[0].buffer, vectors[0].byteOffset, vectors[0].byteLength),
203+
}
204+
yield* db
205+
.insert(RecallChunkTable)
206+
.values(values)
207+
.onConflictDoUpdate({ target: RecallChunkTable.id, set: values })
208+
.run()
209+
.pipe(Effect.orDie)
210+
}
211+
})
212+
213+
if (enabled) {
214+
// Backfill: anything in the part table without chunks yet. Chunks are
215+
// keyed deterministically, so overlap with live indexing is harmless.
216+
yield* Effect.gen(function* () {
217+
const indexed = yield* db
218+
.selectDistinct({ part_id: RecallChunkTable.part_id })
219+
.from(RecallChunkTable)
220+
.all()
221+
.pipe(Effect.orDie)
222+
const known = new Set(indexed.map((row) => row.part_id))
223+
const parts = yield* db.select({ id: PartTable.id }).from(PartTable).all().pipe(Effect.orDie)
224+
const missing = parts.map((part) => part.id).filter((id) => !known.has(id))
225+
for (let i = 0; i < missing.length; i += 50) {
226+
yield* indexParts(missing.slice(i, i + 50))
227+
}
228+
const sessions = yield* db.selectDistinct({ id: PartTable.session_id }).from(PartTable).all().pipe(Effect.orDie)
229+
yield* indexSessionMeta(sessions.map((row) => row.id))
230+
}).pipe(
231+
Effect.catchCause((cause) =>
232+
Effect.logWarning("recall backfill failed", { cause: String(cause) }).pipe(Effect.asVoid),
233+
),
234+
Effect.forkScoped,
235+
)
236+
237+
yield* events.subscribe(SessionV1.Event.PartUpdated).pipe(
238+
Stream.runForEach((event) =>
239+
Effect.sync(() => {
240+
touched.add(event.data.part.id)
241+
touchedSessions.add(event.data.part.sessionID)
242+
}),
243+
),
244+
Effect.forkScoped,
245+
)
246+
yield* events.subscribe(SessionV1.Event.Updated).pipe(
247+
Stream.runForEach((event) =>
248+
Effect.sync(() => {
249+
touchedSessions.add(event.data.info.id)
250+
}),
251+
),
252+
Effect.forkScoped,
253+
)
254+
yield* events.subscribe(SessionV1.Event.PartRemoved).pipe(
255+
Stream.runForEach((event) => deletePart(event.data.partID)),
256+
Effect.forkScoped,
257+
)
258+
yield* events.subscribe(SessionV1.Event.MessageRemoved).pipe(
259+
Stream.runForEach((event) => deleteMessage(event.data)),
260+
Effect.forkScoped,
261+
)
262+
yield* events.subscribe(SessionV1.Event.Deleted).pipe(
263+
Stream.runForEach((event) => deleteSession(event.data.sessionID)),
264+
Effect.forkScoped,
265+
)
266+
yield* Effect.forever(
267+
Effect.gen(function* () {
268+
yield* Effect.sleep(Duration.millis(FLUSH_MILLIS))
269+
if (touched.size === 0 && touchedSessions.size === 0) return
270+
const batch = [...touched]
271+
const sessions = [...touchedSessions]
272+
touched.clear()
273+
touchedSessions.clear()
274+
yield* Effect.all([
275+
indexParts(batch).pipe(
276+
Effect.catchCause((cause) =>
277+
Effect.logWarning("recall index flush failed", { cause: String(cause) }).pipe(Effect.asVoid),
278+
),
279+
),
280+
indexSessionMeta(sessions).pipe(
281+
Effect.catchCause((cause) =>
282+
Effect.logWarning("recall meta flush failed", { cause: String(cause) }).pipe(Effect.asVoid),
283+
),
284+
),
285+
], { discard: true })
286+
}),
287+
).pipe(Effect.forkScoped)
288+
}
289+
290+
const search: Interface["search"] = ({ query, limit }) =>
291+
Effect.gen(function* () {
292+
if (!enabled) return []
293+
const trimmed = query.trim()
294+
if (!trimmed) return []
295+
const vectors = yield* provider.embed([trimmed])
296+
const rows = yield* db.select().from(RecallChunkTable).all().pipe(Effect.orDie)
297+
const scored: Hit[] = []
298+
for (const row of rows) {
299+
if (row.dim !== provider.dim || row.provider !== provider.id) continue
300+
const bytes = new Uint8Array(row.vec)
301+
const copy = new Float32Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength))
302+
scored.push({
303+
sessionID: row.session_id,
304+
messageID: row.message_id,
305+
partID: row.part_id,
306+
text: row.text,
307+
score: cosine(vectors[0], copy),
308+
})
309+
}
310+
scored.sort((a, b) => b.score - a.score)
311+
return scored.slice(0, limit ?? 5)
312+
})
313+
314+
return Service.of({ search })
315+
}),
316+
)
317+
318+
export const node = makeGlobalNode({ name: "transcript-recall", layer, deps: [EventV2.node, Database.node] })

0 commit comments

Comments
 (0)