Skip to content

Commit 5d73a45

Browse files
committed
refactor: replace PGlite with file-based vector store for RAG
- Created file-based vector store using Float32Array + JSON + markdown files - Stores vectors in binary format (vectors.bin) and metadata in index.json - Documents saved as markdown files in documents/ directory - Works globally from any folder (~/.opencode/knowledge/) - Works in Bun compiled binaries (no bunfs filesystem issues) - In-memory cosine similarity search with configurable thresholds - Removed PGlite, pg, and pgvector dependencies - Simplified storage: one embedding per document (no chunking) - Storage path: ~/.opencode/knowledge/ - documents/*.md (human-readable content) - vectors.bin (Float32Array binary) - index.json (metadata + offsets) Fixes anomalyco#414, anomalyco#15032 (PGlite Bun compile incompatibility)
1 parent b7ebe13 commit 5d73a45

7 files changed

Lines changed: 556 additions & 421 deletions

File tree

‎packages/opencode/package.json‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
"@types/babel__core": "7.20.5",
3434
"@types/bun": "catalog:",
3535
"@types/jsdom": "27.0.0",
36-
"@types/pg": "8.15.5",
3736
"@types/turndown": "5.0.5",
3837
"@types/yargs": "17.0.33",
3938
"@typescript/native-preview": "catalog:",
@@ -47,7 +46,6 @@
4746
"@actions/github": "6.0.1",
4847
"@agentclientprotocol/sdk": "0.4.9",
4948
"@clack/prompts": "1.0.0-alpha.1",
50-
"@electric-sql/pglite": "^0.2.0",
5149
"@hono/standard-validator": "0.1.5",
5250
"@hono/zod-validator": "catalog:",
5351
"@modelcontextprotocol/sdk": "1.15.1",
@@ -81,8 +79,6 @@
8179
"ollama": "^0.5.9",
8280
"open": "10.1.2",
8381
"partial-json": "0.1.7",
84-
"pg": "^8.13.1",
85-
"pgvector": "^0.2.0",
8682
"remeda": "catalog:",
8783
"solid-js": "catalog:",
8884
"tiktoken": "^1.0.17",

‎packages/opencode/src/rag/init/init.ts‎

Lines changed: 27 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
import { confirm, log, spinner } from "@clack/prompts"
2-
import { PGlite } from "@electric-sql/pglite"
3-
import { vector } from "@electric-sql/pglite/vector"
4-
import { existsSync, mkdirSync } from "fs"
2+
import { existsSync } from "fs"
53
import { homedir } from "os"
64
import { join } from "path"
75
import type { RagConfig } from "../config"
8-
import { initializeSchema } from "../db/schema"
96
import { OllamaEmbeddings } from "../embeddings/ollama"
7+
import { createFileVectorStore } from "../storage/file-vector-store"
108

119
export interface InitOptions {
1210
config: RagConfig
@@ -16,31 +14,28 @@ export interface InitOptions {
1614
export interface InitResult {
1715
success: boolean
1816
message: string
19-
dbPath?: string
17+
knowledgePath?: string
2018
}
2119

2220
/**
2321
* Initialize RAG system
2422
* - Check Ollama availability
2523
* - Download required models with user consent
26-
* - Initialize database
27-
* - Create schema
24+
* - Initialize file-based vector store
25+
* - Create directory structure
2826
*/
2927
export async function initializeRag(options: InitOptions): Promise<InitResult> {
3028
const { config, force = false } = options
3129

32-
// Expand home directory in path and ensure absolute path
33-
let dbPath = config.database.path.replace(/^~/, homedir())
34-
if (!dbPath.startsWith('/')) {
35-
const { resolve } = await import('path')
36-
dbPath = resolve(dbPath)
37-
}
30+
// Knowledge base path (always global)
31+
const knowledgePath = join(homedir(), ".opencode", "knowledge")
3832

3933
// Check if already initialized
40-
if (!force && existsSync(dbPath)) {
34+
const vectorStore = createFileVectorStore(config.embeddings.dimensions)
35+
if (!force && vectorStore.isInitialized()) {
4136
return {
4237
success: false,
43-
message: `RAG database already exists at ${dbPath}. Use --force to reinitialize.`,
38+
message: `RAG knowledge base already exists at ${knowledgePath}. Use --force to reinitialize.`,
4439
}
4540
}
4641

@@ -150,36 +145,25 @@ export async function initializeRag(options: InitOptions): Promise<InitResult> {
150145
}
151146
}
152147

153-
// Step 4: Initialize database
154-
s.start("Initializing database...")
148+
// Step 4: Initialize file-based vector store
149+
s.start("Initializing vector store...")
155150

156151
try {
157-
// Create database directory if it doesn't exist
158-
mkdirSync(dbPath, { recursive: true })
159-
160-
// Initialize PGlite with vector extension
161-
const db = new PGlite(dbPath, {
162-
extensions: { vector },
163-
})
152+
// Initialize the vector store (creates directory structure and empty files)
153+
await vectorStore.initialize()
164154

165-
// Create schema with configured dimensions
166-
await initializeSchema(db, config.embeddings.dimensions)
167-
168-
// Close database
169-
await db.close()
170-
171-
s.stop("Database initialized")
155+
s.stop("Vector store initialized")
172156

173157
return {
174158
success: true,
175-
message: "RAG Knowledge Base initialized successfully!",
176-
dbPath,
159+
message: `RAG Knowledge Base initialized successfully!\n\nLocation: ${knowledgePath}\n\nYou can now use Perplexity search and other tools to populate your knowledge base.`,
160+
knowledgePath,
177161
}
178162
} catch (error) {
179-
s.stop("Database initialization failed")
163+
s.stop("Vector store initialization failed")
180164
return {
181165
success: false,
182-
message: `Failed to initialize database: ${error instanceof Error ? error.message : String(error)}`,
166+
message: `Failed to initialize vector store: ${error instanceof Error ? error.message : String(error)}`,
183167
}
184168
}
185169
}
@@ -220,25 +204,20 @@ function getModelInfo(model: string): { size: string; description: string } {
220204
}
221205

222206
/**
223-
* Clean up RAG database (for testing/reset)
207+
* Clean up RAG knowledge base (for testing/reset)
224208
*/
225209
export async function cleanupRag(config: RagConfig): Promise<InitResult> {
226-
// Expand home directory in path and ensure absolute path
227-
let dbPath = config.database.path.replace(/^~/, homedir())
228-
if (!dbPath.startsWith('/')) {
229-
const { resolve } = await import('path')
230-
dbPath = resolve(dbPath)
231-
}
210+
const knowledgePath = join(homedir(), ".opencode", "knowledge")
232211

233-
if (!existsSync(dbPath)) {
212+
if (!existsSync(knowledgePath)) {
234213
return {
235214
success: false,
236-
message: `RAG database does not exist at ${dbPath}`,
215+
message: `RAG knowledge base does not exist at ${knowledgePath}`,
237216
}
238217
}
239218

240219
const shouldDelete = await confirm({
241-
message: `Delete RAG database at ${dbPath}? This cannot be undone.`,
220+
message: `Delete RAG knowledge base at ${knowledgePath}? This cannot be undone.`,
242221
initialValue: false,
243222
})
244223

@@ -250,18 +229,17 @@ export async function cleanupRag(config: RagConfig): Promise<InitResult> {
250229
}
251230

252231
try {
253-
// TODO: Add recursive delete once we confirm safe path
254232
const { rmSync } = await import("fs")
255-
rmSync(dbPath, { recursive: true, force: true })
233+
rmSync(knowledgePath, { recursive: true, force: true })
256234

257235
return {
258236
success: true,
259-
message: "RAG database deleted successfully.",
237+
message: "RAG knowledge base deleted successfully.",
260238
}
261239
} catch (error) {
262240
return {
263241
success: false,
264-
message: `Failed to delete database: ${error instanceof Error ? error.message : String(error)}`,
242+
message: `Failed to delete knowledge base: ${error instanceof Error ? error.message : String(error)}`,
265243
}
266244
}
267245
}

0 commit comments

Comments
 (0)