Skip to content

Commit 6c4ad81

Browse files
authored
fix(turso): multiprocess_wal for concurrent sessions + re-embed recovery harden (#358)
* fix(turso): enable multiprocess_wal and harden re-embed recovery Allow concurrent OpenCode sessions on macOS/Linux via Turso multiprocess_wal, handle .db-tshm sidecars, clarify lock errors, and make interrupted re-embed swap recovery Windows-safe (#350, #357). * fix(i18n): fill missing Dutch keys for UI catalog parity Package smoke failed because nl was missing 14 keys added with the explorer overhaul (filter/nav/confirm/profile-meta). * fix(turso): enable multiprocess_wal only on Unix Windows rejects the flag with "not supported by the active IO backend"; keep encryption-only opens there so package smoke stays green.
1 parent 960dbb3 commit 6c4ad81

10 files changed

Lines changed: 260 additions & 36 deletions

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ This plugin uses embedded Turso (`@tursodatabase/database`) with `F32_BLOB` vect
3636

3737
Startup recovers interrupted re-embed swaps, converts libSQL DiskANN indexes to the current Turso engine, and then verifies or upgrades the legacy shard schema. Engine conversion runs even when a store already has a completed legacy migration marker, and preserves stored vectors without re-embedding. Each converted database is backed up as `<database>.pre-tursodb-<timestamp>.bak`.
3838

39+
On macOS and Linux, multiple OpenCode sessions can share the same `storagePath` via Turso’s experimental `multiprocess_wal` (every process must use the same mode — restart all sessions after upgrading). On Windows the engine rejects that flag, so only one OpenCode session can own the memory databases at a time.
40+
3941
On first startup after upgrading, opencode-mem automatically migrates existing memory shard databases to native Turso/libSQL vector format:
4042

4143
- Each shard is backed up as `<shard>.db.legacy.bak` before rewrite

‎src/index.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,11 +324,18 @@ export const OpenCodeMemPlugin: Plugin = async (ctx: PluginInput) => {
324324
} catch (error) {
325325
log("Turso ready gate failed before web server start", { error: String(error) });
326326
if (ctx.client?.tui) {
327+
const { isTursoMultiProcessLockError } =
328+
await import("./services/turso/connection-manager.js");
329+
const lockHeld = isTursoMultiProcessLockError(error);
327330
ctx.client.tui
328331
.showToast({
329332
body: {
330333
title: "Memory Explorer",
331-
message: "Database migration failed; web UI not started",
334+
message: lockHeld
335+
? process.platform === "win32"
336+
? "Memory DB locked by another OpenCode session (Windows is single-owner)"
337+
: "Memory DB locked by another session — close it or restart all OpenCode windows"
338+
: "Database migration failed; web UI not started",
332339
variant: "error",
333340
duration: 8000,
334341
},

‎src/services/turso/connection-manager.ts‎

Lines changed: 56 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,29 @@ import { resolveOrCreateDatabaseEncryptionKey } from "./encryption-key.js";
1010

1111
export type ConnectFactory = (path: string, opts?: DatabaseOpts) => Promise<Database>;
1212

13+
/** Always-on experimental flags for every Turso open. */
14+
export const TURSO_BASE_EXPERIMENTAL_FEATURES = ["encryption"] as const;
15+
16+
/**
17+
* Multiprocess WAL is Unix-only. On Windows the default IO backend rejects the
18+
* flag (`experimental multiprocess WAL is not supported by the active IO backend`).
19+
*/
20+
export function supportsTursoMultiprocessWal(
21+
platform: NodeJS.Platform = process.platform
22+
): boolean {
23+
return platform !== "win32";
24+
}
25+
26+
/** Experimental flags required on every Turso open for the current platform. */
27+
export function tursoExperimentalFeatures(
28+
platform: NodeJS.Platform = process.platform
29+
): Array<(typeof TURSO_BASE_EXPERIMENTAL_FEATURES)[number] | "multiprocess_wal"> {
30+
if (supportsTursoMultiprocessWal(platform)) {
31+
return [...TURSO_BASE_EXPERIMENTAL_FEATURES, "multiprocess_wal"];
32+
}
33+
return [...TURSO_BASE_EXPERIMENTAL_FEATURES];
34+
}
35+
1336
function assertPathInsideStorage(dbPath: string): void {
1437
const storageRoot = resolve(CONFIG.storagePath);
1538
const resolvedPath = resolve(dbPath);
@@ -20,9 +43,14 @@ function assertPathInsideStorage(dbPath: string): void {
2043
}
2144
}
2245

23-
function buildConnectOptions(encryption?: EncryptionOpts | null): DatabaseOpts {
46+
/**
47+
* Shared connect options for `@tursodatabase/database`.
48+
* Always enables encryption; adds multiprocess_wal on Unix so concurrent
49+
* OpenCode sessions can share the store. Windows stays single-process.
50+
*/
51+
export function buildConnectOptions(encryption?: EncryptionOpts | null): DatabaseOpts {
2452
const opts: DatabaseOpts = {
25-
experimental: ["encryption"],
53+
experimental: tursoExperimentalFeatures(),
2654
};
2755
if (encryption) {
2856
opts.encryption = encryption;
@@ -39,6 +67,28 @@ export function resolveDatabaseEncryption(): EncryptionOpts | null {
3967
};
4068
}
4169

70+
const LOCK_ERROR_RE =
71+
/File is locked by another process|already open (with|without) experimental multiprocess WAL|Locking error/i;
72+
73+
export function isTursoMultiProcessLockError(error: unknown): boolean {
74+
const message = error instanceof Error ? error.message : String(error);
75+
return LOCK_ERROR_RE.test(message);
76+
}
77+
78+
export function wrapTursoOpenError(dbPath: string, error: unknown): Error {
79+
const message = error instanceof Error ? error.message : String(error);
80+
if (isTursoMultiProcessLockError(error)) {
81+
const platformHint =
82+
process.platform === "win32"
83+
? "On Windows the Turso engine still allows only one process as database owner — close other OpenCode sessions."
84+
: "Another OpenCode session (or an older opencode-mem without multiprocess_wal) still holds the database — close it and retry.";
85+
return new Error(`Failed to open database ${dbPath}: ${message}. ${platformHint}`, {
86+
cause: error,
87+
});
88+
}
89+
return error instanceof Error ? error : new Error(message, { cause: error });
90+
}
91+
4292
export class TursoConnectionManager {
4393
private readonly connections = new Map<string, TursoDb>();
4494
private readonly pending = new Map<string, Promise<TursoDb>>();
@@ -92,13 +142,16 @@ export class TursoConnectionManager {
92142
}
93143
if (encryption) {
94144
const message = error instanceof Error ? error.message : String(error);
145+
if (isTursoMultiProcessLockError(error)) {
146+
throw wrapTursoOpenError(dbPath, error);
147+
}
95148
throw new Error(
96149
`Failed to open encrypted database ${dbPath}: ${message}. ` +
97150
`Check databaseEncryptionKey / cipher, or remove encryption config for plaintext shards.`,
98151
{ cause: error }
99152
);
100153
}
101-
throw error;
154+
throw wrapTursoOpenError(dbPath, error);
102155
}
103156
})();
104157

‎src/services/turso/encryption-migrator.ts‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,11 @@ import { existsSync, readdirSync, writeFileSync } from "node:fs";
33
import { join } from "node:path";
44
import { CONFIG } from "../../config.js";
55
import { log } from "../logger.js";
6-
import { resolveDatabaseEncryption, tursoConnectionManager } from "./connection-manager.js";
6+
import {
7+
resolveDatabaseEncryption,
8+
tursoConnectionManager,
9+
buildConnectOptions,
10+
} from "./connection-manager.js";
711
import { acquireTursoOperationLock } from "./operation-lock.js";
812
import {
913
withSqliteFileLockRetry,
@@ -51,10 +55,7 @@ async function isAlreadyEncrypted(dbPath: string): Promise<boolean> {
5155
const encryption = resolveDatabaseEncryption();
5256
if (!encryption) return false;
5357
try {
54-
const db = await connect(dbPath, {
55-
encryption,
56-
experimental: ["encryption"],
57-
});
58+
const db = await connect(dbPath, buildConnectOptions(encryption));
5859
await db.close();
5960
return true;
6061
} catch {
@@ -64,7 +65,7 @@ async function isAlreadyEncrypted(dbPath: string): Promise<boolean> {
6465

6566
async function canOpenPlain(dbPath: string): Promise<boolean> {
6667
try {
67-
const db = await connect(dbPath, { experimental: ["encryption"] });
68+
const db = await connect(dbPath, buildConnectOptions());
6869
await db.close();
6970
return true;
7071
} catch {
@@ -84,12 +85,9 @@ async function dumpAndReloadEncrypted(dbPath: string): Promise<void> {
8485
const backupPath = `${dbPath}.pre-encrypt-${Date.now()}.bak`;
8586

8687
// Re-open plain to copy rows into encrypted staged DB.
87-
const source = await connect(dbPath, { experimental: ["encryption"] });
88+
const source = await connect(dbPath, buildConnectOptions());
8889
const sourceDb = new TursoDb(source);
89-
const staged = await connect(stagedPath, {
90-
encryption,
91-
experimental: ["encryption"],
92-
});
90+
const staged = await connect(stagedPath, buildConnectOptions(encryption));
9391
const stagedDb = new TursoDb(staged);
9492

9593
try {
@@ -214,7 +212,7 @@ async function dumpAndReloadEncrypted(dbPath: string): Promise<void> {
214212
});
215213

216214
// Verify encrypted open
217-
const verify = await connect(dbPath, { encryption, experimental: ["encryption"] });
215+
const verify = await connect(dbPath, buildConnectOptions(encryption));
218216
await verify.close();
219217

220218
log("Encrypted database at rest", { dbPath, backupPath });

‎src/services/turso/engine-migrator.ts‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ import {
1212
removeSqliteDatabase,
1313
collectReleasedSqliteHandles,
1414
} from "./sqlite-handle-release.js";
15-
import { tursoConnectionManager, resolveDatabaseEncryption } from "./connection-manager.js";
15+
import {
16+
tursoConnectionManager,
17+
resolveDatabaseEncryption,
18+
buildConnectOptions,
19+
} from "./connection-manager.js";
1620
import { TursoDb } from "./turso-db.js";
1721
import { tursoShardManager } from "./shard-manager.js";
1822
import { parseExtractedVector } from "./vector-utils.js";
@@ -52,10 +56,7 @@ function collectCandidateDbs(): string[] {
5256
async function needsEngineRewrite(dbPath: string): Promise<boolean> {
5357
try {
5458
const encryption = resolveDatabaseEncryption();
55-
const opts = encryption
56-
? { encryption, experimental: ["encryption" as const] }
57-
: { experimental: ["encryption" as const] };
58-
const db = await connect(dbPath, opts);
59+
const db = await connect(dbPath, buildConnectOptions(encryption));
5960
try {
6061
await db.all("SELECT 1");
6162
return false;
@@ -222,10 +223,7 @@ async function rewriteGenericDb(dbPath: string, tables: string[]): Promise<void>
222223

223224
await tursoConnectionManager.closeConnection(dbPath);
224225
const encryption = resolveDatabaseEncryption();
225-
const opts = encryption
226-
? { encryption, experimental: ["encryption" as const] }
227-
: { experimental: ["encryption" as const] };
228-
const stagedNative = await connect(stagedPath, opts);
226+
const stagedNative = await connect(stagedPath, buildConnectOptions(encryption));
229227
const staged = new TursoDb(stagedNative);
230228
try {
231229
// Recreate schema by copying CREATE SQL from source via libsql.

‎src/services/turso/legacy-migrator.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ async function allShardsComplete(): Promise<boolean> {
513513
return true;
514514
}
515515

516-
function recoverInterruptedReembedSwaps(): void {
516+
async function recoverInterruptedReembedSwaps(): Promise<void> {
517517
for (const dirName of ["users", "projects"] as const) {
518518
const dir = join(CONFIG.storagePath, dirName);
519519
if (!existsSync(dir)) continue;
@@ -543,11 +543,21 @@ function recoverInterruptedReembedSwaps(): void {
543543
}
544544

545545
if (existsSync(expectedDbPath)) {
546-
if (existsSync(stagedPath)) removeSqliteDatabase(stagedPath);
546+
if (existsSync(stagedPath)) {
547+
await withSqliteFileLockRetry(() => removeSqliteDatabase(stagedPath));
548+
}
547549
} else if (existsSync(stagedPath)) {
548-
renameSqliteDatabase(stagedPath, expectedDbPath);
550+
// Target is absent after a crash mid-swap; retry covers Windows EPERM
551+
// while native handles release (same pattern as engine/encryption swaps).
552+
await withSqliteFileLockRetry(() => {
553+
if (existsSync(expectedDbPath)) removeSqliteDatabase(expectedDbPath);
554+
renameSqliteDatabase(stagedPath, expectedDbPath);
555+
});
549556
} else if (existsSync(backupPath)) {
550-
renameSqliteDatabase(backupPath, expectedDbPath);
557+
await withSqliteFileLockRetry(() => {
558+
if (existsSync(expectedDbPath)) removeSqliteDatabase(expectedDbPath);
559+
renameSqliteDatabase(backupPath, expectedDbPath);
560+
});
551561
} else {
552562
throw new Error("neither staged replacement nor source backup exists");
553563
}
@@ -612,7 +622,7 @@ export async function runLegacyTursoMigration(): Promise<void> {
612622
mkdirSync(CONFIG.storagePath, { recursive: true });
613623
}
614624

615-
recoverInterruptedReembedSwaps();
625+
await recoverInterruptedReembedSwaps();
616626

617627
// Recovery may put a libSQL shard back at its active path. Convert DiskANN
618628
// indexes before any legacy verification opens it with the Turso engine,

‎src/services/turso/sqlite-handle-release.ts‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@ const GC_PASSES = 3;
1111
const FILE_LOCK_RETRY_DELAYS_MS = [25, 50, 100, 200, 400, 800, 1600, 3200];
1212
export const RETRYABLE_FILE_LOCK_CODES = new Set(["EBUSY", "EPERM", "EACCES"]);
1313

14-
const SQLITE_SIDE_SUFFIXES = ["-wal", "-shm"] as const;
14+
const SQLITE_SIDE_SUFFIXES = ["-wal", "-shm", "-tshm"] as const;
1515

1616
/**
17-
* Rename a SQLite/Turso database file together with its WAL/SHM sidecars.
17+
* Rename a SQLite/Turso database file together with its WAL/SHM/TSHM sidecars.
1818
* Leaving a `.tmp-wal` behind after renaming only the main file drops commits.
19+
* `-tshm` is Turso multiprocess WAL shared memory (multiprocess_wal).
1920
*/
2021
export function renameSqliteDatabase(fromPath: string, toPath: string): void {
2122
renameSync(fromPath, toPath);
@@ -32,7 +33,7 @@ export function renameSqliteDatabase(fromPath: string, toPath: string): void {
3233
}
3334

3435
/**
35-
* Copy a SQLite/Turso database file together with its WAL/SHM sidecars.
36+
* Copy a SQLite/Turso database file together with its WAL/SHM/TSHM sidecars.
3637
*/
3738
export function copySqliteDatabase(fromPath: string, toPath: string): void {
3839
copyFileSync(fromPath, toPath);
@@ -49,10 +50,10 @@ export function copySqliteDatabase(fromPath: string, toPath: string): void {
4950
}
5051

5152
/**
52-
* Remove a SQLite/Turso database file together with WAL/SHM sidecars.
53+
* Remove a SQLite/Turso database file together with WAL/SHM/TSHM sidecars.
5354
*/
5455
export function removeSqliteDatabase(dbPath: string): void {
55-
for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) {
56+
for (const path of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`, `${dbPath}-tshm`]) {
5657
if (existsSync(path)) unlinkSync(path);
5758
}
5859
}

‎tests/turso-connection-manager.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,8 @@ describe("turso connection manager", () => {
169169
expect(opens).toHaveLength(1);
170170
expect(opens[0]?.path).toBe(dbPath);
171171
expect(opens[0]?.opts).toMatchObject({
172-
experimental: ["encryption"],
172+
experimental:
173+
process.platform === "win32" ? ["encryption"] : ["encryption", "multiprocess_wal"],
173174
});
174175
expect(opens[0]?.opts).not.toHaveProperty("encryption");
175176
} finally {

0 commit comments

Comments
 (0)