Skip to content

Commit 5b21a03

Browse files
849261680849261680
authored
fix(state): avoid sqlite wal on nfs state volumes
Detects NFS-backed SQLite database paths in the shared WAL helper and uses rollback journaling for those paths while preserving WAL/checkpoint maintenance on local filesystems. The NFS path now verifies SQLite's effective journal mode before disabling WAL maintenance, and core/memory/proxy-capture callers pass database path context into the centralized helper. Fixes #90491. Proof: local focused Vitest/format/lint; autoreview clean after fixing the journal-mode verification finding; Crabbox AWS focused test run `run_2ea7014350da`; Crabbox AWS changed gate `run_c828bbfe7d23`; exact-head GitHub CI green on `59674305ecd863d4815eec6098ccd3daab79ca4f`.
1 parent dbf24fe commit 5b21a03

6 files changed

Lines changed: 344 additions & 12 deletions

File tree

‎extensions/memory-core/src/memory/manager-db.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,7 @@ export function openMemoryDatabaseAtPath(
2828
probe.close();
2929
} catch (err) {
3030
const msg = (err as Error).message ?? "";
31-
if (
32-
msg.includes("unable to open database file") ||
33-
msg.includes("SQLITE_CANTOPEN")
34-
) {
31+
if (msg.includes("unable to open database file") || msg.includes("SQLITE_CANTOPEN")) {
3532
throw new Error(
3633
`Memory database not found at ${dbPath}; refusing to auto-create an empty database during an index swap window.`,
3734
{ cause: err },
@@ -40,7 +37,10 @@ export function openMemoryDatabaseAtPath(
4037
}
4138
}
4239
const db = new DatabaseSync(dbPath, { allowExtension });
43-
configureMemorySqliteWalMaintenance(db);
40+
configureMemorySqliteWalMaintenance(db, { databasePath: dbPath });
41+
// busy_timeout is per-connection and resets to 0 on restart.
42+
// Set it on every open so concurrent processes retry instead of
43+
// failing immediately with SQLITE_BUSY.
4444
db.exec("PRAGMA busy_timeout = 5000");
4545
return db;
4646
}

‎src/infra/sqlite-wal.test.ts‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
// Covers SQLite WAL maintenance configuration.
2+
import childProcess from "node:child_process";
3+
import fs from "node:fs";
4+
import os from "node:os";
5+
import path from "node:path";
26
import type { DatabaseSync } from "node:sqlite";
37
import { afterEach, describe, expect, it, vi } from "vitest";
48
import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js";
@@ -10,11 +14,27 @@ import {
1014
function createMockDb(): DatabaseSync {
1115
return {
1216
exec: vi.fn(),
17+
prepare: vi.fn(() => ({
18+
get: vi.fn(() => ({ journal_mode: "delete" })),
19+
})),
1320
} as unknown as DatabaseSync;
1421
}
1522

23+
function statfsFixture(type: number): ReturnType<typeof fs.statfsSync> {
24+
return {
25+
type,
26+
bsize: 1024,
27+
blocks: 1,
28+
bfree: 1,
29+
bavail: 1,
30+
files: 0,
31+
ffree: 0,
32+
};
33+
}
34+
1635
describe("sqlite WAL maintenance", () => {
1736
afterEach(() => {
37+
vi.restoreAllMocks();
1838
vi.useRealTimers();
1939
});
2040

@@ -30,6 +50,115 @@ describe("sqlite WAL maintenance", () => {
3050
);
3151
});
3252

53+
it("uses rollback journaling for databases on NFS-backed volumes", () => {
54+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-"));
55+
try {
56+
const db = createMockDb();
57+
const statfs = vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0x6969));
58+
59+
const maintenance = configureSqliteWalMaintenance(db, {
60+
checkpointIntervalMs: 0,
61+
databasePath: path.join(tempDir, "missing", "openclaw.sqlite"),
62+
});
63+
64+
expect(statfs).toHaveBeenCalledWith(tempDir);
65+
expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;");
66+
expect(db["exec"]).not.toHaveBeenCalled();
67+
expect(maintenance.checkpoint()).toBe(true);
68+
expect(maintenance.close()).toBe(true);
69+
expect(db["exec"]).not.toHaveBeenCalled();
70+
} finally {
71+
fs.rmSync(tempDir, { recursive: true, force: true });
72+
}
73+
});
74+
75+
it("refuses NFS-backed databases when SQLite keeps WAL active", () => {
76+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-"));
77+
try {
78+
const db = createMockDb();
79+
vi.mocked(db["prepare"]).mockReturnValue({
80+
get: vi.fn(() => ({ journal_mode: "wal" })),
81+
} as unknown as ReturnType<DatabaseSync["prepare"]>);
82+
vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0x6969));
83+
84+
expect(() =>
85+
configureSqliteWalMaintenance(db, {
86+
checkpointIntervalMs: 0,
87+
databaseLabel: "test-db",
88+
databasePath: path.join(tempDir, "openclaw.sqlite"),
89+
}),
90+
).toThrow(/test-db .*journal_mode=wal/);
91+
} finally {
92+
fs.rmSync(tempDir, { recursive: true, force: true });
93+
}
94+
});
95+
96+
it("uses mountinfo filesystem names when statfs magic is not enough", () => {
97+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-"));
98+
try {
99+
const db = createMockDb();
100+
vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0));
101+
vi.spyOn(fs, "readFileSync").mockReturnValue(
102+
`42 12 0:41 / ${tempDir} rw,relatime - nfs4 server:/share rw\n`,
103+
);
104+
105+
configureSqliteWalMaintenance(db, {
106+
checkpointIntervalMs: 0,
107+
databasePath: path.join(tempDir, "openclaw.sqlite"),
108+
});
109+
110+
expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;");
111+
} finally {
112+
fs.rmSync(tempDir, { recursive: true, force: true });
113+
}
114+
});
115+
116+
it("uses mount command filesystem names on platforms without proc mountinfo", () => {
117+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-"));
118+
try {
119+
const db = createMockDb();
120+
vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0));
121+
vi.spyOn(fs, "readFileSync").mockImplementation(() => {
122+
throw new Error("no proc mountinfo");
123+
});
124+
vi.spyOn(childProcess, "execFileSync").mockReturnValue(
125+
Buffer.from(`server:/share on ${tempDir} (nfs, nodev, nosuid)\n`),
126+
);
127+
128+
configureSqliteWalMaintenance(db, {
129+
checkpointIntervalMs: 0,
130+
databasePath: path.join(tempDir, "openclaw.sqlite"),
131+
});
132+
133+
expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;");
134+
} finally {
135+
fs.rmSync(tempDir, { recursive: true, force: true });
136+
}
137+
});
138+
139+
it("parses Linux mount command filesystem names when proc mountinfo is unavailable", () => {
140+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-"));
141+
try {
142+
const db = createMockDb();
143+
vi.spyOn(fs, "statfsSync").mockReturnValue(statfsFixture(0));
144+
vi.spyOn(fs, "readFileSync").mockImplementation(() => {
145+
throw new Error("no proc mountinfo");
146+
});
147+
vi.spyOn(childProcess, "execFileSync").mockReturnValue(
148+
Buffer.from(`server:/share on ${tempDir} type nfs4 (rw,relatime)\n`),
149+
);
150+
151+
configureSqliteWalMaintenance(db, {
152+
checkpointIntervalMs: 0,
153+
databasePath: path.join(tempDir, "openclaw.sqlite"),
154+
});
155+
156+
expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode = DELETE;");
157+
} finally {
158+
fs.rmSync(tempDir, { recursive: true, force: true });
159+
}
160+
});
161+
33162
it("runs periodic TRUNCATE checkpoints and stops them on close", () => {
34163
vi.useFakeTimers();
35164
const db = createMockDb();

‎src/infra/sqlite-wal.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
11
// Configures SQLite WAL and related pragmas for local stores.
2+
import childProcess from "node:child_process";
3+
import fs from "node:fs";
4+
import path from "node:path";
25
import type { DatabaseSync } from "node:sqlite";
36
import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js";
47

58
// WAL maintenance configures SQLite write-ahead logging and schedules bounded
69
// checkpoints so state databases do not accumulate unbounded WAL files.
710
export const DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES = 1000;
811
export const DEFAULT_SQLITE_WAL_TRUNCATE_INTERVAL_MS = 30 * 60 * 1000;
12+
const LINUX_NFS_SUPER_MAGIC = 0x6969;
13+
const PROC_MOUNTINFO_PATH = "/proc/self/mountinfo";
914

1015
type IntervalHandle = ReturnType<typeof setInterval> & {
1116
unref?: () => void;
@@ -35,6 +40,139 @@ function normalizeNonNegativeInteger(value: number, label: string): number {
3540
return value;
3641
}
3742

43+
function findExistingVolumePath(targetPath: string): string | null {
44+
let current = path.resolve(targetPath);
45+
while (true) {
46+
try {
47+
const stats = fs.statSync(current);
48+
return stats.isDirectory() ? current : path.dirname(current);
49+
} catch {
50+
const parent = path.dirname(current);
51+
if (parent === current) {
52+
return null;
53+
}
54+
current = parent;
55+
}
56+
}
57+
}
58+
59+
function decodeMountPath(value: string): string {
60+
return value.replace(/\\([0-7]{3})/g, (_match, octal: string) =>
61+
String.fromCharCode(Number.parseInt(octal, 8)),
62+
);
63+
}
64+
65+
function parseProcMountInfoEntries(
66+
contents: string,
67+
): Array<{ mountPoint: string; fsType: string }> {
68+
const entries: Array<{ mountPoint: string; fsType: string }> = [];
69+
for (const line of contents.split("\n")) {
70+
const separator = line.indexOf(" - ");
71+
if (separator === -1) {
72+
continue;
73+
}
74+
const fields = line.slice(0, separator).split(" ");
75+
const suffixFields = line.slice(separator + 3).split(" ");
76+
const mountPoint = fields[4];
77+
const fsType = suffixFields[0];
78+
if (mountPoint && fsType) {
79+
entries.push({ mountPoint: decodeMountPath(mountPoint), fsType });
80+
}
81+
}
82+
return entries;
83+
}
84+
85+
function parseMountCommandEntries(contents: string): Array<{ mountPoint: string; fsType: string }> {
86+
const entries: Array<{ mountPoint: string; fsType: string }> = [];
87+
for (const line of contents.split("\n")) {
88+
const linuxMatch = /^.* on (.+) type ([^,\s)]+) \(/.exec(line);
89+
if (linuxMatch) {
90+
entries.push({ mountPoint: linuxMatch[1], fsType: linuxMatch[2] });
91+
continue;
92+
}
93+
const bsdMatch = /^.* on (.+) \(([^,\s)]+)/.exec(line);
94+
if (bsdMatch) {
95+
entries.push({ mountPoint: bsdMatch[1], fsType: bsdMatch[2] });
96+
}
97+
}
98+
return entries;
99+
}
100+
101+
function readMountEntries(): Array<{ mountPoint: string; fsType: string }> {
102+
try {
103+
return parseProcMountInfoEntries(fs.readFileSync(PROC_MOUNTINFO_PATH, "utf8"));
104+
} catch {
105+
// macOS/BSD expose filesystem type names in `mount` output instead of
106+
// Linux superblock magic, so keep this fallback for non-Linux NFS mounts.
107+
}
108+
try {
109+
return parseMountCommandEntries(String(childProcess.execFileSync("mount", [])));
110+
} catch {
111+
return [];
112+
}
113+
}
114+
115+
function isPathWithinMount(targetPath: string, mountPoint: string): boolean {
116+
const resolvedTarget = path.resolve(targetPath);
117+
const resolvedMountPoint = path.resolve(mountPoint);
118+
return (
119+
resolvedTarget === resolvedMountPoint ||
120+
resolvedMountPoint === path.parse(resolvedMountPoint).root ||
121+
resolvedTarget.startsWith(`${resolvedMountPoint}${path.sep}`)
122+
);
123+
}
124+
125+
function isNfsMountType(fsType: string): boolean {
126+
return fsType.toLowerCase().startsWith("nfs");
127+
}
128+
129+
function isNfsMountEntryPath(targetPath: string): boolean {
130+
const mountEntry = readMountEntries()
131+
.filter((entry) => isPathWithinMount(targetPath, entry.mountPoint))
132+
.toSorted((a, b) => b.mountPoint.length - a.mountPoint.length)[0];
133+
return mountEntry ? isNfsMountType(mountEntry.fsType) : false;
134+
}
135+
136+
function isNfsBackedPath(targetPath: string): boolean {
137+
if (typeof fs.statfsSync !== "function") {
138+
return isNfsMountEntryPath(targetPath);
139+
}
140+
const checkedPath = findExistingVolumePath(targetPath);
141+
if (!checkedPath) {
142+
return false;
143+
}
144+
try {
145+
if (fs.statfsSync(checkedPath).type === LINUX_NFS_SUPER_MAGIC) {
146+
return true;
147+
}
148+
} catch {
149+
return isNfsMountEntryPath(checkedPath);
150+
}
151+
return isNfsMountEntryPath(checkedPath);
152+
}
153+
154+
function readJournalModeResult(row: unknown): string | null {
155+
if (!row || typeof row !== "object") {
156+
return null;
157+
}
158+
const record = row as Record<string, unknown>;
159+
const value = record.journal_mode ?? Object.values(record)[0];
160+
return typeof value === "string" ? value.toLowerCase() : null;
161+
}
162+
163+
function requireRollbackJournalMode(db: DatabaseSync, options: SqliteWalMaintenanceOptions): void {
164+
const row = db.prepare("PRAGMA journal_mode = DELETE;").get();
165+
const journalMode = readJournalModeResult(row);
166+
if (journalMode !== "delete") {
167+
const label = options.databaseLabel ?? "sqlite database";
168+
const location = options.databasePath ? ` at ${options.databasePath}` : "";
169+
const actual = journalMode ?? "unknown";
170+
throw new Error(
171+
`${label}${location} is on an NFS-backed volume but SQLite kept journal_mode=${actual}; refusing to continue with WAL on NFS.`,
172+
);
173+
}
174+
}
175+
38176
/** Configure WAL pragmas and return a handle for checkpoint/close maintenance. */
39177
export function configureSqliteWalMaintenance(
40178
db: DatabaseSync,
@@ -50,6 +188,13 @@ export function configureSqliteWalMaintenance(
50188
);
51189
const timerIntervalMs = Math.min(checkpointIntervalMs, MAX_TIMER_TIMEOUT_MS);
52190
const checkpointMode = options.checkpointMode ?? "TRUNCATE";
191+
if (options.databasePath && isNfsBackedPath(options.databasePath)) {
192+
requireRollbackJournalMode(db, options);
193+
return {
194+
checkpoint: () => true,
195+
close: () => true,
196+
};
197+
}
53198
db.exec("PRAGMA journal_mode = WAL;");
54199
db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages};`);
55200

0 commit comments

Comments
 (0)