Skip to content

Commit ef196b6

Browse files
vincentkocgiodl73-repo
vincentkoc
and
giodl73-repo
authored
refactor(sqlite): centralize verified snapshot publication (#105412)
Co-authored-by: Gio Della-Libera <[email protected]>
1 parent 8c236c3 commit ef196b6

5 files changed

Lines changed: 590 additions & 78 deletions

File tree

‎src/commands/doctor-state-sqlite-compact.ts‎

Lines changed: 3 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,9 @@
11
/** Explicit doctor maintenance for the canonical shared state SQLite database. */
22
import fs from "node:fs";
3-
import type { DatabaseSync } from "node:sqlite";
4-
import {
5-
createNewerSqliteSchemaVersionError,
6-
readSqliteUserVersion,
7-
} from "../infra/sqlite-user-version.js";
83
import {
4+
assertOpenClawStateDatabaseForMaintenance,
95
ensureOpenClawStatePermissions,
106
isOpenClawStateDatabaseOpen,
11-
OPENCLAW_STATE_SCHEMA_VERSION,
127
} from "../state/openclaw-state-db.js";
138
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
149
import {
@@ -65,7 +60,8 @@ export function runDoctorStateSqliteCompact(
6560
const compact = compactDoctorSqliteFile({
6661
afterMutation: () => ensureOpenClawStatePermissions(sqlitePath, env),
6762
sqlitePath,
68-
validateBeforeMutation: (database) => validateCanonicalStateDatabase(database, sqlitePath),
63+
validateBeforeMutation: (database) =>
64+
assertOpenClawStateDatabaseForMaintenance(database, { pathname: sqlitePath }),
6965
});
7066
return {
7167
...compact,
@@ -85,37 +81,3 @@ function readCanonicalStateDatabaseStat(sqlitePath: string): fs.Stats | undefine
8581
throw error;
8682
}
8783
}
88-
89-
function validateCanonicalStateDatabase(database: DatabaseSync, sqlitePath: string): void {
90-
const userVersion = readSqliteUserVersion(database);
91-
if (userVersion > OPENCLAW_STATE_SCHEMA_VERSION) {
92-
throw createNewerSqliteSchemaVersionError(
93-
"OpenClaw state database",
94-
sqlitePath,
95-
userVersion,
96-
OPENCLAW_STATE_SCHEMA_VERSION,
97-
);
98-
}
99-
if (userVersion !== OPENCLAW_STATE_SCHEMA_VERSION) {
100-
throw new Error(
101-
`OpenClaw state database ${sqlitePath} uses schema version ${userVersion}; run openclaw doctor --fix before compacting it.`,
102-
);
103-
}
104-
105-
const metadata = database
106-
.prepare("SELECT role, schema_version FROM schema_meta WHERE meta_key = 'primary' LIMIT 1")
107-
.get() as { role?: unknown; schema_version?: unknown } | undefined;
108-
if (metadata?.role !== "global") {
109-
const role = typeof metadata?.role === "string" ? metadata.role : "missing";
110-
throw new Error(
111-
`OpenClaw state database ${sqlitePath} has schema role ${role}; expected global.`,
112-
);
113-
}
114-
if (metadata.schema_version !== OPENCLAW_STATE_SCHEMA_VERSION) {
115-
const schemaVersion =
116-
typeof metadata.schema_version === "number" ? metadata.schema_version : "invalid";
117-
throw new Error(
118-
`OpenClaw state database ${sqlitePath} metadata schema version ${schemaVersion} does not match ${OPENCLAW_STATE_SCHEMA_VERSION}; run openclaw doctor --fix before compacting it.`,
119-
);
120-
}
121-
}

‎src/infra/backup-create.ts‎

Lines changed: 10 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import path from "node:path";
77
import type { DatabaseSync } from "node:sqlite";
88
import { pipeline } from "node:stream/promises";
99
import { resolveDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
10-
import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js";
1110
import {
1211
buildBackupArchiveBasename,
1312
buildBackupArchivePath,
@@ -24,8 +23,7 @@ import { resolveRuntimeServiceVersion } from "../version.js";
2423
import { isVolatileBackupPath } from "./backup-volatile-filter.js";
2524
import { formatErrorMessage } from "./errors.js";
2625
import { writeJson } from "./json-files.js";
27-
import { requireNodeSqlite } from "./node-sqlite.js";
28-
import { assertSqliteIntegrity } from "./sqlite-integrity.js";
26+
import { createVerifiedSqliteSnapshot } from "./sqlite-snapshot.js";
2927

3028
const loadTarRuntime = createLazyRuntimeModule(() => import("tar"));
3129

@@ -631,7 +629,6 @@ function tableExistsSql(db: DatabaseSync, tableName: string): boolean {
631629
function sanitizeGlobalStateSqliteSnapshot(db: DatabaseSync): void {
632630
if (tableExistsSql(db, "delivery_queue_entries")) {
633631
db.prepare("DELETE FROM delivery_queue_entries").run();
634-
db.exec("VACUUM;");
635632
}
636633
}
637634

@@ -738,7 +735,6 @@ async function createStateSqliteBackupPlan(params: {
738735
stateDir: params.stateDir,
739736
globalStateSqlitePath,
740737
});
741-
const sqlite = requireNodeSqlite();
742738
const snapshots: SqliteBackupAsset[] = [];
743739
for (const archiveSourcePath of discovery.snapshotPaths) {
744740
// A discovered *.sqlite file that SQLite cannot snapshot aborts backup.
@@ -749,44 +745,21 @@ async function createStateSqliteBackupPlan(params: {
749745
path.resolve(archiveSourcePath) === globalStateSqlitePath
750746
? await fs.realpath(archiveSourcePath)
751747
: archiveSourcePath;
752-
const source = new sqlite.DatabaseSync(sourceDatabasePath, {
753-
allowExtension: true,
754-
readOnly: true,
755-
});
756748
const sourcePath = path.join(params.tempDir, `openclaw-state-db-${snapshots.length}.sqlite`);
757749
try {
758-
source.exec("PRAGMA busy_timeout = 30000;");
759-
try {
760-
// VACUUM INTO removes deleted-page remnants before the snapshot enters
761-
// the archive. Load known bundled extensions, but fail closed when an
762-
// owner schema needs capabilities core cannot safely reproduce.
763-
await loadSqliteVecExtension({ db: source });
764-
assertSqliteIntegrity(source, archiveSourcePath);
765-
source.prepare("VACUUM INTO ?").run(sourcePath);
766-
} catch (err) {
767-
throw new Error(
768-
`SQLite database cannot be compacted safely for backup: ${archiveSourcePath}. ${formatErrorMessage(err)}. The source must pass full integrity checks and VACUUM INTO with its required SQLite capabilities; raw page backup was refused because it can retain deleted data.`,
769-
{ cause: err },
770-
);
771-
}
772-
} finally {
773-
source.close();
774-
}
775-
await fs.chmod(sourcePath, 0o600);
776-
const snapshot = new sqlite.DatabaseSync(sourcePath, { allowExtension: true });
777-
try {
778-
await loadSqliteVecExtension({ db: snapshot });
779-
if (path.resolve(archiveSourcePath) === globalStateSqlitePath) {
780-
sanitizeGlobalStateSqliteSnapshot(snapshot);
781-
}
782-
assertSqliteIntegrity(snapshot, sourcePath);
750+
await createVerifiedSqliteSnapshot({
751+
sourcePath: sourceDatabasePath,
752+
targetPath: sourcePath,
753+
transform:
754+
path.resolve(archiveSourcePath) === globalStateSqlitePath
755+
? sanitizeGlobalStateSqliteSnapshot
756+
: undefined,
757+
});
783758
} catch (err) {
784759
throw new Error(
785-
`SQLite backup snapshot failed verification for ${archiveSourcePath}: ${formatErrorMessage(err)}`,
760+
`SQLite database cannot be compacted safely for backup: ${archiveSourcePath}. ${formatErrorMessage(err)}. The source must pass full integrity checks and VACUUM INTO with its required SQLite capabilities; raw page backup was refused because it can retain deleted data.`,
786761
{ cause: err },
787762
);
788-
} finally {
789-
snapshot.close();
790763
}
791764
snapshots.push({
792765
sourcePath,

0 commit comments

Comments
 (0)