Skip to content

Commit d1b9171

Browse files
giodl73-repogiodl73-repo
authored
Persist hosted catalog snapshots in state (#95964)
Merged via squash. Prepared head SHA: 372ec63 Co-authored-by: giodl73-repo <[email protected]> Co-authored-by: giodl73-repo <[email protected]> Reviewed-by: @giodl73-repo
1 parent 78f7de0 commit d1b9171

6 files changed

Lines changed: 334 additions & 9 deletions

‎src/plugins/official-external-plugin-catalog-snapshot-store.ts‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/** Persists hosted official external plugin catalog snapshots in OpenClaw state. */
2+
import { existsSync } from "node:fs";
3+
import {
4+
executeSqliteQuerySync,
5+
executeSqliteQueryTakeFirstSync,
6+
getNodeSqliteKysely,
7+
} from "../infra/kysely-sync.js";
8+
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
9+
import {
10+
openOpenClawStateDatabase,
11+
runOpenClawStateWriteTransaction,
12+
type OpenClawStateDatabaseOptions,
13+
} from "../state/openclaw-state-db.js";
14+
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
15+
import type {
16+
HostedOfficialExternalPluginCatalogMetadata,
17+
HostedOfficialExternalPluginCatalogSnapshot,
18+
HostedOfficialExternalPluginCatalogSnapshotStore,
19+
} from "./official-external-plugin-catalog.js";
20+
21+
export type HostedOfficialExternalPluginCatalogSnapshotStoreOptions = {
22+
env?: NodeJS.ProcessEnv;
23+
stateDir?: string;
24+
stateDatabasePath?: string;
25+
};
26+
27+
type HostedCatalogSnapshotRow = {
28+
feed_url: string;
29+
body: string;
30+
status: number | bigint;
31+
etag: string | null;
32+
last_modified: string | null;
33+
checksum: string;
34+
saved_at: string;
35+
};
36+
37+
type HostedCatalogSnapshotDatabase = Pick<
38+
OpenClawStateKyselyDatabase,
39+
"official_external_plugin_catalog_snapshots"
40+
>;
41+
42+
function resolveStoreEnv(
43+
options: HostedOfficialExternalPluginCatalogSnapshotStoreOptions,
44+
): NodeJS.ProcessEnv | undefined {
45+
if (!options.stateDir) {
46+
return options.env;
47+
}
48+
return {
49+
...(options.env ?? process.env),
50+
OPENCLAW_STATE_DIR: options.stateDir,
51+
};
52+
}
53+
54+
function resolveStateDatabaseOptions(
55+
options: HostedOfficialExternalPluginCatalogSnapshotStoreOptions,
56+
): OpenClawStateDatabaseOptions {
57+
const env = resolveStoreEnv(options);
58+
return {
59+
...(env ? { env } : {}),
60+
...(options.stateDatabasePath ? { path: options.stateDatabasePath } : {}),
61+
};
62+
}
63+
64+
function resolveStateDatabasePath(
65+
options: HostedOfficialExternalPluginCatalogSnapshotStoreOptions,
66+
): string {
67+
if (options.stateDatabasePath) {
68+
return options.stateDatabasePath;
69+
}
70+
return resolveOpenClawStateSqlitePath(resolveStoreEnv(options) ?? process.env);
71+
}
72+
73+
function rowToSnapshot(
74+
row: HostedCatalogSnapshotRow | undefined,
75+
): HostedOfficialExternalPluginCatalogSnapshot | null {
76+
if (!row) {
77+
return null;
78+
}
79+
const metadata: HostedOfficialExternalPluginCatalogMetadata = {
80+
url: row.feed_url,
81+
status: Number(row.status),
82+
checksum: row.checksum,
83+
...(row.etag ? { etag: row.etag } : {}),
84+
...(row.last_modified ? { lastModified: row.last_modified } : {}),
85+
};
86+
return {
87+
body: row.body,
88+
metadata,
89+
savedAt: row.saved_at,
90+
};
91+
}
92+
93+
/** Creates a snapshot store backed by the shared `state/openclaw.sqlite` database. */
94+
export function createSqliteHostedOfficialExternalPluginCatalogSnapshotStore(
95+
options: HostedOfficialExternalPluginCatalogSnapshotStoreOptions = {},
96+
): HostedOfficialExternalPluginCatalogSnapshotStore {
97+
return {
98+
async read(url) {
99+
const pathname = resolveStateDatabasePath(options);
100+
if (!existsSync(pathname)) {
101+
return null;
102+
}
103+
const database = openOpenClawStateDatabase(resolveStateDatabaseOptions(options));
104+
const stateDb = getNodeSqliteKysely<HostedCatalogSnapshotDatabase>(database.db);
105+
const row = executeSqliteQueryTakeFirstSync(
106+
database.db,
107+
stateDb
108+
.selectFrom("official_external_plugin_catalog_snapshots")
109+
.select(["feed_url", "body", "status", "etag", "last_modified", "checksum", "saved_at"])
110+
.where("feed_url", "=", url),
111+
) as HostedCatalogSnapshotRow | undefined;
112+
return rowToSnapshot(row);
113+
},
114+
async write(snapshot) {
115+
const now = Date.now();
116+
runOpenClawStateWriteTransaction((database) => {
117+
const stateDb = getNodeSqliteKysely<HostedCatalogSnapshotDatabase>(database.db);
118+
executeSqliteQuerySync(
119+
database.db,
120+
stateDb
121+
.insertInto("official_external_plugin_catalog_snapshots")
122+
.values({
123+
feed_url: snapshot.metadata.url,
124+
body: snapshot.body,
125+
status: snapshot.metadata.status,
126+
etag: snapshot.metadata.etag ?? null,
127+
last_modified: snapshot.metadata.lastModified ?? null,
128+
checksum: snapshot.metadata.checksum,
129+
saved_at: snapshot.savedAt,
130+
updated_at_ms: now,
131+
})
132+
.onConflict((conflict) =>
133+
conflict.column("feed_url").doUpdateSet({
134+
body: snapshot.body,
135+
status: snapshot.metadata.status,
136+
etag: snapshot.metadata.etag ?? null,
137+
last_modified: snapshot.metadata.lastModified ?? null,
138+
checksum: snapshot.metadata.checksum,
139+
saved_at: snapshot.savedAt,
140+
updated_at_ms: now,
141+
}),
142+
),
143+
);
144+
}, resolveStateDatabaseOptions(options));
145+
},
146+
};
147+
}

‎src/plugins/official-external-plugin-catalog.test.ts‎

Lines changed: 112 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1-
import { readFileSync } from "node:fs";
1+
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
2+
import os from "node:os";
3+
import path from "node:path";
24
import { describe, expect, it, vi } from "vitest";
35
import officialExternalPluginCatalog from "../../scripts/lib/official-external-plugin-catalog.json" with { type: "json" };
6+
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
7+
import { createSqliteHostedOfficialExternalPluginCatalogSnapshotStore } from "./official-external-plugin-catalog-snapshot-store.js";
48
import {
59
type OfficialExternalPluginCatalogEntry,
610
DEFAULT_OFFICIAL_EXTERNAL_PLUGIN_CATALOG_FEED_URL,
@@ -245,6 +249,7 @@ describe("official external plugin catalog", () => {
245249
fetchImpl,
246250
ifNoneMatch: '"old"',
247251
ifModifiedSince: "Mon, 22 Jun 2026 00:00:00 GMT",
252+
snapshotStore: null,
248253
});
249254

250255
expect(result.source).toBe("hosted");
@@ -315,6 +320,7 @@ describe("official external plugin catalog", () => {
315320

316321
it("falls back to the bundled catalog when hosted feed validation fails", async () => {
317322
const result = await loadHostedOfficialExternalPluginCatalogEntries({
323+
snapshotStore: null,
318324
fetchImpl: vi.fn(
319325
async () =>
320326
new Response(JSON.stringify({ schemaVersion: 1, id: " ", entries: [] }), {
@@ -333,6 +339,7 @@ describe("official external plugin catalog", () => {
333339

334340
it("falls back to the bundled catalog on HTTP 304 until a snapshot cache exists", async () => {
335341
const result = await loadHostedOfficialExternalPluginCatalogEntries({
342+
snapshotStore: null,
336343
fetchImpl: vi.fn(
337344
async () =>
338345
new Response(null, {
@@ -395,6 +402,108 @@ describe("official external plugin catalog", () => {
395402
expect(snapshot?.metadata.checksum).toMatch(/^sha256:[0-9a-f]{64}$/);
396403
});
397404

405+
it("persists hosted feed snapshots in OpenClaw state for HTTP 304 reuse", async () => {
406+
const stateDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-hosted-catalog-"));
407+
try {
408+
const body = JSON.stringify({
409+
schemaVersion: 1,
410+
id: "openclaw-official-external-plugins",
411+
generatedAt: "2026-06-22T00:00:00.000Z",
412+
sequence: 7,
413+
entries: [
414+
{
415+
name: "@openclaw/sqlite-snapshot-proof",
416+
kind: "plugin",
417+
openclaw: { plugin: { id: "sqlite-snapshot-proof" } },
418+
},
419+
],
420+
});
421+
422+
const seeded = await loadHostedOfficialExternalPluginCatalogEntries({
423+
stateDir,
424+
now: () => new Date("2026-06-22T02:03:04.000Z"),
425+
fetchImpl: vi.fn(
426+
async () =>
427+
new Response(body, {
428+
status: 200,
429+
headers: {
430+
etag: '"sqlite"',
431+
"last-modified": "Mon, 22 Jun 2026 02:00:00 GMT",
432+
},
433+
}),
434+
),
435+
});
436+
if (seeded.source !== "hosted") {
437+
throw new Error("expected seeded hosted feed");
438+
}
439+
closeOpenClawStateDatabaseForTest();
440+
441+
const result = await loadHostedOfficialExternalPluginCatalogEntries({
442+
stateDir,
443+
fetchImpl: vi.fn(async () => new Response(null, { status: 304 })),
444+
});
445+
446+
expect(result.source).toBe("hosted-snapshot");
447+
expect(result.entries.map((entry) => entry.name)).toEqual([
448+
"@openclaw/sqlite-snapshot-proof",
449+
]);
450+
if (result.source === "hosted-snapshot") {
451+
expect(result.snapshot.savedAt).toBe("2026-06-22T02:03:04.000Z");
452+
expect(result.metadata.checksum).toBe(seeded.metadata.checksum);
453+
}
454+
} finally {
455+
closeOpenClawStateDatabaseForTest();
456+
rmSync(stateDir, { recursive: true, force: true });
457+
}
458+
});
459+
460+
it("reads and updates hosted catalog snapshots in the SQLite store", async () => {
461+
const stateDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-hosted-store-"));
462+
try {
463+
const store = createSqliteHostedOfficialExternalPluginCatalogSnapshotStore({ stateDir });
464+
const url = "https://register.openclaw.ai/official-external-plugin-catalog.json";
465+
466+
const firstBody = JSON.stringify({ entries: [] });
467+
const secondBody = JSON.stringify({ entries: [{}] });
468+
469+
await expect(store.read(url)).resolves.toBeNull();
470+
await store.write({
471+
body: firstBody,
472+
metadata: {
473+
url,
474+
status: 200,
475+
etag: '"first"',
476+
checksum: "sha256:first",
477+
},
478+
savedAt: "2026-06-22T02:03:04.000Z",
479+
});
480+
await store.write({
481+
body: secondBody,
482+
metadata: {
483+
url,
484+
status: 200,
485+
lastModified: "Mon, 22 Jun 2026 03:00:00 GMT",
486+
checksum: "sha256:second",
487+
},
488+
savedAt: "2026-06-22T03:04:05.000Z",
489+
});
490+
491+
await expect(store.read(url)).resolves.toMatchObject({
492+
body: secondBody,
493+
metadata: {
494+
url,
495+
status: 200,
496+
lastModified: "Mon, 22 Jun 2026 03:00:00 GMT",
497+
checksum: "sha256:second",
498+
},
499+
savedAt: "2026-06-22T03:04:05.000Z",
500+
});
501+
} finally {
502+
closeOpenClawStateDatabaseForTest();
503+
rmSync(stateDir, { recursive: true, force: true });
504+
}
505+
});
506+
398507
it("uses the last known good snapshot when the hosted feed returns HTTP 304", async () => {
399508
const body = JSON.stringify({
400509
schemaVersion: 1,
@@ -655,6 +764,7 @@ describe("official external plugin catalog", () => {
655764

656765
it("falls back to the bundled catalog on checksum mismatch and oversized bodies", async () => {
657766
const mismatch = await loadHostedOfficialExternalPluginCatalogEntries({
767+
snapshotStore: null,
658768
expectedSha256: "sha256:not-current",
659769
fetchImpl: vi.fn(
660770
async () =>
@@ -677,6 +787,7 @@ describe("official external plugin catalog", () => {
677787
}
678788

679789
const oversized = await loadHostedOfficialExternalPluginCatalogEntries({
790+
snapshotStore: null,
680791
maxBytes: 4,
681792
fetchImpl: vi.fn(async () => new Response("12345", { status: 200 })),
682793
});

0 commit comments

Comments
 (0)