Skip to content

Commit d0c4a58

Browse files
vincentkocvincentkoc
committed
feat(catalog): publish cached hosted plugin feed
1 parent 422c287 commit d0c4a58

19 files changed

Lines changed: 461 additions & 6 deletions

‎convex/_generated/api.d.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import type * as appMeta from "../appMeta.js";
1212
import type * as auth from "../auth.js";
1313
import type * as catalogClassification from "../catalogClassification.js";
1414
import type * as catalogClassificationNode from "../catalogClassificationNode.js";
15+
import type * as catalogFeed from "../catalogFeed.js";
1516
import type * as catalogTopics from "../catalogTopics.js";
1617
import type * as cliDeviceAuth from "../cliDeviceAuth.js";
1718
import type * as crons from "../crons.js";
@@ -31,6 +32,7 @@ import type * as githubSkillSyncNode from "../githubSkillSyncNode.js";
3132
import type * as http from "../http.js";
3233
import type * as httpApi from "../httpApi.js";
3334
import type * as httpApiV1 from "../httpApiV1.js";
35+
import type * as httpApiV1_catalogFeedV1 from "../httpApiV1/catalogFeedV1.js";
3436
import type * as httpApiV1_contentRightsV1 from "../httpApiV1/contentRightsV1.js";
3537
import type * as httpApiV1_docsSessionV1 from "../httpApiV1/docsSessionV1.js";
3638
import type * as httpApiV1_packagesV1 from "../httpApiV1/packagesV1.js";
@@ -160,6 +162,7 @@ declare const fullApi: ApiFromModules<{
160162
auth: typeof auth;
161163
catalogClassification: typeof catalogClassification;
162164
catalogClassificationNode: typeof catalogClassificationNode;
165+
catalogFeed: typeof catalogFeed;
163166
catalogTopics: typeof catalogTopics;
164167
cliDeviceAuth: typeof cliDeviceAuth;
165168
crons: typeof crons;
@@ -179,6 +182,7 @@ declare const fullApi: ApiFromModules<{
179182
http: typeof http;
180183
httpApi: typeof httpApi;
181184
httpApiV1: typeof httpApiV1;
185+
"httpApiV1/catalogFeedV1": typeof httpApiV1_catalogFeedV1;
182186
"httpApiV1/contentRightsV1": typeof httpApiV1_contentRightsV1;
183187
"httpApiV1/docsSessionV1": typeof httpApiV1_docsSessionV1;
184188
"httpApiV1/packagesV1": typeof httpApiV1_packagesV1;

‎convex/catalogFeed.ts‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
import {
2+
CATALOG_FEED_ID,
3+
CATALOG_FEED_SCHEMA_VERSION,
4+
CATALOG_FEED_SOURCE_REF,
5+
serializeCatalogFeed,
6+
type CatalogFeedEntry,
7+
} from "clawhub-schema";
8+
import { v } from "convex/values";
9+
import { internal } from "./_generated/api";
10+
import type { Doc } from "./_generated/dataModel";
11+
import { internalAction, internalMutation, internalQuery } from "./_generated/server";
12+
import type { QueryCtx } from "./_generated/server";
13+
import { sha256Hex } from "./lib/clawpack";
14+
import { getPackageReleaseArtifactSha256 } from "./lib/packageArtifacts";
15+
import { isPackageBlockedFromPublic, resolvePackageReleaseScanStatus } from "./lib/packageSecurity";
16+
import { getOwnerPublisher } from "./lib/publishers";
17+
18+
const CATALOG_FEED_DESCRIPTION = "Official OpenClaw plugins published on ClawHub.";
19+
const CATALOG_FEED_PAGE_SIZE = 100;
20+
const MAX_CATALOG_FEED_ENTRIES = 500;
21+
const CATALOG_FEED_FAMILIES = ["code-plugin", "bundle-plugin"] as const;
22+
23+
type CatalogQueryCtx = Pick<QueryCtx, "db">;
24+
type CatalogFeedPublicationResult = {
25+
publicationId: string;
26+
feedId: string;
27+
sequence: number;
28+
payloadSha256: string;
29+
publishedAt: number;
30+
entryCount: number;
31+
};
32+
33+
const catalogFeedEntryValidator = v.object({
34+
type: v.literal("plugin"),
35+
id: v.string(),
36+
title: v.string(),
37+
version: v.string(),
38+
state: v.union(
39+
v.literal("available"),
40+
v.literal("recommended"),
41+
v.literal("disabled"),
42+
v.literal("blocked"),
43+
v.literal("deprecated"),
44+
),
45+
publisher: v.object({
46+
id: v.string(),
47+
trust: v.union(v.literal("official"), v.literal("community")),
48+
}),
49+
install: v.object({
50+
candidates: v.array(
51+
v.object({
52+
sourceRef: v.string(),
53+
package: v.string(),
54+
version: v.string(),
55+
integrity: v.string(),
56+
}),
57+
),
58+
}),
59+
});
60+
61+
async function buildEntry(
62+
ctx: CatalogQueryCtx,
63+
pkg: Doc<"packages">,
64+
): Promise<CatalogFeedEntry | null> {
65+
if (pkg.channel !== "official" || !pkg.latestReleaseId) return null;
66+
const release = await ctx.db.get(pkg.latestReleaseId);
67+
if (!release || release.softDeletedAt) return null;
68+
69+
const scanStatus = resolvePackageReleaseScanStatus(release);
70+
if (isPackageBlockedFromPublic(scanStatus)) return null;
71+
const artifactSha256 = getPackageReleaseArtifactSha256(release);
72+
if (!artifactSha256) return null;
73+
74+
const owner = await getOwnerPublisher(ctx, {
75+
ownerPublisherId: pkg.ownerPublisherId,
76+
ownerUserId: pkg.ownerUserId,
77+
});
78+
const publisherId = owner?.handle?.trim();
79+
if (!publisherId) return null;
80+
81+
const packageName = pkg.name.trim();
82+
const id = pkg.normalizedName.trim();
83+
const title = pkg.displayName.trim() || packageName;
84+
const version = release.version.trim();
85+
if (!packageName || !id || !title || !version) return null;
86+
87+
return {
88+
type: "plugin",
89+
id,
90+
title,
91+
version,
92+
state: "available",
93+
publisher: {
94+
id: publisherId,
95+
trust: "official",
96+
},
97+
install: {
98+
candidates: [
99+
{
100+
sourceRef: CATALOG_FEED_SOURCE_REF,
101+
package: packageName,
102+
version,
103+
integrity: `sha256:${artifactSha256}`,
104+
},
105+
],
106+
},
107+
};
108+
}
109+
110+
async function listFamilyEntries(
111+
ctx: CatalogQueryCtx,
112+
family: (typeof CATALOG_FEED_FAMILIES)[number],
113+
) {
114+
const entries: CatalogFeedEntry[] = [];
115+
let cursor: string | null = null;
116+
117+
while (true) {
118+
const page = await ctx.db
119+
.query("packages")
120+
.withIndex("by_active_family_official_downloads", (q) =>
121+
q.eq("softDeletedAt", undefined).eq("family", family).eq("isOfficial", true),
122+
)
123+
.order("desc")
124+
.paginate({ cursor, numItems: CATALOG_FEED_PAGE_SIZE });
125+
126+
for (const pkg of page.page) {
127+
const entry = await buildEntry(ctx, pkg);
128+
if (entry) entries.push(entry);
129+
if (entries.length > MAX_CATALOG_FEED_ENTRIES) {
130+
throw new Error(`Catalog feed exceeds ${MAX_CATALOG_FEED_ENTRIES} entries`);
131+
}
132+
}
133+
if (page.isDone) return entries;
134+
cursor = page.continueCursor;
135+
}
136+
}
137+
138+
async function listOfficialEntriesFromDb(ctx: CatalogQueryCtx) {
139+
const entries = (
140+
await Promise.all(CATALOG_FEED_FAMILIES.map((family) => listFamilyEntries(ctx, family)))
141+
).flat();
142+
return entries.sort((left, right) => left.id.localeCompare(right.id));
143+
}
144+
145+
export const listOfficialEntries = internalQuery({
146+
args: {},
147+
handler: async (ctx) => await listOfficialEntriesFromDb(ctx),
148+
});
149+
150+
export const buildFeed = internalQuery({
151+
args: {
152+
generatedAt: v.string(),
153+
expiresAt: v.string(),
154+
sequence: v.number(),
155+
},
156+
handler: async (ctx, args) => ({
157+
schemaVersion: CATALOG_FEED_SCHEMA_VERSION,
158+
id: CATALOG_FEED_ID,
159+
generatedAt: args.generatedAt,
160+
sequence: args.sequence,
161+
expiresAt: args.expiresAt,
162+
description: CATALOG_FEED_DESCRIPTION,
163+
entries: await listOfficialEntriesFromDb(ctx),
164+
}),
165+
});
166+
167+
export const storePublication = internalMutation({
168+
args: {
169+
generatedAt: v.string(),
170+
expiresAt: v.string(),
171+
entries: v.array(catalogFeedEntryValidator),
172+
},
173+
handler: async (ctx, args) => {
174+
const latest = await ctx.db
175+
.query("catalogFeedPublications")
176+
.withIndex("by_feed", (q) => q.eq("feedId", CATALOG_FEED_ID))
177+
.unique();
178+
const sequence = (latest?.sequence ?? 0) + 1;
179+
const payload = serializeCatalogFeed({
180+
schemaVersion: CATALOG_FEED_SCHEMA_VERSION,
181+
id: CATALOG_FEED_ID,
182+
generatedAt: args.generatedAt,
183+
sequence,
184+
expiresAt: args.expiresAt,
185+
description: CATALOG_FEED_DESCRIPTION,
186+
entries: args.entries,
187+
});
188+
const payloadSha256 = await sha256Hex(new TextEncoder().encode(payload));
189+
const publishedAt = Date.now();
190+
const publication = {
191+
feedId: CATALOG_FEED_ID,
192+
sequence,
193+
generatedAt: args.generatedAt,
194+
expiresAt: args.expiresAt,
195+
payload,
196+
payloadSha256,
197+
publishedAt,
198+
};
199+
const publicationId = latest
200+
? (await ctx.db.patch(latest._id, publication), latest._id)
201+
: await ctx.db.insert("catalogFeedPublications", publication);
202+
return {
203+
publicationId,
204+
feedId: CATALOG_FEED_ID,
205+
sequence,
206+
payloadSha256,
207+
publishedAt,
208+
entryCount: args.entries.length,
209+
};
210+
},
211+
});
212+
213+
export const publish = internalAction({
214+
args: {
215+
expiresAt: v.string(),
216+
},
217+
handler: async (ctx, args): Promise<CatalogFeedPublicationResult> => {
218+
const generatedAt = new Date().toISOString();
219+
const feed: { entries: CatalogFeedEntry[] } = await ctx.runQuery(
220+
internal.catalogFeed.buildFeed,
221+
{
222+
generatedAt,
223+
expiresAt: args.expiresAt,
224+
sequence: 0,
225+
},
226+
);
227+
const result: CatalogFeedPublicationResult = await ctx.runMutation(
228+
internal.catalogFeed.storePublication,
229+
{
230+
generatedAt,
231+
expiresAt: args.expiresAt,
232+
entries: feed.entries,
233+
},
234+
);
235+
return result;
236+
},
237+
});
238+
239+
export const getLatestPublication = internalQuery({
240+
args: {},
241+
handler: async (ctx) =>
242+
await ctx.db
243+
.query("catalogFeedPublications")
244+
.withIndex("by_feed", (q) => q.eq("feedId", CATALOG_FEED_ID))
245+
.unique(),
246+
});

‎convex/http.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import {
4646
starsPostRouterV1Http,
4747
transfersGetRouterV1Http,
4848
banAppealContextV1Http,
49+
catalogFeedV1Http,
4950
usersGetRouterV1Http,
5051
usersListV1Http,
5152
usersPostRouterV1Http,
@@ -130,6 +131,12 @@ http.route({
130131
handler: listBundlePluginsV1Http,
131132
});
132133

134+
http.route({
135+
path: ApiRoutes.catalogFeed,
136+
method: "GET",
137+
handler: catalogFeedV1Http,
138+
});
139+
133140
http.route({
134141
pathPrefix: `${ApiRoutes.skills}/`,
135142
method: "GET",

‎convex/httpApiV1.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { httpAction } from "./functions";
2+
import { catalogFeedV1Handler } from "./httpApiV1/catalogFeedV1";
23
import { contentRightsV1Handler } from "./httpApiV1/contentRightsV1";
34
import { verifyDocsSessionV1Handler } from "./httpApiV1/docsSessionV1";
45
import {
@@ -56,6 +57,7 @@ export const listBundlePluginsV1Http = httpAction(listBundlePluginsV1Handler);
5657
export const verifyDocsSessionV1Http = httpAction(verifyDocsSessionV1Handler);
5758
export const createPublisherV1Http = httpAction(createPublisherV1Handler);
5859
export const contentRightsV1Http = httpAction(contentRightsV1Handler);
60+
export const catalogFeedV1Http = httpAction(catalogFeedV1Handler);
5961

6062
export const searchSkillsV1Http = httpAction(searchSkillsV1Handler);
6163
export const resolveSkillVersionV1Http = httpAction(resolveSkillVersionV1Handler);
@@ -97,6 +99,7 @@ export const __handlers = {
9799
verifyDocsSessionV1Handler,
98100
createPublisherV1Handler,
99101
contentRightsV1Handler,
102+
catalogFeedV1Handler,
100103
searchSkillsV1Handler,
101104
resolveSkillVersionV1Handler,
102105
listSkillsV1Handler,

‎convex/httpApiV1/catalogFeedV1.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { internal } from "../_generated/api";
2+
import type { ActionCtx } from "../_generated/server";
3+
import { corsHeaders, mergeHeaders } from "../lib/httpHeaders";
4+
5+
function matchesEtag(request: Request, etag: string) {
6+
const header = request.headers.get("if-none-match");
7+
if (!header) return false;
8+
return header
9+
.split(",")
10+
.map((value) => value.trim())
11+
.some((value) => value === "*" || value === etag);
12+
}
13+
14+
export async function catalogFeedV1Handler(ctx: ActionCtx, request: Request) {
15+
const publication = await ctx.runQuery(internal.catalogFeed.getLatestPublication, {});
16+
if (!publication) {
17+
return new Response("Catalog feed is not published", {
18+
status: 503,
19+
headers: mergeHeaders(
20+
{
21+
"Content-Type": "text/plain; charset=utf-8",
22+
"Cache-Control": "no-store",
23+
},
24+
corsHeaders(),
25+
),
26+
});
27+
}
28+
29+
const etag = `"sha256:${publication.payloadSha256}"`;
30+
const headers = mergeHeaders(
31+
{
32+
"Content-Type": "application/json; charset=utf-8",
33+
"Cache-Control": "public, max-age=60, s-maxage=300, stale-while-revalidate=86400",
34+
"Surrogate-Control": "max-age=300, stale-while-revalidate=86400",
35+
ETag: etag,
36+
"Last-Modified": new Date(publication.publishedAt).toUTCString(),
37+
"X-Catalog-Feed-Sequence": String(publication.sequence),
38+
"X-Content-SHA256": publication.payloadSha256,
39+
"X-Content-Type-Options": "nosniff",
40+
Vary: "Accept-Encoding",
41+
},
42+
corsHeaders(),
43+
);
44+
45+
if (matchesEtag(request, etag)) return new Response(null, { status: 304, headers });
46+
return new Response(publication.payload, { status: 200, headers });
47+
}

‎convex/lib/retentionPolicy.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ export const RETENTION_POLICIES = {
177177
packageAppeals: permanent("Package moderation appeals and audit history."),
178178
packageModerationEventLogs: permanent("Package moderation event audit log."),
179179
officialPluginMigrations: permanent("Official plugin migration state."),
180+
catalogFeedPublications: permanent("Current published hosted catalog feed snapshot."),
180181
stars: permanent("User star records."),
181182
auditLogs: permanent("Audit logs are durable compliance/security history."),
182183
publisherAbuseScoreRuns: permanent("Abuse scoring run history."),

0 commit comments

Comments
 (0)