Skip to content

Commit f9821d5

Browse files
houkohouko
committed
ci: build plugins-index.json in-repo so worker refresh stays under budget
Walking ~40+ plugin TOMLs via the GitHub Contents API from the worker exceeded the Workers Free 50-subrequest-per-invocation limit, leaving the daemon's signed plugins index either empty or partial after every forced refresh. Move the walk into the repo: scripts/build-plugins-index.mjs reads each plugins/<name>/plugin.toml directly from the checked-out tree and emits a sorted flat array (name, version?, description?, needs?) at plugins-index.json. The CI workflow regenerates and commits this file on every push under plugins/, then pokes the worker's /api/registry/refresh — which now fetches the single committed plugins-index.json (1 subrequest), validates the JSON shape, and re-signs it with Ed25519. Refresh cost is now constant in registry size, not linear. The dashboard's dict-shaped /api/registry payload is unchanged — that still rebuilds via the daily 02:00 UTC cron.
1 parent 14a5766 commit f9821d5

3 files changed

Lines changed: 98 additions & 14 deletions

File tree

‎.github/workflows/refresh-cache.yml‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,52 @@
11
name: Refresh registry-worker cache
22

3-
# Triggers the registry-worker to rebuild its D1 cache and re-sign the
4-
# plugins index right after content changes — without this, dashboard /
5-
# daemon would have to wait for the next 02:00 UTC cron tick.
3+
# On every push that changes plugins/, regenerate plugins-index.json
4+
# (the daemon-shaped flat array the registry-worker signs and the
5+
# librefang daemon installs from), commit it back, then poke the
6+
# worker's forced-refresh endpoint so it re-signs the new bytes
7+
# immediately — without this, daemon installs lag up to ~24h behind
8+
# the next 02:00 UTC cron tick.
69
#
7-
# Auth: REGISTRY_REFRESH_TOKEN (repo secret) is matched against the
8-
# REGISTRY_REFRESH_TOKEN worker secret in librefang-registry. Until both
9-
# are set, the worker endpoint returns 503.
10+
# Walking the on-disk repo (instead of the worker hitting GitHub
11+
# Contents API per-file) keeps the worker's refresh path under the
12+
# Workers Free 50-subrequest budget regardless of registry size.
1013

1114
on:
1215
push:
1316
branches: [main]
1417
paths:
1518
- 'plugins/**'
16-
- 'agents/**'
17-
- 'skills/**'
18-
- 'hands/**'
19-
- 'channels/**'
20-
- 'providers/**'
21-
- 'workflows/**'
22-
- 'mcp/**'
23-
workflow_dispatch: # manual trigger for ops
19+
- 'scripts/build-plugins-index.mjs'
20+
workflow_dispatch: # manual trigger for ops / first-deploy
21+
22+
permissions:
23+
contents: write # needed to commit the regenerated index back
2424

2525
jobs:
2626
refresh:
2727
runs-on: ubuntu-latest
2828
steps:
29+
- uses: actions/checkout@v4
30+
31+
- uses: actions/setup-node@v4
32+
with:
33+
node-version: '20'
34+
35+
- name: Rebuild plugins-index.json
36+
run: node scripts/build-plugins-index.mjs
37+
38+
- name: Commit regenerated index if changed
39+
run: |
40+
git config user.name "github-actions[bot]"
41+
git config user.email "github-actions[bot]@users.noreply.github.com"
42+
git add plugins-index.json
43+
if git diff --cached --quiet; then
44+
echo "plugins-index.json already up-to-date"
45+
else
46+
git commit -m "chore: regenerate plugins-index.json"
47+
git push
48+
fi
49+
2950
- name: Trigger worker refresh
3051
env:
3152
REGISTRY_REFRESH_TOKEN: ${{ secrets.REGISTRY_REFRESH_TOKEN }}

‎plugins-index.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[{"name":"auto-summarizer","version":"0.1.0","description":"Maintains a running conversation summary to help agents handle long conversations without losing context"},{"name":"context-decay","version":"0.1.0","description":"Time-based memory decay with relevance scoring for natural context forgetting"},{"name":"conversation-logger","version":"0.1.0","description":"Logs all conversations to JSONL files for auditing, analytics, and debugging"},{"name":"episodic-memory","version":"0.1.0","description":"Episode-based memory segmentation and recall for cross-conversation context continuity"},{"name":"guardrails","version":"0.1.0","description":"Safety filter that detects potentially harmful content patterns and injects warnings into agent context"},{"name":"keyword-memory","version":"0.1.0","description":"Extracts keywords and named entities from user messages and returns them as contextual memories"},{"name":"mempalace-indexer","version":"0.3.0","description":"Auto-index conversations into MemPalace and recall relevant memories. No API keys, no cloud."},{"name":"sentiment-tracker","version":"0.1.0","description":"Analyzes user message sentiment and injects emotional context so agents can respond with appropriate tone"},{"name":"todo-tracker","version":"0.1.0","description":"Detects action items and tasks mentioned in conversations, persists them, and recalls them as context"},{"name":"topic-memory","version":"0.1.0","description":"Topic-aware memory recall with keyword clustering for cross-conversation context"},{"name":"user-profile","version":"0.1.0","description":"Persistent user profiling from conversation patterns for personalized agent responses"}]

‎scripts/build-plugins-index.mjs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env node
2+
// Build plugins-index.json from the on-disk plugins/ tree.
3+
//
4+
// Walks plugins/<name>/plugin.toml, extracts the fields the LibreFang
5+
// daemon actually consumes (name, version?, description?, needs?), sorts
6+
// the result by name for byte-determinism, and writes it to the repo
7+
// root. The committed file is what the registry-worker forced-refresh
8+
// path signs — keeping the worker's input to a single subrequest
9+
// regardless of how large the registry grows.
10+
//
11+
// Run by .github/workflows/refresh-cache.yml on every push under
12+
// plugins/**, and committed back via github-actions[bot].
13+
14+
import fs from 'node:fs'
15+
import path from 'node:path'
16+
17+
const repoRoot = path.resolve(new URL('.', import.meta.url).pathname, '..')
18+
const pluginsDir = path.join(repoRoot, 'plugins')
19+
const outPath = path.join(repoRoot, 'plugins-index.json')
20+
21+
function pickString(text, key) {
22+
const m = text.match(new RegExp(`^${key}\\s*=\\s*"([^"]*)"`, 'm'))
23+
return m ? m[1] : ''
24+
}
25+
26+
function pickStringArray(text, key) {
27+
const m = text.match(new RegExp(`^${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'm'))
28+
if (!m) return undefined
29+
const items = m[1].match(/"([^"]*)"/g)?.map(s => s.replace(/"/g, ''))
30+
return items?.length ? items : undefined
31+
}
32+
33+
const entries = []
34+
for (const dir of fs.readdirSync(pluginsDir, { withFileTypes: true })) {
35+
if (!dir.isDirectory()) continue
36+
const manifest = path.join(pluginsDir, dir.name, 'plugin.toml')
37+
if (!fs.existsSync(manifest)) continue
38+
const text = fs.readFileSync(manifest, 'utf8')
39+
40+
const name = pickString(text, 'name')
41+
if (!name) continue
42+
43+
const out = { name }
44+
const version = pickString(text, 'version')
45+
if (version) out.version = version
46+
const description = pickString(text, 'description')
47+
if (description) out.description = description
48+
const needs = pickStringArray(text, 'needs')
49+
if (needs?.length) out.needs = needs
50+
entries.push(out)
51+
}
52+
53+
entries.sort((a, b) => a.name.localeCompare(b.name))
54+
55+
const json = JSON.stringify(entries)
56+
const prev = fs.existsSync(outPath) ? fs.readFileSync(outPath, 'utf8') : null
57+
if (prev === json) {
58+
console.log(`plugins-index.json unchanged (${entries.length} entries)`)
59+
process.exit(0)
60+
}
61+
fs.writeFileSync(outPath, json)
62+
console.log(`plugins-index.json updated: ${entries.length} entries`)

0 commit comments

Comments
 (0)