Skip to content

Commit bea6854

Browse files
authored
feat: agent description compression with lazy loading (anomalyco#696)
* feat: add agent description compression with lazy loading (anomalyco#491) Agent descriptions consume ~26k tokens (121KB across 27 agents). This adds a compression library with three modes: - catalog: metadata only (~2-3k tokens) for agent selection - summary: metadata + first paragraph (~4-5k tokens) for routing - full: no compression, for when agent is invoked Includes lazy-load function to fetch full agent body on demand. 21 tests covering parsing, compression, filtering, and real agents dir. * fix: update JSDoc to include all stats fields in buildAgentCatalog Add compressedBytes and mode to the documented return type, matching the actual implementation.
1 parent b981c76 commit bea6854

2 files changed

Lines changed: 272 additions & 280 deletions

File tree

‎scripts/lib/agent-compress.js‎

Lines changed: 47 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ const path = require('path');
88
* Returns { frontmatter: {}, body: string }.
99
*/
1010
function parseFrontmatter(content) {
11-
const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
11+
const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n([\s\S]*))?$/);
1212
if (!match) {
1313
return { frontmatter: {}, body: content };
1414
}
@@ -38,21 +38,29 @@ function parseFrontmatter(content) {
3838
frontmatter[key] = value;
3939
}
4040

41-
return { frontmatter, body: match[2] };
41+
return { frontmatter, body: match[2] || '' };
4242
}
4343

4444
/**
4545
* Extract the first meaningful paragraph from agent body as a summary.
46-
* Skips headings and blank lines, returns up to maxSentences sentences.
46+
* Skips headings, list items, code blocks, and table rows.
4747
*/
4848
function extractSummary(body, maxSentences = 1) {
4949
const lines = body.split('\n');
5050
const paragraphs = [];
5151
let current = [];
52+
let inCodeBlock = false;
5253

5354
for (const line of lines) {
5455
const trimmed = line.trim();
5556

57+
// Track fenced code blocks
58+
if (trimmed.startsWith('```')) {
59+
inCodeBlock = !inCodeBlock;
60+
continue;
61+
}
62+
if (inCodeBlock) continue;
63+
5664
if (trimmed === '') {
5765
if (current.length > 0) {
5866
paragraphs.push(current.join(' '));
@@ -61,40 +69,36 @@ function extractSummary(body, maxSentences = 1) {
6169
continue;
6270
}
6371

64-
// Skip headings
65-
if (trimmed.startsWith('#')) {
72+
// Skip headings, list items (bold, plain, asterisk), numbered lists, table rows
73+
if (
74+
trimmed.startsWith('#') ||
75+
trimmed.startsWith('- ') ||
76+
trimmed.startsWith('* ') ||
77+
/^\d+\.\s/.test(trimmed) ||
78+
trimmed.startsWith('|')
79+
) {
6680
if (current.length > 0) {
6781
paragraphs.push(current.join(' '));
6882
current = [];
6983
}
7084
continue;
7185
}
7286

73-
// Skip list items, code blocks, etc.
74-
if (trimmed.startsWith('```') || trimmed.startsWith('- **') || trimmed.startsWith('|')) {
75-
continue;
76-
}
77-
7887
current.push(trimmed);
7988
}
8089
if (current.length > 0) {
8190
paragraphs.push(current.join(' '));
8291
}
8392

84-
// Find first non-empty paragraph
8593
const firstParagraph = paragraphs.find(p => p.length > 0);
86-
if (!firstParagraph) {
87-
return '';
88-
}
94+
if (!firstParagraph) return '';
8995

90-
// Extract up to maxSentences sentences
9196
const sentences = firstParagraph.match(/[^.!?]+[.!?]+/g) || [firstParagraph];
92-
return sentences.slice(0, maxSentences).join(' ').trim();
97+
return sentences.slice(0, maxSentences).map(s => s.trim()).join(' ').trim();
9398
}
9499

95100
/**
96101
* Load and parse a single agent file.
97-
* Returns the full agent object with frontmatter and body.
98102
*/
99103
function loadAgent(filePath) {
100104
const content = fs.readFileSync(filePath, 'utf8');
@@ -116,9 +120,7 @@ function loadAgent(filePath) {
116120
* Load all agents from a directory.
117121
*/
118122
function loadAgents(agentsDir) {
119-
if (!fs.existsSync(agentsDir)) {
120-
return [];
121-
}
123+
if (!fs.existsSync(agentsDir)) return [];
122124

123125
return fs.readdirSync(agentsDir)
124126
.filter(f => f.endsWith('.md'))
@@ -127,8 +129,7 @@ function loadAgents(agentsDir) {
127129
}
128130

129131
/**
130-
* Compress an agent to its catalog entry (metadata only).
131-
* This is the minimal representation needed for agent selection.
132+
* Compress an agent to catalog entry (metadata only).
132133
*/
133134
function compressToCatalog(agent) {
134135
return {
@@ -140,31 +141,34 @@ function compressToCatalog(agent) {
140141
}
141142

142143
/**
143-
* Compress an agent to a summary entry (metadata + first paragraph).
144-
* More context than catalog, less than full body.
144+
* Compress an agent to summary entry (metadata + first paragraph).
145145
*/
146146
function compressToSummary(agent) {
147147
return {
148-
name: agent.name,
149-
description: agent.description,
150-
tools: agent.tools,
151-
model: agent.model,
148+
...compressToCatalog(agent),
152149
summary: extractSummary(agent.body),
153150
};
154151
}
155152

153+
const allowedModes = ['catalog', 'summary', 'full'];
154+
156155
/**
157-
* Build a full compressed catalog from a directory of agents.
156+
* Build a compressed catalog from a directory of agents.
158157
*
159158
* Modes:
160159
* - 'catalog': name, description, tools, model only (~2-3k tokens for 27 agents)
161160
* - 'summary': catalog + first paragraph summary (~4-5k tokens)
162161
* - 'full': no compression, full body included
163162
*
164-
* Returns { agents: [], stats: { totalAgents, originalBytes, compressedTokenEstimate } }
163+
* Returns { agents: [], stats: { totalAgents, originalBytes, compressedBytes, compressedTokenEstimate, mode } }
165164
*/
166165
function buildAgentCatalog(agentsDir, options = {}) {
167166
const mode = options.mode || 'catalog';
167+
168+
if (!allowedModes.includes(mode)) {
169+
throw new Error(`Invalid mode "${mode}". Allowed modes: ${allowedModes.join(', ')}`);
170+
}
171+
168172
const filter = options.filter || null;
169173

170174
let agents = loadAgents(agentsDir);
@@ -207,14 +211,24 @@ function buildAgentCatalog(agentsDir, options = {}) {
207211
}
208212

209213
/**
210-
* Lazy-load a single agent's full content by name from a directory.
214+
* Lazy-load a single agent's full content by name.
211215
* Returns null if not found.
212216
*/
213217
function lazyLoadAgent(agentsDir, agentName) {
214-
const filePath = path.join(agentsDir, `${agentName}.md`);
215-
if (!fs.existsSync(filePath)) {
218+
// Validate agentName: only allow alphanumeric, hyphen, underscore
219+
if (!/^[\w-]+$/.test(agentName)) {
216220
return null;
217221
}
222+
223+
const filePath = path.resolve(agentsDir, `${agentName}.md`);
224+
225+
// Verify the resolved path is still within agentsDir
226+
const resolvedAgentsDir = path.resolve(agentsDir);
227+
if (!filePath.startsWith(resolvedAgentsDir + path.sep)) {
228+
return null;
229+
}
230+
231+
if (!fs.existsSync(filePath)) return null;
218232
return loadAgent(filePath);
219233
}
220234

0 commit comments

Comments
 (0)