Skip to content

Commit afc89c8

Browse files
authored
feat(mcp): add titles, inspector tags and structured output to agent tools (#411)
Every built-in agent tool has a short title, every tool is tagged with its inspector through devframe's tags field, and 30 list and inspect tools send structuredContent alongside unchanged text. inspect-component and highlight now ask for a selector instead of failing when it is missing. Refs #372.
1 parent 2e71bcc commit afc89c8

20 files changed

Lines changed: 487 additions & 2 deletions

‎apps/docs/src/content/agents/mcp-server.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,20 @@ The stdio server sends none. The HTTP endpoint needs devframe to pass the instru
245245

246246
The server registers tools with a colon, as `pangular:get-routes`. MCP clients see them with an underscore, as `pangular_get-routes`. Calls with either form work.
247247

248+
### Titles and structured output
249+
250+
Each tool also has a short title, such as **List connected pages** for `pangular_list-pages`. The server sends it as the tool's `title` and `annotations.title`, and clients show it in their tool lists and approval prompts.
251+
252+
A tool answers with a JSON object, sent as text. Most tools answer `{ "markdown": "..." }`. The tools below also declare an `outputSchema` and send the same object as `structuredContent`, so a client can read the answer without parsing the text. The text is the same either way.
253+
254+
| Tools | `structuredContent` |
255+
| ----------------------------------------------------------------------------------------------------------- | ------------------------------ |
256+
| Every tool whose name starts with `list-` or `inspect-` | `{ "markdown": "..." }` |
257+
| `analog-routes`, `analog-api-routes`, `analog-server-functions`, `analog-server-calls` and `analog-content` | `{ "markdown": "..." }` |
258+
| `build-meta` | The project and version fields |
259+
260+
The other source scan tools answer with a JSON array, and `screenshot-component` with an image, so they send text and images only.
261+
248262
### Read and action tools
249263

250264
Each tool has a safety tier: `read`, `action` or `destructive`. The server marks `read` tools as read-only for your client, and `destructive` tools as destructive. Twenty tools act on the app, so the server does not mark them read-only:

‎apps/docs/src/content/agents/tools.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ This page lists every tool the [MCP server](./mcp-server.md) exposes. Each group
1717

1818
Tool ids use a colon, as `pangular:get-routes`. MCP clients see them with an underscore, as `pangular_get-routes`. The tables below drop the `pangular:` prefix. The `read` tools are also registered on the page under the same names, for [browser agents](./browser-agents.md).
1919

20+
Each tool has a short title that clients show in their tool lists, and the `list-` and `inspect-` tools also send their answer as structured content. See [Titles and structured output](./mcp-server.md#titles-and-structured-output).
21+
2022
### Source and live tools
2123

2224
Each tool reads from one of three places. The stdio server registers only the tools that read your source, because no page reaches it.
Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
import { createHostContext } from 'devframe/node';
2+
import { afterEach, describe, expect, it, vi } from 'vitest';
3+
import { PANGULAR_INSPECTORS, agentInspector, agentTags } from '../config.ts';
4+
import pangular from '../devframe.ts';
5+
import { initPangularHub } from '../hub.ts';
6+
import { MARKDOWN_OUTPUT_SCHEMA } from '../rpc/agent-output.ts';
7+
import { makeProject } from './analog-fixture.ts';
8+
9+
const closers: (() => Promise<void>)[] = [];
10+
11+
afterEach(async () => {
12+
vi.restoreAllMocks();
13+
for (const close of closers.splice(0)) await close();
14+
delete (globalThis as { __PANGULAR_HUB__?: unknown }).__PANGULAR_HUB__;
15+
});
16+
17+
async function manifest() {
18+
const host = {
19+
mountStatic: () => {},
20+
resolveOrigin: () => 'http://localhost',
21+
getStorageDir: () => '',
22+
};
23+
const ctx = await createHostContext({ cwd: process.cwd(), mode: 'dev', host: host as never });
24+
await pangular.setup(ctx as never);
25+
return ctx.agent.list().tools.filter((tool) => tool.id.startsWith('pangular:'));
26+
}
27+
28+
interface McpTool {
29+
name: string;
30+
title?: string;
31+
outputSchema?: Record<string, unknown>;
32+
annotations?: { title?: string };
33+
}
34+
35+
interface McpCallResult {
36+
content: { type: string; text: string }[];
37+
structuredContent?: unknown;
38+
isError?: boolean;
39+
}
40+
41+
/** A hub with its MCP endpoint, and a JSON-RPC client for it. */
42+
async function bootMcp() {
43+
vi.spyOn(console, 'log').mockImplementation(() => {});
44+
const hub = initPangularHub({
45+
cwd: makeProject({ 'package.json': '{}' }),
46+
ws: false,
47+
auth: false,
48+
allowedOrigins: false,
49+
});
50+
closers.push(() => hub.close());
51+
await hub.ready;
52+
const ctx = await hub.context;
53+
let id = 0;
54+
const call = async <T>(method: string, params: object = {}): Promise<T> => {
55+
const response = await hub.handler(
56+
new Request('http://localhost/__devframes/__mcp', {
57+
method: 'POST',
58+
headers: {
59+
'content-type': 'application/json',
60+
accept: 'application/json, text/event-stream',
61+
origin: 'http://localhost:4000',
62+
},
63+
body: JSON.stringify({ jsonrpc: '2.0', id: ++id, method, params }),
64+
}),
65+
);
66+
const text = await response.text();
67+
const json = text.startsWith('{') ? text : /^data: (.*)$/m.exec(text)?.[1];
68+
return (JSON.parse(json ?? '{}') as { result: T }).result;
69+
};
70+
await call('initialize', {
71+
protocolVersion: '2025-06-18',
72+
capabilities: {},
73+
clientInfo: { name: 'test', version: '0' },
74+
});
75+
const tools = (await call<{ tools: McpTool[] }>('tools/list')).tools.filter((tool) =>
76+
tool.name.startsWith('pangular_'),
77+
);
78+
const invoke = (name: string, args: Record<string, unknown> = {}) =>
79+
call<McpCallResult>('tools/call', { name, arguments: args });
80+
const push = (name: string, payload: unknown) =>
81+
ctx.rpc.invokeLocal(`pangular:${name}` as never, ...([payload] as never));
82+
return { tools, invoke, push };
83+
}
84+
85+
/** Checks a value against the JSON Schema keywords the output schemas use. */
86+
function schemaErrors(schema: Record<string, unknown>, value: unknown, at = '$'): string[] {
87+
const errors: string[] = [];
88+
const type = schema['type'];
89+
const actual = Array.isArray(value)
90+
? 'array'
91+
: value === null
92+
? 'null'
93+
: Number.isInteger(value)
94+
? 'integer'
95+
: typeof value;
96+
const types = Array.isArray(type) ? type : type === undefined ? [] : [type];
97+
if (
98+
types.length &&
99+
!types.includes(actual) &&
100+
!(actual === 'integer' && types.includes('number'))
101+
) {
102+
return [`${at} is ${actual}, expected ${types.join(' or ')}`];
103+
}
104+
if (Array.isArray(schema['enum']) && !schema['enum'].includes(value)) {
105+
errors.push(`${at} is not one of the enum values`);
106+
}
107+
if (Array.isArray(schema['anyOf'])) {
108+
const options = schema['anyOf'] as Record<string, unknown>[];
109+
if (!options.some((option) => schemaErrors(option, value, at).length === 0)) {
110+
errors.push(`${at} matches no anyOf option`);
111+
}
112+
}
113+
if (actual === 'object' && value) {
114+
const record = value as Record<string, unknown>;
115+
const properties = (schema['properties'] ?? {}) as Record<string, Record<string, unknown>>;
116+
for (const key of (schema['required'] ?? []) as string[]) {
117+
if (!(key in record)) errors.push(`${at}.${key} is missing`);
118+
}
119+
for (const [key, item] of Object.entries(record)) {
120+
if (properties[key]) errors.push(...schemaErrors(properties[key], item, `${at}.${key}`));
121+
else if (schema['additionalProperties'] === false) errors.push(`${at}.${key} is not allowed`);
122+
}
123+
}
124+
if (actual === 'array' && schema['items']) {
125+
(value as unknown[]).forEach((item, index) =>
126+
errors.push(
127+
...schemaErrors(schema['items'] as Record<string, unknown>, item, `${at}[${index}]`),
128+
),
129+
);
130+
}
131+
return errors;
132+
}
133+
134+
/** What `tools/call` sent as text before the tools had titles and output schemas. */
135+
const TEXT_BEFORE = {
136+
'pangular_list-pages': JSON.stringify(
137+
{
138+
markdown:
139+
'No page is reporting. Live data needs a page: connect through the MCP endpoint of the server that runs the app, with the app open in a browser.',
140+
},
141+
null,
142+
2,
143+
),
144+
'pangular_list-components': JSON.stringify(
145+
{
146+
markdown:
147+
'No component tree has been reported. This is what a page that has never connected reports, and also what a connected page reports when its components are not readable. Live data needs a page: connect through the MCP endpoint of the server that runs the app, with the app open in a browser. The stdio server has no page attached and only ever reports this.',
148+
},
149+
null,
150+
2,
151+
),
152+
'pangular_get-routes': '[]',
153+
};
154+
155+
describe('agent tool titles', () => {
156+
it('gives every built-in tool a short title of its own', async () => {
157+
const tools = (await manifest()).filter((tool) => !tool.id.startsWith('pangular:plugin:'));
158+
expect(tools.length).toBeGreaterThan(80);
159+
const seen = new Map<string, string>();
160+
for (const tool of tools) {
161+
expect(tool.title, `${tool.id} has no title`).not.toBe(tool.id);
162+
expect(tool.title.length, `${tool.id}: "${tool.title}"`).toBeLessThanOrEqual(40);
163+
expect(tool.title, tool.id).not.toMatch(/pangular/i);
164+
expect(tool.title, tool.id).toMatch(/^[A-Z]/);
165+
expect(seen.get(tool.title), `"${tool.title}" is used twice`).toBeUndefined();
166+
seen.set(tool.title, tool.id);
167+
}
168+
});
169+
170+
it('sends the title as the MCP title and the title annotation', async () => {
171+
const { tools } = await bootMcp();
172+
for (const tool of tools) {
173+
expect(tool.title, tool.name).toBeTruthy();
174+
expect(tool.title, tool.name).not.toMatch(/^pangular[:_]/);
175+
expect(tool.annotations?.title, tool.name).toBe(tool.title);
176+
}
177+
});
178+
});
179+
180+
describe('agent tool tags', () => {
181+
it('tags each tool with its inspector and leaves shared tools untagged', async () => {
182+
for (const tool of await manifest()) {
183+
const inspector = agentInspector(tool.id);
184+
if (tool.id === 'pangular:clear-recordings') {
185+
expect(tool.tags).toEqual(['http', 'router', 'analog']);
186+
} else if (inspector) {
187+
expect(tool.tags, tool.id).toEqual([inspector]);
188+
} else {
189+
expect(tool.tags, tool.id).toBeUndefined();
190+
}
191+
for (const tag of tool.tags ?? []) expect(PANGULAR_INSPECTORS).toContain(tag);
192+
}
193+
});
194+
195+
it('tags plugin tools as plugins', () => {
196+
expect(agentTags('pangular:plugin:cart:totals')).toEqual(['plugins']);
197+
expect(agentTags('pangular:list-pages')).toBeUndefined();
198+
});
199+
});
200+
201+
describe('agent tool output schemas', () => {
202+
it('advertises only object output schemas', async () => {
203+
const { tools } = await bootMcp();
204+
const withSchema = tools.filter((tool) => tool.outputSchema);
205+
// The set the MCP server docs list under "Titles and structured output".
206+
const documented = (name: string) =>
207+
/^pangular_(list|inspect)-/.test(name) ||
208+
[
209+
'pangular_analog-routes',
210+
'pangular_analog-api-routes',
211+
'pangular_analog-server-functions',
212+
'pangular_analog-server-calls',
213+
'pangular_analog-content',
214+
'pangular_build-meta',
215+
].includes(name);
216+
expect(withSchema.map((tool) => tool.name).sort()).toEqual(
217+
tools
218+
.map((tool) => tool.name)
219+
.filter(documented)
220+
.sort(),
221+
);
222+
expect(withSchema.length).toBeGreaterThan(25);
223+
for (const tool of withSchema) expect(tool.outputSchema?.['type'], tool.name).toBe('object');
224+
// Array answers and image answers carry no structured content, so they advertise no schema.
225+
for (const name of [
226+
'pangular_get-routes',
227+
'pangular_get-components',
228+
'pangular_screenshot-component',
229+
]) {
230+
expect(tools.find((tool) => tool.name === name)?.outputSchema, name).toBeUndefined();
231+
}
232+
});
233+
234+
it('returns structured content that matches the schema and the text', async () => {
235+
const { tools, invoke, push } = await bootMcp();
236+
const withSchema = tools.filter((tool) => tool.outputSchema);
237+
const check = async (tool: McpTool, args: Record<string, unknown>) => {
238+
const result = await invoke(tool.name, args);
239+
expect(result.isError, `${tool.name} ${JSON.stringify(args)}`).toBeFalsy();
240+
expect(result.structuredContent, tool.name).toBeDefined();
241+
expect(schemaErrors(tool.outputSchema!, result.structuredContent), tool.name).toEqual([]);
242+
expect(result.content).toHaveLength(1);
243+
expect(JSON.parse(result.content[0]!.text), tool.name).toEqual(result.structuredContent);
244+
};
245+
246+
// No page connected.
247+
for (const tool of withSchema) await check(tool, {});
248+
249+
// One page reporting, asked about it and about a page that is not there.
250+
await push('push-component-tree', {
251+
pageId: 'p1',
252+
roots: [{ id: 'c1', name: 'App', tag: 'app-root', children: [] }],
253+
count: 1,
254+
detail: null,
255+
});
256+
for (const tool of withSchema) {
257+
await check(tool, { selector: 'App', page: 'p1' });
258+
await check(tool, { selector: 'App', page: 'gone' });
259+
}
260+
});
261+
262+
it('asks for a selector instead of failing when it is missing', async () => {
263+
const { invoke, push } = await bootMcp();
264+
await push('push-component-tree', {
265+
pageId: 'p1',
266+
roots: [{ id: 'c1', name: 'App', tag: 'app-root', children: [] }],
267+
count: 1,
268+
detail: null,
269+
});
270+
for (const name of ['pangular_inspect-component', 'pangular_highlight']) {
271+
const result = await invoke(name, {});
272+
expect(result.isError, name).toBeFalsy();
273+
expect(result.content[0]!.text, name).toContain('Pass `selector`');
274+
}
275+
});
276+
277+
it('keeps the markdown schema to one string field', () => {
278+
expect(MARKDOWN_OUTPUT_SCHEMA).toMatchObject({
279+
type: 'object',
280+
properties: { markdown: { type: 'string' } },
281+
required: ['markdown'],
282+
});
283+
});
284+
285+
it('leaves the text content as it was', async () => {
286+
const { invoke } = await bootMcp();
287+
for (const [name, text] of Object.entries(TEXT_BEFORE)) {
288+
const result = await invoke(name);
289+
expect(result.content, name).toEqual([{ type: 'text', text }]);
290+
}
291+
});
292+
});

‎packages/devtools/src/config.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1402,6 +1402,18 @@ export function agentInspector(id: string): PangularInspector | undefined {
14021402
);
14031403
}
14041404

1405+
/**
1406+
* The tags of an agent tool: the inspector it belongs to, by the same key as
1407+
* `AGENT_INSPECTOR`. `clear-recordings` erases what the HTTP, router and
1408+
* Analog inspectors record, so it carries all three. Shared tools have none.
1409+
* Devframe keeps them in its agent manifest; MCP `tools/list` does not send them.
1410+
*/
1411+
export function agentTags(id: string): readonly PangularInspector[] | undefined {
1412+
if (agentName(id) === 'clear-recordings') return ['http', 'router', 'analog'];
1413+
const inspector = agentInspector(id);
1414+
return inspector ? [inspector] : undefined;
1415+
}
1416+
14051417
export function rpcAllowed(name: string, config: ResolvedPangularConfig): boolean {
14061418
const inspector = RPC_INSPECTOR[name];
14071419
return !inspector || config.inspectors[inspector];

0 commit comments

Comments
 (0)