|
| 1 | +import { createHostContext } from 'devframe/node'; |
| 2 | +import { afterEach, describe, expect, it, vi } from 'vitest'; |
| 3 | +import { PANGULAR_INSPECTORS, agentInspector, agentTags } from '../config.ts'; |
| 4 | +import pangular from '../devframe.ts'; |
| 5 | +import { initPangularHub } from '../hub.ts'; |
| 6 | +import { MARKDOWN_OUTPUT_SCHEMA } from '../rpc/agent-output.ts'; |
| 7 | +import { makeProject } from './analog-fixture.ts'; |
| 8 | + |
| 9 | +const closers: (() => Promise<void>)[] = []; |
| 10 | + |
| 11 | +afterEach(async () => { |
| 12 | + vi.restoreAllMocks(); |
| 13 | + for (const close of closers.splice(0)) await close(); |
| 14 | + delete (globalThis as { __PANGULAR_HUB__?: unknown }).__PANGULAR_HUB__; |
| 15 | +}); |
| 16 | + |
| 17 | +async function manifest() { |
| 18 | + const host = { |
| 19 | + mountStatic: () => {}, |
| 20 | + resolveOrigin: () => 'http://localhost', |
| 21 | + getStorageDir: () => '', |
| 22 | + }; |
| 23 | + const ctx = await createHostContext({ cwd: process.cwd(), mode: 'dev', host: host as never }); |
| 24 | + await pangular.setup(ctx as never); |
| 25 | + return ctx.agent.list().tools.filter((tool) => tool.id.startsWith('pangular:')); |
| 26 | +} |
| 27 | + |
| 28 | +interface McpTool { |
| 29 | + name: string; |
| 30 | + title?: string; |
| 31 | + outputSchema?: Record<string, unknown>; |
| 32 | + annotations?: { title?: string }; |
| 33 | +} |
| 34 | + |
| 35 | +interface McpCallResult { |
| 36 | + content: { type: string; text: string }[]; |
| 37 | + structuredContent?: unknown; |
| 38 | + isError?: boolean; |
| 39 | +} |
| 40 | + |
| 41 | +/** A hub with its MCP endpoint, and a JSON-RPC client for it. */ |
| 42 | +async function bootMcp() { |
| 43 | + vi.spyOn(console, 'log').mockImplementation(() => {}); |
| 44 | + const hub = initPangularHub({ |
| 45 | + cwd: makeProject({ 'package.json': '{}' }), |
| 46 | + ws: false, |
| 47 | + auth: false, |
| 48 | + allowedOrigins: false, |
| 49 | + }); |
| 50 | + closers.push(() => hub.close()); |
| 51 | + await hub.ready; |
| 52 | + const ctx = await hub.context; |
| 53 | + let id = 0; |
| 54 | + const call = async <T>(method: string, params: object = {}): Promise<T> => { |
| 55 | + const response = await hub.handler( |
| 56 | + new Request('http://localhost/__devframes/__mcp', { |
| 57 | + method: 'POST', |
| 58 | + headers: { |
| 59 | + 'content-type': 'application/json', |
| 60 | + accept: 'application/json, text/event-stream', |
| 61 | + origin: 'http://localhost:4000', |
| 62 | + }, |
| 63 | + body: JSON.stringify({ jsonrpc: '2.0', id: ++id, method, params }), |
| 64 | + }), |
| 65 | + ); |
| 66 | + const text = await response.text(); |
| 67 | + const json = text.startsWith('{') ? text : /^data: (.*)$/m.exec(text)?.[1]; |
| 68 | + return (JSON.parse(json ?? '{}') as { result: T }).result; |
| 69 | + }; |
| 70 | + await call('initialize', { |
| 71 | + protocolVersion: '2025-06-18', |
| 72 | + capabilities: {}, |
| 73 | + clientInfo: { name: 'test', version: '0' }, |
| 74 | + }); |
| 75 | + const tools = (await call<{ tools: McpTool[] }>('tools/list')).tools.filter((tool) => |
| 76 | + tool.name.startsWith('pangular_'), |
| 77 | + ); |
| 78 | + const invoke = (name: string, args: Record<string, unknown> = {}) => |
| 79 | + call<McpCallResult>('tools/call', { name, arguments: args }); |
| 80 | + const push = (name: string, payload: unknown) => |
| 81 | + ctx.rpc.invokeLocal(`pangular:${name}` as never, ...([payload] as never)); |
| 82 | + return { tools, invoke, push }; |
| 83 | +} |
| 84 | + |
| 85 | +/** Checks a value against the JSON Schema keywords the output schemas use. */ |
| 86 | +function schemaErrors(schema: Record<string, unknown>, value: unknown, at = '$'): string[] { |
| 87 | + const errors: string[] = []; |
| 88 | + const type = schema['type']; |
| 89 | + const actual = Array.isArray(value) |
| 90 | + ? 'array' |
| 91 | + : value === null |
| 92 | + ? 'null' |
| 93 | + : Number.isInteger(value) |
| 94 | + ? 'integer' |
| 95 | + : typeof value; |
| 96 | + const types = Array.isArray(type) ? type : type === undefined ? [] : [type]; |
| 97 | + if ( |
| 98 | + types.length && |
| 99 | + !types.includes(actual) && |
| 100 | + !(actual === 'integer' && types.includes('number')) |
| 101 | + ) { |
| 102 | + return [`${at} is ${actual}, expected ${types.join(' or ')}`]; |
| 103 | + } |
| 104 | + if (Array.isArray(schema['enum']) && !schema['enum'].includes(value)) { |
| 105 | + errors.push(`${at} is not one of the enum values`); |
| 106 | + } |
| 107 | + if (Array.isArray(schema['anyOf'])) { |
| 108 | + const options = schema['anyOf'] as Record<string, unknown>[]; |
| 109 | + if (!options.some((option) => schemaErrors(option, value, at).length === 0)) { |
| 110 | + errors.push(`${at} matches no anyOf option`); |
| 111 | + } |
| 112 | + } |
| 113 | + if (actual === 'object' && value) { |
| 114 | + const record = value as Record<string, unknown>; |
| 115 | + const properties = (schema['properties'] ?? {}) as Record<string, Record<string, unknown>>; |
| 116 | + for (const key of (schema['required'] ?? []) as string[]) { |
| 117 | + if (!(key in record)) errors.push(`${at}.${key} is missing`); |
| 118 | + } |
| 119 | + for (const [key, item] of Object.entries(record)) { |
| 120 | + if (properties[key]) errors.push(...schemaErrors(properties[key], item, `${at}.${key}`)); |
| 121 | + else if (schema['additionalProperties'] === false) errors.push(`${at}.${key} is not allowed`); |
| 122 | + } |
| 123 | + } |
| 124 | + if (actual === 'array' && schema['items']) { |
| 125 | + (value as unknown[]).forEach((item, index) => |
| 126 | + errors.push( |
| 127 | + ...schemaErrors(schema['items'] as Record<string, unknown>, item, `${at}[${index}]`), |
| 128 | + ), |
| 129 | + ); |
| 130 | + } |
| 131 | + return errors; |
| 132 | +} |
| 133 | + |
| 134 | +/** What `tools/call` sent as text before the tools had titles and output schemas. */ |
| 135 | +const TEXT_BEFORE = { |
| 136 | + 'pangular_list-pages': JSON.stringify( |
| 137 | + { |
| 138 | + markdown: |
| 139 | + 'No page is reporting. Live data needs a page: connect through the MCP endpoint of the server that runs the app, with the app open in a browser.', |
| 140 | + }, |
| 141 | + null, |
| 142 | + 2, |
| 143 | + ), |
| 144 | + 'pangular_list-components': JSON.stringify( |
| 145 | + { |
| 146 | + markdown: |
| 147 | + 'No component tree has been reported. This is what a page that has never connected reports, and also what a connected page reports when its components are not readable. Live data needs a page: connect through the MCP endpoint of the server that runs the app, with the app open in a browser. The stdio server has no page attached and only ever reports this.', |
| 148 | + }, |
| 149 | + null, |
| 150 | + 2, |
| 151 | + ), |
| 152 | + 'pangular_get-routes': '[]', |
| 153 | +}; |
| 154 | + |
| 155 | +describe('agent tool titles', () => { |
| 156 | + it('gives every built-in tool a short title of its own', async () => { |
| 157 | + const tools = (await manifest()).filter((tool) => !tool.id.startsWith('pangular:plugin:')); |
| 158 | + expect(tools.length).toBeGreaterThan(80); |
| 159 | + const seen = new Map<string, string>(); |
| 160 | + for (const tool of tools) { |
| 161 | + expect(tool.title, `${tool.id} has no title`).not.toBe(tool.id); |
| 162 | + expect(tool.title.length, `${tool.id}: "${tool.title}"`).toBeLessThanOrEqual(40); |
| 163 | + expect(tool.title, tool.id).not.toMatch(/pangular/i); |
| 164 | + expect(tool.title, tool.id).toMatch(/^[A-Z]/); |
| 165 | + expect(seen.get(tool.title), `"${tool.title}" is used twice`).toBeUndefined(); |
| 166 | + seen.set(tool.title, tool.id); |
| 167 | + } |
| 168 | + }); |
| 169 | + |
| 170 | + it('sends the title as the MCP title and the title annotation', async () => { |
| 171 | + const { tools } = await bootMcp(); |
| 172 | + for (const tool of tools) { |
| 173 | + expect(tool.title, tool.name).toBeTruthy(); |
| 174 | + expect(tool.title, tool.name).not.toMatch(/^pangular[:_]/); |
| 175 | + expect(tool.annotations?.title, tool.name).toBe(tool.title); |
| 176 | + } |
| 177 | + }); |
| 178 | +}); |
| 179 | + |
| 180 | +describe('agent tool tags', () => { |
| 181 | + it('tags each tool with its inspector and leaves shared tools untagged', async () => { |
| 182 | + for (const tool of await manifest()) { |
| 183 | + const inspector = agentInspector(tool.id); |
| 184 | + if (tool.id === 'pangular:clear-recordings') { |
| 185 | + expect(tool.tags).toEqual(['http', 'router', 'analog']); |
| 186 | + } else if (inspector) { |
| 187 | + expect(tool.tags, tool.id).toEqual([inspector]); |
| 188 | + } else { |
| 189 | + expect(tool.tags, tool.id).toBeUndefined(); |
| 190 | + } |
| 191 | + for (const tag of tool.tags ?? []) expect(PANGULAR_INSPECTORS).toContain(tag); |
| 192 | + } |
| 193 | + }); |
| 194 | + |
| 195 | + it('tags plugin tools as plugins', () => { |
| 196 | + expect(agentTags('pangular:plugin:cart:totals')).toEqual(['plugins']); |
| 197 | + expect(agentTags('pangular:list-pages')).toBeUndefined(); |
| 198 | + }); |
| 199 | +}); |
| 200 | + |
| 201 | +describe('agent tool output schemas', () => { |
| 202 | + it('advertises only object output schemas', async () => { |
| 203 | + const { tools } = await bootMcp(); |
| 204 | + const withSchema = tools.filter((tool) => tool.outputSchema); |
| 205 | + // The set the MCP server docs list under "Titles and structured output". |
| 206 | + const documented = (name: string) => |
| 207 | + /^pangular_(list|inspect)-/.test(name) || |
| 208 | + [ |
| 209 | + 'pangular_analog-routes', |
| 210 | + 'pangular_analog-api-routes', |
| 211 | + 'pangular_analog-server-functions', |
| 212 | + 'pangular_analog-server-calls', |
| 213 | + 'pangular_analog-content', |
| 214 | + 'pangular_build-meta', |
| 215 | + ].includes(name); |
| 216 | + expect(withSchema.map((tool) => tool.name).sort()).toEqual( |
| 217 | + tools |
| 218 | + .map((tool) => tool.name) |
| 219 | + .filter(documented) |
| 220 | + .sort(), |
| 221 | + ); |
| 222 | + expect(withSchema.length).toBeGreaterThan(25); |
| 223 | + for (const tool of withSchema) expect(tool.outputSchema?.['type'], tool.name).toBe('object'); |
| 224 | + // Array answers and image answers carry no structured content, so they advertise no schema. |
| 225 | + for (const name of [ |
| 226 | + 'pangular_get-routes', |
| 227 | + 'pangular_get-components', |
| 228 | + 'pangular_screenshot-component', |
| 229 | + ]) { |
| 230 | + expect(tools.find((tool) => tool.name === name)?.outputSchema, name).toBeUndefined(); |
| 231 | + } |
| 232 | + }); |
| 233 | + |
| 234 | + it('returns structured content that matches the schema and the text', async () => { |
| 235 | + const { tools, invoke, push } = await bootMcp(); |
| 236 | + const withSchema = tools.filter((tool) => tool.outputSchema); |
| 237 | + const check = async (tool: McpTool, args: Record<string, unknown>) => { |
| 238 | + const result = await invoke(tool.name, args); |
| 239 | + expect(result.isError, `${tool.name} ${JSON.stringify(args)}`).toBeFalsy(); |
| 240 | + expect(result.structuredContent, tool.name).toBeDefined(); |
| 241 | + expect(schemaErrors(tool.outputSchema!, result.structuredContent), tool.name).toEqual([]); |
| 242 | + expect(result.content).toHaveLength(1); |
| 243 | + expect(JSON.parse(result.content[0]!.text), tool.name).toEqual(result.structuredContent); |
| 244 | + }; |
| 245 | + |
| 246 | + // No page connected. |
| 247 | + for (const tool of withSchema) await check(tool, {}); |
| 248 | + |
| 249 | + // One page reporting, asked about it and about a page that is not there. |
| 250 | + await push('push-component-tree', { |
| 251 | + pageId: 'p1', |
| 252 | + roots: [{ id: 'c1', name: 'App', tag: 'app-root', children: [] }], |
| 253 | + count: 1, |
| 254 | + detail: null, |
| 255 | + }); |
| 256 | + for (const tool of withSchema) { |
| 257 | + await check(tool, { selector: 'App', page: 'p1' }); |
| 258 | + await check(tool, { selector: 'App', page: 'gone' }); |
| 259 | + } |
| 260 | + }); |
| 261 | + |
| 262 | + it('asks for a selector instead of failing when it is missing', async () => { |
| 263 | + const { invoke, push } = await bootMcp(); |
| 264 | + await push('push-component-tree', { |
| 265 | + pageId: 'p1', |
| 266 | + roots: [{ id: 'c1', name: 'App', tag: 'app-root', children: [] }], |
| 267 | + count: 1, |
| 268 | + detail: null, |
| 269 | + }); |
| 270 | + for (const name of ['pangular_inspect-component', 'pangular_highlight']) { |
| 271 | + const result = await invoke(name, {}); |
| 272 | + expect(result.isError, name).toBeFalsy(); |
| 273 | + expect(result.content[0]!.text, name).toContain('Pass `selector`'); |
| 274 | + } |
| 275 | + }); |
| 276 | + |
| 277 | + it('keeps the markdown schema to one string field', () => { |
| 278 | + expect(MARKDOWN_OUTPUT_SCHEMA).toMatchObject({ |
| 279 | + type: 'object', |
| 280 | + properties: { markdown: { type: 'string' } }, |
| 281 | + required: ['markdown'], |
| 282 | + }); |
| 283 | + }); |
| 284 | + |
| 285 | + it('leaves the text content as it was', async () => { |
| 286 | + const { invoke } = await bootMcp(); |
| 287 | + for (const [name, text] of Object.entries(TEXT_BEFORE)) { |
| 288 | + const result = await invoke(name); |
| 289 | + expect(result.content, name).toEqual([{ type: 'text', text }]); |
| 290 | + } |
| 291 | + }); |
| 292 | +}); |
0 commit comments