Skip to content

Commit fd3173d

Browse files
authored
refactor(mcp): project interaction family output schemas from their owning module (#3025)
* refactor(mcp): project interaction family output schemas from their owning module Moves the 6 hand-authored MCP outputSchema entries for press, click, fill, longpress, hover, and find out of command-output-schemas.ts into a frozen INTERACTION_COMMAND_OUTPUT_SCHEMAS map exported from src/commands/interaction/index.ts, following the #2810 projection seam. Registers the family in the PROJECTED_FAMILIES disjointness check and adds a colocated command-tools-interaction-schemas.test.ts proving reference equality with the module object, including the settle-observation copy for the five settle-capable entries (find has no such trait). postActionSurfaceChangeSchema is duplicated locally rather than imported back from command-output-schemas.ts (which still needs it for the generic --settle observation), matching the constSchema duplication precedent from the system family's move; responseCostSchema and the other interaction-only helpers had no other consumer and moved outright. Part of #2819. * refactor(mcp): declare shared schema primitives once
1 parent 3d903d4 commit fd3173d

5 files changed

Lines changed: 353 additions & 268 deletions

File tree

‎src/commands/command-input.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,10 @@ export function stringSchema(description?: string): JsonSchema {
115115
return { type: 'string', ...(description ? { description } : {}) };
116116
}
117117

118+
export function nullableStringSchema(description?: string): JsonSchema {
119+
return { type: ['string', 'null'], ...(description ? { description } : {}) };
120+
}
121+
118122
export function numberSchema(
119123
description?: string,
120124
options: { min?: number; max?: number } = {},

‎src/commands/interaction/index.ts‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,23 @@ import type {
2020
TypeTextOptions,
2121
} from '@agent-device/contracts/client';
2222
import type { CommandSchemaOverride } from '@agent-device/command-registry/command-schema';
23+
import type { CommandResultMap } from '@agent-device/command-registry/command-result';
2324
import {
2425
REPEATED_TOUCH_FLAGS,
2526
SELECTOR_SNAPSHOT_FLAGS,
2627
} from '@agent-device/command-registry/flag-groups';
2728
import { postActionObservationCliFlags } from '../post-action-observation-grammar.ts';
29+
import type { JsonSchema } from '../command-contract.ts';
2830
import {
31+
booleanSchema,
32+
constSchema,
33+
enumSchema,
34+
looseObjectSchema,
35+
nullableStringSchema,
36+
numberSchema,
37+
objectSchema,
38+
stringArraySchema,
39+
stringSchema,
2940
toClientElementTarget,
3041
toClientInteractionTarget,
3142
toRepeatedOptions,
@@ -54,6 +65,279 @@ import {
5465
import { interactionCliOutputFormatters } from './output.ts';
5566
import { selectorCliReaders, selectorDaemonWriters } from './selectors.ts';
5667

68+
// PostActionSurfaceChange (packages/contracts/src/interaction.ts) — the post-action capture
69+
// describes a different surface than the pre-action baseline (#2438), so no same-surface
70+
// comparison is presented across it.
71+
export const postActionSurfaceChangeSchema: JsonSchema = objectSchema(
72+
{
73+
from: stringSchema('Surface the pre-action baseline described: a host bundle id, or app.'),
74+
to: stringSchema('Surface the post-action capture describes: a host bundle id, or app.'),
75+
disclosure: stringSchema('Agent-facing sentence explaining the surface transition.'),
76+
},
77+
['from', 'to', 'disclosure'],
78+
'Present when an in-place system surface (web sign-in or Apple Pay sheet) was presented over the app, or left it.',
79+
);
80+
81+
// InteractionEvidence (packages/contracts/src/interaction.ts) — opt-in `--verify` cheap
82+
// post-condition evidence (#1047).
83+
const interactionEvidenceSchema: JsonSchema = objectSchema(
84+
{
85+
foregroundApp: stringSchema('Foreground app bundle id or name, when the capture carries it.'),
86+
nodeCount: numberSchema('Node count in the post-action interactive-only capture.'),
87+
interactiveNodeCount: numberSchema('Subset of nodeCount the platform reports as hittable.'),
88+
digest: stringSchema('Order-independent digest of the post-action node multiset.'),
89+
changedFromBefore: booleanSchema(
90+
'Whether the post-action digest differs from the pre-action capture digest. false is evidence, not failure. With surfaceChange present, no digest comparison is made: it reports that surface transition.',
91+
),
92+
surfaceChange: postActionSurfaceChangeSchema,
93+
},
94+
['nodeCount', 'interactiveNodeCount', 'digest', 'changedFromBefore'],
95+
);
96+
97+
const responseCostSchema: JsonSchema = objectSchema(
98+
{
99+
wallClockMs: numberSchema('Total wall-clock time for the request in milliseconds.'),
100+
runnerRoundTrips: numberSchema(
101+
'Number of real runner round-trips made while serving the request.',
102+
),
103+
nodeCount: numberSchema(
104+
'Number of nodes in the original node tree when the response carries one.',
105+
),
106+
},
107+
['wallClockMs', 'runnerRoundTrips'],
108+
);
109+
110+
// ResolutionDiagnosticEntry (packages/contracts/src/interaction.ts) — a disambiguation
111+
// winner or losing alternative. Never a snapshot ref.
112+
const resolutionDiagnosticEntrySchema: JsonSchema = objectSchema(
113+
{
114+
diagnosticRef: stringSchema(
115+
'Opaque non-@ diagnostic token. Never a snapshot ref: not issued via refsGeneration and cannot be pinned or reused as an @ref target. UTF-8 truncated to 256 bytes.',
116+
),
117+
role: stringSchema('UTF-8 truncated to 256 bytes.'),
118+
label: stringSchema('UTF-8 truncated to 256 bytes.'),
119+
},
120+
['diagnosticRef'],
121+
);
122+
123+
// ResolutionDisclosure (packages/contracts/src/interaction.ts) — never ref-issuing;
124+
// absent on paths where the guarantee is inapplicable (ADR 0012 decision 2).
125+
// `alternatives` rides default/full levels only; the digest view omits it.
126+
const resolutionDisclosureSchema: JsonSchema = {
127+
type: 'object',
128+
description:
129+
'Pre-action disclosure of how the acting path resolved its target. Absent when resolutionDisclosure is inapplicable for the path.',
130+
oneOf: [
131+
objectSchema(
132+
{
133+
source: constSchema('runtime'),
134+
phase: constSchema('pre-action'),
135+
kind: constSchema('unique'),
136+
},
137+
['source', 'phase', 'kind'],
138+
),
139+
objectSchema(
140+
{
141+
source: constSchema('runtime'),
142+
phase: constSchema('pre-action'),
143+
kind: constSchema('disambiguated'),
144+
matchCount: numberSchema('Total matches resolveSelectorChain found before disambiguation.'),
145+
winnerDiagnostic: resolutionDiagnosticEntrySchema,
146+
tiebreak: enumSchema(
147+
['visible', 'deepest', 'smallest-area', 'structural-equivalence'],
148+
'The comparison that decided the winner.',
149+
),
150+
alternatives: {
151+
type: 'array',
152+
description:
153+
'At most 5 losing candidates, document order. Present at default/full response levels and omitted in digest. The winner is never included.',
154+
items: resolutionDiagnosticEntrySchema,
155+
},
156+
},
157+
['source', 'phase', 'kind', 'matchCount', 'winnerDiagnostic', 'tiebreak'],
158+
),
159+
objectSchema(
160+
{ source: constSchema('ref'), phase: constSchema('pre-action'), kind: constSchema('exact') },
161+
['source', 'phase', 'kind'],
162+
),
163+
objectSchema(
164+
{
165+
source: constSchema('ref'),
166+
phase: constSchema('pre-action'),
167+
kind: constSchema('label-fallback'),
168+
},
169+
['source', 'phase', 'kind'],
170+
),
171+
objectSchema({ source: constSchema('direct-ios'), kind: constSchema('not-observed') }, [
172+
'source',
173+
'kind',
174+
]),
175+
],
176+
};
177+
178+
type InteractionExtra = {
179+
properties?: Record<string, JsonSchema>;
180+
required?: readonly string[];
181+
};
182+
183+
/**
184+
* Canonical interaction response data built by buildInteractionResponseData:
185+
* shared target/coordinate/evidence fields plus per-command extras. The runtime
186+
* result still has richer internal node/backend data; this schema documents the
187+
* JSON payload returned to clients.
188+
*/
189+
function interactionResponseDataSchema(extra: InteractionExtra = {}): JsonSchema {
190+
const extraProperties = extra.properties ?? {};
191+
const extraRequired = extra.required ?? [];
192+
return objectSchema(
193+
{
194+
targetKind: enumSchema(['point', 'ref', 'selector'], 'Resolved interaction target kind.'),
195+
x: numberSchema('Resolved interaction x coordinate when available.'),
196+
y: numberSchema('Resolved interaction y coordinate when available.'),
197+
referenceWidth: numberSchema('Reference frame width for visualizing the interaction point.'),
198+
referenceHeight: numberSchema(
199+
'Reference frame height for visualizing the interaction point.',
200+
),
201+
ref: stringSchema('Snapshot ref without the @ prefix when the target was an @ref.'),
202+
selector: stringSchema('Selector expression when the target was a selector.'),
203+
selectorChain: stringArraySchema(),
204+
refLabel: stringSchema(),
205+
targetHittable: booleanSchema(),
206+
hint: stringSchema(),
207+
warning: stringSchema(),
208+
message: stringSchema(),
209+
evidence: interactionEvidenceSchema,
210+
resolution: resolutionDisclosureSchema,
211+
cost: responseCostSchema,
212+
maestroNonHittableCoordinateFallbackAllowed: booleanSchema(
213+
'Whether the direct iOS Maestro coordinate fallback was allowed for this selector.',
214+
),
215+
maestroNonHittableCoordinateFallbackUsed: booleanSchema(
216+
'Whether the direct iOS Maestro coordinate fallback was actually used.',
217+
),
218+
maestroFallbackReason: constSchema('non-hittable-coordinate'),
219+
...extraProperties,
220+
},
221+
['targetKind', ...extraRequired],
222+
);
223+
}
224+
225+
const tapInteractionResponseDataSchema = interactionResponseDataSchema({
226+
properties: {
227+
evidence: interactionEvidenceSchema,
228+
button: enumSchema(['secondary', 'middle']),
229+
count: numberSchema('Number of press/click repetitions.'),
230+
intervalMs: numberSchema('Delay between repeated press/click actions.'),
231+
holdMs: numberSchema('Hold duration for each action.'),
232+
jitterPx: numberSchema('Randomization radius in pixels.'),
233+
doubleTap: booleanSchema('Whether the command requested a double-tap action.'),
234+
},
235+
});
236+
237+
const fillResponseProperties = {
238+
text: stringSchema('Text submitted to the field.'),
239+
delayMs: numberSchema('Delay between typed characters in milliseconds.'),
240+
evidence: interactionEvidenceSchema,
241+
};
242+
243+
const fillVerificationTargetSchema = objectSchema(
244+
{
245+
resourceId: nullableStringSchema('Android resource id of the exact field that changed.'),
246+
className: nullableStringSchema('Android class name of the exact field that changed.'),
247+
packageName: nullableStringSchema('Android package name that owns the exact field.'),
248+
rect: objectSchema(
249+
{
250+
x: numberSchema(),
251+
y: numberSchema(),
252+
width: numberSchema(),
253+
height: numberSchema(),
254+
},
255+
['x', 'y', 'width', 'height'],
256+
'Screen-space rectangle of the exact field that changed.',
257+
),
258+
},
259+
['resourceId', 'className', 'packageName', 'rect'],
260+
'Target identity captured before fill and matched after fill.',
261+
);
262+
263+
const confirmedFillResponseSchema: JsonSchema = {
264+
...interactionResponseDataSchema({
265+
properties: fillResponseProperties,
266+
required: ['text'],
267+
}),
268+
// The public result contract omits verification evidence on an ordinary
269+
// confirmed fill. Keep this branch disjoint from the unconfirmed branch
270+
// without making the response strict to unrelated additive fields.
271+
not: objectSchema({}, ['verification']),
272+
};
273+
274+
const unconfirmedFillResponseSchema = interactionResponseDataSchema({
275+
properties: {
276+
...fillResponseProperties,
277+
verification: constSchema('unconfirmed'),
278+
requested: stringSchema('Literal text requested by the fill command.'),
279+
before: nullableStringSchema('Raw target text captured before the fill.'),
280+
after: nullableStringSchema('Raw target text captured after the fill.'),
281+
target: fillVerificationTargetSchema,
282+
},
283+
required: ['text', 'verification', 'requested', 'before', 'after', 'target'],
284+
});
285+
286+
/**
287+
* This family's advertised MCP `outputSchema`s, keyed by daemon command name and projected into
288+
* the command map by `src/mcp/command-output-schemas.ts`. Non-strict like every other entry: no
289+
* `additionalProperties: false`, so additive response fields such as `settle`/`cost` keep
290+
* validating. #1652: the opt-in `settle` observation is NOT listed here — the trait derivation
291+
* pass in that file grafts it onto settle-capable entries.
292+
*/
293+
export const INTERACTION_COMMAND_OUTPUT_SCHEMAS = {
294+
press: tapInteractionResponseDataSchema,
295+
click: tapInteractionResponseDataSchema,
296+
fill: {
297+
type: 'object',
298+
description:
299+
'Fill response. Android may return target-bound unconfirmed evidence when the exact app-owned field changed but formatting prevented raw equality.',
300+
oneOf: [confirmedFillResponseSchema, unconfirmedFillResponseSchema],
301+
},
302+
longpress: interactionResponseDataSchema({
303+
properties: {
304+
durationMs: numberSchema(),
305+
gesture: constSchema('longpress'),
306+
},
307+
}),
308+
hover: interactionResponseDataSchema({
309+
properties: {
310+
gesture: constSchema('hover'),
311+
},
312+
}),
313+
find: objectSchema(
314+
{
315+
ref: stringSchema('Snapshot ref without the @ prefix when the find action returns one.'),
316+
refsGeneration: numberSchema('ADR 0014 ref frame epoch for read-only find actions.'),
317+
found: booleanSchema('Whether a wait/exists/read-only find satisfied its condition.'),
318+
waitedMs: numberSchema('Milliseconds waited for a read-only find condition.'),
319+
text: stringSchema('Text value returned by find get_text.'),
320+
node: looseObjectSchema('Snapshot node for find get_attrs/get_text.'),
321+
matches: {
322+
type: 'array',
323+
description: 'Every match for the read-only find list action (#1625): { ref, node } each.',
324+
items: looseObjectSchema('One listed match with its snapshot ref and node.'),
325+
},
326+
locator: stringSchema('Locator kind used for the find action.'),
327+
query: stringSchema('Query argument used for the find action.'),
328+
x: numberSchema('Resolved x coordinate for mutating find actions.'),
329+
y: numberSchema('Resolved y coordinate for mutating find actions.'),
330+
message: stringSchema('Diagnostic message for mutating find actions.'),
331+
cost: responseCostSchema,
332+
},
333+
[],
334+
'Daemon response data for the find command.',
335+
),
336+
} satisfies Pick<
337+
Record<keyof CommandResultMap, JsonSchema>,
338+
'press' | 'click' | 'fill' | 'longpress' | 'hover' | 'find'
339+
>;
340+
57341
const interactionCliSchemas = {
58342
get: {
59343
usageOverride: 'get text|attrs <@ref|selector>',
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { ownerFilesForCommand } from '@agent-device/command-registry/owner-files';
4+
import { INTERACTION_COMMAND_OUTPUT_SCHEMAS } from '../../commands/interaction/index.ts';
5+
import { COMMAND_OUTPUT_SCHEMAS } from '../command-output-schemas.ts';
6+
7+
const INTERACTION_COMMANDS = Object.keys(INTERACTION_COMMAND_OUTPUT_SCHEMAS) as Array<
8+
keyof typeof INTERACTION_COMMAND_OUTPUT_SCHEMAS
9+
>;
10+
11+
// press, click, fill, longpress, and hover carry the post-action observation trait (#1652): the
12+
// composed map grafts a `settle` property onto a COPY, so they are not reference-equal to the
13+
// module's own object. find carries no such trait.
14+
const SETTLE_DERIVED_COMMANDS = new Set(['press', 'click', 'fill', 'longpress', 'hover']);
15+
16+
test('MCP interaction family output schemas are the family module entries, not copies', () => {
17+
for (const command of INTERACTION_COMMANDS) {
18+
if (SETTLE_DERIVED_COMMANDS.has(command)) continue;
19+
assert.equal(
20+
COMMAND_OUTPUT_SCHEMAS[command],
21+
INTERACTION_COMMAND_OUTPUT_SCHEMAS[command],
22+
`${command} is not reference-equal to the interaction module's own schema object`,
23+
);
24+
}
25+
});
26+
27+
test('a settle-derived interaction entry still grafts onto the module object, not a foreign copy', () => {
28+
const modulePress = INTERACTION_COMMAND_OUTPUT_SCHEMAS.press;
29+
const derivedPress = COMMAND_OUTPUT_SCHEMAS.press;
30+
assert.notEqual(
31+
derivedPress,
32+
modulePress,
33+
'the derivation pass must copy, never mutate in place',
34+
);
35+
assert.equal(derivedPress.required, modulePress.required, 'required list is not re-derived');
36+
assert.equal(
37+
derivedPress.properties?.targetKind,
38+
modulePress.properties?.targetKind,
39+
'a field object nested under the derived copy must still be the module property, not a rebuild',
40+
);
41+
assert.ok(
42+
derivedPress.properties?.settle,
43+
'the settle-capable command must carry the grafted observation property',
44+
);
45+
});
46+
47+
test('press and click share the same tap response schema object', () => {
48+
assert.equal(INTERACTION_COMMAND_OUTPUT_SCHEMAS.press, INTERACTION_COMMAND_OUTPUT_SCHEMAS.click);
49+
});
50+
51+
test('the projected family map declares exactly the commands its module owns', () => {
52+
for (const command of INTERACTION_COMMANDS) {
53+
assert.ok(
54+
ownerFilesForCommand(command).includes('src/commands/interaction/index.ts'),
55+
`${command} projects its output schema from this module but does not name it as its owner`,
56+
);
57+
}
58+
});

‎src/mcp/__tests__/command-tools-replay-schemas.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { test } from 'vitest';
33
import type { ReplayCommandResult, ReplaySuiteResult } from '@agent-device/contracts/replay';
44
import { ownerFilesForCommand } from '@agent-device/command-registry/owner-files';
55
import { commandSupportsSettleObservation } from '@agent-device/command-registry/registry';
6+
import { INTERACTION_COMMAND_OUTPUT_SCHEMAS } from '../../commands/interaction/index.ts';
67
import { REPLAY_COMMAND_OUTPUT_SCHEMAS } from '../../commands/replay/index.ts';
78
import { SYSTEM_COMMAND_OUTPUT_SCHEMAS } from '../../commands/system/index.ts';
89
import { COMMAND_OUTPUT_SCHEMAS } from '../command-output-schemas.ts';
@@ -71,6 +72,7 @@ test('the projected family map declares exactly the commands its module owns', (
7172
const PROJECTED_FAMILIES = [
7273
{ name: 'replay', schemas: REPLAY_COMMAND_OUTPUT_SCHEMAS },
7374
{ name: 'system', schemas: SYSTEM_COMMAND_OUTPUT_SCHEMAS },
75+
{ name: 'interaction', schemas: INTERACTION_COMMAND_OUTPUT_SCHEMAS },
7476
] as const;
7577

7678
test('projected output-schema families claim disjoint commands and survive the composition intact', () => {

0 commit comments

Comments
 (0)