|
| 1 | +import assert from 'node:assert/strict'; |
| 2 | +import { test } from 'vitest'; |
| 3 | +import { usage, usageForCommand } from '../../cli/parser/args.ts'; |
| 4 | +import { helpTopicIds } from './cli-help.ts'; |
| 5 | + |
| 6 | +// Agents often read only the first screen of a help text, and the help benchmark caps the root |
| 7 | +// help at 30 lines (`--help:first30`). Each entry pins the rules that keep an agent safe on that |
| 8 | +// screen: prerequisites, refusals, verification steps, and recovery. A new topic must declare its |
| 9 | +// own entry. |
| 10 | +const FIRST_SCREEN_LINES = 30; |
| 11 | +const ROOT_HELP = '--help'; |
| 12 | + |
| 13 | +const FIRST_SCREEN_RULES: Record<string, readonly string[]> = { |
| 14 | + [ROOT_HELP]: [ |
| 15 | + 'agent-device open <app> --foreground', |
| 16 | + 'Do not probe first with devices, apps, appstate, snapshot, or screenshot', |
| 17 | + 'Unknown app id: devices, then apps, then open <discovered-id>. Never invent ids', |
| 18 | + 'is, get, or find. A bare screenshot is not verification', |
| 19 | + 'End with: agent-device close', |
| 20 | + ], |
| 21 | + commands: [ |
| 22 | + 'Full command catalog. Use agent-device help <command> for exact flags and behavior', |
| 23 | + 'Commands:', |
| 24 | + ], |
| 25 | + 'manual-qa': [ |
| 26 | + 'Execute the script. Do not explore unrelated screens', |
| 27 | + 'Stop and report ambiguity', |
| 28 | + '1. Open the requested app; use --relaunch only when the script needs fresh state', |
| 29 | + '5. If --settle prints not settled, follow its hint', |
| 30 | + '6. Verify named expectations with wait text/selector/absent, get, is, find, or the settled diff. A bare screenshot/snapshot is not verification', |
| 31 | + ], |
| 32 | + maestro: [ |
| 33 | + 'Run Maestro compatibility flows with replay <flow.yaml> --maestro', |
| 34 | + '- Flows: launchApp (with clearState, permissions, and Apple-only launch arguments; permissions apply after state clearing but before launch, and a launchApp without permissions touches nothing — there is no silent all: allow default', |
| 35 | + '- Failure diagnostics: resolved targets and runFlow paths are rendered, while inputText payloads remain hidden; do not place secrets in diagnostic identifiers', |
| 36 | + '- Trust: runScript, evalScript, and JavaScript condition true: fields execute flow scripts in-process via node:vm, which is not a security sandbox', |
| 37 | + 'Unsupported syntax fails loudly', |
| 38 | + ], |
| 39 | + workflow: [ |
| 40 | + 'Refs look like @e12; use the exact ref from the latest snapshot -i, never a placeholder (@ref, @eN, @Label_Name)', |
| 41 | + 'Refs look like @e12; use the exact ref from the latest snapshot -i, never a placeholder (@ref, @eN, @Label_Name). Pin with ~s<n> (press @e12~s4); every platform rejects a stale pinned ref', |
| 42 | + 'Unknown app id: devices, then apps, then open <discovered-app-id>. Never open artifact paths or invent package ids', |
| 43 | + 'Truncated preview: snapshot -s @e12 (the current concrete ref), not get text. Missing target: scroll <dir> --until <selector>. TV/D-pad focus: help tv', |
| 44 | + 'Mutating selector ambiguity: press/click/fill/longpress collapse duplicate accessibility wrappers only when every match is one ancestor-descendant chain resolving to the same actionable node. Matches in distinct subtrees fail with AMBIGUOUS_MATCH', |
| 45 | + ], |
| 46 | + scripting: [ |
| 47 | + 'Reaching an off-screen target is viewport-independent in a script the same way it is at the CLI: write scroll down --until <selector>', |
| 48 | + 'press/click/longpress resolve their target once. Before interacting with a loading screen, add an explicit landmark wait', |
| 49 | + 'agent-device fill \'id="password"\' "$AD_VAR_PASSWORD" --record-as PASSWORD', |
| 50 | + 'The live app receives the value; recording state and the published script contain only ${PASSWORD}', |
| 51 | + '--text-stdin reads at most 64 KiB, removes exactly one trailing newline, and refuses a terminal, empty input, invalid UTF-8', |
| 52 | + ], |
| 53 | + gestures: [ |
| 54 | + 'longpress accepts coordinates, @refs, or selectors; prefer @ref/selector, coordinates only as a fallback', |
| 55 | + 'iOS simulator transform/pinch/rotate use private XCTest synthesis for a continuous two-finger pan/scale/rotation path; verify app metrics instead of assuming requested values map exactly to recognizer output', |
| 56 | + 'Rare iOS accessibility gap: a row shown disabled/hittable:false where press reports success but no UI change, or a collapsed composite control with no child refs -- run snapshot -i --json, compute the target center from rects, press x y, then diff snapshot -i. Coordinates are fallback-only; document why you used them.', |
| 57 | + ], |
| 58 | + tv: [ |
| 59 | + 'Use this when the target is Android TV, Apple TV/tvOS, or an Amazon Vega OS TV app running in the Vega Virtual Device (VVD). TV surfaces are focus-first: move focus with remote/D-pad buttons, then activate the focused control', |
| 60 | + 'Vega OS is driven through the SDK-matched Vega CLI and VDA, not ADB', |
| 61 | + 'Initial support is VVD-only. Physical Fire TV devices remain unsupported', |
| 62 | + 'The VVD is never booted implicitly', |
| 63 | + 'Snapshot, screenshot, selectors, install, touch/text/gesture, logs, and performance commands remain unsupported until their Vega backends are implemented', |
| 64 | + ], |
| 65 | + debugging: [ |
| 66 | + 'Do not cat a full stale log into agent context', |
| 67 | + 'logs clear --restart is the compact command to clear old logs and start a fresh capture; do not split it into logs stop, logs clear, logs start', |
| 68 | + '--launch-console is only for direct iOS simulator app launches, not URL opens', |
| 69 | + 'network log is a supported alias, but network dump --include headers is the clearest plan form. Do not write network log headers', |
| 70 | + ], |
| 71 | + 'react-devtools': [ |
| 72 | + "Every command except profile report and profile commit, which read the saved profile, attaches to the app's JavaScript runtime through its Metro dev server: a development build on React Native 0.74 or newer", |
| 73 | + 'Refs are DevTools element ids: they change when a component remounts or the app reloads, so find again after either', |
| 74 | + 'With a React DevTools window connected to the app, tree, find, errors, and inspect by name fail with tree-unavailable', |
| 75 | + '1. Run profile start immediately before the interaction', |
| 76 | + ], |
| 77 | + cdp: [ |
| 78 | + 'state. Do not use this as the default React Native profiler', |
| 79 | + 'Start Metro and open the app first. For Android devices/emulators, make sure Metro is reachable from the app, typically with adb reverse tcp:8081 tcp:8081', |
| 80 | + 'agent-device cdp target select <target-id>', |
| 81 | + 'agent-device cdp memory snapshot leak-triplet', |
| 82 | + ], |
| 83 | + 'react-native': [ |
| 84 | + 'Do not run doctor as routine QA/dogfood prep', |
| 85 | + 'For "start from screen X" flows, prefer open --relaunch before the first snapshot', |
| 86 | + 'Do not use agent-device reload', |
| 87 | + 'Verify Metro/Re.Pack from the same host context that owns the dev server. If a sandboxed shell cannot curl localhost:8081/status but an unrestricted host shell can, the dev server is running and the sandbox probe is not authoritative', |
| 88 | + 'Expo Go/dev clients are host shells. Use provided project URLs, verify with snapshot -i after opening, and ask instead of inventing app ids or URLs', |
| 89 | + ], |
| 90 | + 'physical-device': [ |
| 91 | + 'The device must be paired/trusted, connected, unlocked when needed, and have Developer Mode enabled', |
| 92 | + 'The AgentDeviceRunner XCTest host must be signed before commands can run on a physical device', |
| 93 | + 'A runner startup failure names its class in error details.reason rather than only in prose:', |
| 94 | + 'Enable USB debugging', |
| 95 | + ], |
| 96 | + 'ios-system-ui': [ |
| 97 | + '1. Reach the app state you want to prepare (for example, arrange the widget/Live Activity data the app should show) with normal app automation, then agent-device open com.apple.springboard --platform ios', |
| 98 | + 'This works today by opening SpringBoard as the session app; there is no separate widget/system command. System labels vary by iOS version and locale, so discover them from the current snapshot instead of relying on the literal strings', |
| 99 | + '3. agent-device longpress <x> <y> on an empty area of the home screen to enter edit mode. This is the one deliberate coordinate step', |
| 100 | + 'Do not hard-code Edit/Done/Add Widget', |
| 101 | + '9. Reopen the app bundle under test', |
| 102 | + ], |
| 103 | + foldable: [ |
| 104 | + "Each iOS simulator capture resolves the CoreDevice display table, captures the lit panel explicitly, and normalizes density with that panel's own point scale. Do not add a screen flag to the normal loop", |
| 105 | + 'The two panels are different sizes (iPhone Duo: 466x678 points closed on the outer panel, 669x951 open on the inner). A pose change therefore invalidates every ref and coordinate. Re-snapshot after any pose change and never carry coordinates or refs across', |
| 106 | + 'Expect a fold to take 10-16 seconds: each hinge read is a five-second devicectl stream, and half-open waits for the hinge to stop moving. Re-snapshot after every fold', |
| 107 | + 'fold sends a private HID hinge event inside the selected simulator', |
| 108 | + 'If a task asserts behavior for more than one pose, fold to each pose and re-snapshot, and report which poses the run covered', |
| 109 | + ], |
| 110 | + remote: [ |
| 111 | + 'connect -> install/open -> commands -> close -> disconnect', |
| 112 | + 'BrowserStack, AWS Device Farm, and TestMu AI create the hosted session on open. open needs the installed package or bundle identifier, not the app artifact name, ARN, or lt:// id', |
| 113 | + 'AWS Device Farm cannot install after allocation. If connect reports no attached app, run its printed reconnect command', |
| 114 | + 'Do not run devices as a pre-open catalog probe for direct providers', |
| 115 | + 'MCP exposes operational tools such as open, snapshot, click, close, and artifacts. It does not expose connect/disconnect', |
| 116 | + ], |
| 117 | + macos: [ |
| 118 | + 'Use open and snapshot -i for menu bar inspection. Do not output inspect as a command', |
| 119 | + 'agent-device click @e66 --button secondary', |
| 120 | + 'Do not let iOS simulator-set scoping hide macOS desktop targets', |
| 121 | + 'macOS snapshot rects are window-space; use current refs or overlay refs instead of guessing coordinates', |
| 122 | + 'On the default XCTest backend a read (snapshot, get, find) of a session app that sits behind other windows is answered from where it sits, without raising the app', |
| 123 | + ], |
| 124 | + web: [ |
| 125 | + 'Use --platform web only for the minimal browser command loop', |
| 126 | + 'agent-device web doctor', |
| 127 | + 'Web automation requires Node 24+', |
| 128 | + 'For web command plans, output only agent-device command lines', |
| 129 | + ], |
| 130 | + dogfood: [ |
| 131 | + 'Find user-visible issues from runtime behavior. Do not read app source or invent findings from code', |
| 132 | + '1. Identify target app/platform; ask only if missing', |
| 133 | + 'Keep stateful commands serial within the same session', |
| 134 | + 'React Native warning/error overlays can be real findings or test blockers. Capture them, use react-native dismiss-overlay', |
| 135 | + ], |
| 136 | + validate: [ |
| 137 | + 'Prove the changed behavior through public agent-device surfaces. Do not validate against stale dist output', |
| 138 | + 'For a TypeScript runtime or CLI output change, start with pnpm build', |
| 139 | + 'Before local Android verification, run pnpm build:android before pnpm clean:daemon', |
| 140 | + 'For an Apple runner change, run pnpm build:xcuitest and avoid inherited retained runners from older source. Do not build the Apple runner for TypeScript-only changes', |
| 141 | + '6. Close sessions and release leases before finishing', |
| 142 | + ], |
| 143 | +}; |
| 144 | + |
| 145 | +// A rule counts only when a whole rendered line carries it: the trimmed line equals the |
| 146 | +// rule, or starts with it and the rule ends at a word boundary. A substring match would |
| 147 | +// let a short generic rule like 'Commands:' pass from inside unrelated prose. |
| 148 | +function ruleMatchesLine(line: string, rule: string): boolean { |
| 149 | + const trimmed = line.trim(); |
| 150 | + if (trimmed === rule) return true; |
| 151 | + if (!trimmed.startsWith(rule)) return false; |
| 152 | + return !/[\w-]/.test(trimmed[rule.length] ?? ''); |
| 153 | +} |
| 154 | + |
| 155 | +function assertRulesInFirstScreen(lines: readonly string[], rules: readonly string[]): void { |
| 156 | + for (const rule of rules) { |
| 157 | + const line = lines.findIndex((text) => ruleMatchesLine(text, rule)) + 1; |
| 158 | + assert.ok( |
| 159 | + line > 0 && line <= FIRST_SCREEN_LINES, |
| 160 | + line > 0 |
| 161 | + ? `"${rule}" is on line ${line}, below the first ${FIRST_SCREEN_LINES} lines` |
| 162 | + : `"${rule}" has no matching line`, |
| 163 | + ); |
| 164 | + } |
| 165 | +} |
| 166 | + |
| 167 | +async function renderHelp(id: string): Promise<string> { |
| 168 | + if (id === ROOT_HELP) return await usage(); |
| 169 | + const text = await usageForCommand(id); |
| 170 | + if (text === null) throw new Error(`Expected help text for topic ${id}`); |
| 171 | + return text; |
| 172 | +} |
| 173 | + |
| 174 | +test('every help topic and the root help declare first-screen rules', () => { |
| 175 | + assert.deepEqual(Object.keys(FIRST_SCREEN_RULES).sort(), [ROOT_HELP, ...helpTopicIds()].sort()); |
| 176 | +}); |
| 177 | + |
| 178 | +test.each(Object.entries(FIRST_SCREEN_RULES))( |
| 179 | + 'help %s keeps its key rules in the first 30 lines', |
| 180 | + async (id, rules) => { |
| 181 | + assertRulesInFirstScreen((await renderHelp(id)).split('\n'), rules); |
| 182 | + }, |
| 183 | +); |
| 184 | + |
| 185 | +test('a rule matches only whole lines, not prose, a moved rule, or a deleted rule', () => { |
| 186 | + // Short generic rules must not pass from inside unrelated prose. |
| 187 | + const prose = [ |
| 188 | + 'Some unrelated sentence that merely quotes Commands: in passing, far from any list.', |
| 189 | + ]; |
| 190 | + assert.equal(ruleMatchesLine(prose[0]!, 'Commands:'), false); |
| 191 | + assert.throws(() => assertRulesInFirstScreen(prose, ['Commands:']), /no matching line/); |
| 192 | + |
| 193 | + // A rule pushed below the window fails. |
| 194 | + const filler = Array.from({ length: 40 }, (_, i) => `filler line ${i + 1}`); |
| 195 | + filler[34] = 'Commands:'; |
| 196 | + assert.throws( |
| 197 | + () => assertRulesInFirstScreen(filler, ['Commands:']), |
| 198 | + /"Commands:" is on line 35, below the first 30 lines/, |
| 199 | + ); |
| 200 | + |
| 201 | + // A deleted rule fails. |
| 202 | + filler[34] = 'filler line 35'; |
| 203 | + assert.throws(() => assertRulesInFirstScreen(filler, ['Commands:']), /no matching line/); |
| 204 | + |
| 205 | + // Prefixes count only at a word boundary, not mid-word. |
| 206 | + assert.equal(ruleMatchesLine(' Commands: are listed below', 'Commands:'), true); |
| 207 | + assert.equal(ruleMatchesLine('SubCommands: here', 'Commands:'), false); |
| 208 | +}); |
0 commit comments