|
| 1 | +import assert from 'node:assert/strict'; |
| 2 | +import { test } from 'vitest'; |
| 3 | +import { usage, usageForCommand } from '../../cli/parser/args.ts'; |
| 4 | +import { helpTopicIds } from './cli-help.ts'; |
| 5 | + |
| 6 | +// Agents often read only the first screen of a help text, and the help benchmark caps the root |
| 7 | +// help at 30 lines (`--help:first30`). Each entry pins the rules that keep an agent safe on that |
| 8 | +// screen: prerequisites, refusals, verification steps, and recovery. A new topic must declare its |
| 9 | +// own entry. |
| 10 | +const FIRST_SCREEN_LINES = 30; |
| 11 | +const ROOT_HELP = '--help'; |
| 12 | + |
| 13 | +const FIRST_SCREEN_RULES: Record<string, readonly string[]> = { |
| 14 | + [ROOT_HELP]: [ |
| 15 | + 'agent-device open <app> --foreground', |
| 16 | + 'Do not probe first with devices, apps, appstate, snapshot, or screenshot', |
| 17 | + 'Never invent ids', |
| 18 | + 'A bare screenshot is not verification', |
| 19 | + 'End with: agent-device close', |
| 20 | + ], |
| 21 | + commands: ['Use agent-device help <command> for exact flags and behavior', 'Commands:'], |
| 22 | + 'manual-qa': [ |
| 23 | + 'Do not explore unrelated screens', |
| 24 | + 'stop and report the ambiguity', |
| 25 | + 'Use --relaunch only when the script needs fresh state', |
| 26 | + 'If --settle prints not settled, follow its hint', |
| 27 | + 'A bare screenshot/snapshot is not verification', |
| 28 | + ], |
| 29 | + maestro: [ |
| 30 | + 'replay <flow.yaml> --maestro', |
| 31 | + 'there is no silent all: allow default', |
| 32 | + 'do not place secrets in diagnostic identifiers', |
| 33 | + 'not a security sandbox', |
| 34 | + 'Unsupported syntax fails with an explicit error', |
| 35 | + ], |
| 36 | + workflow: [ |
| 37 | + 'never a placeholder (@ref, @eN, @Label_Name)', |
| 38 | + 'every platform rejects a stale pinned ref', |
| 39 | + 'Never open artifact paths or invent package ids', |
| 40 | + 'help tv (Android TV, tvOS, Vega VVD remote)', |
| 41 | + 'Matches in distinct subtrees fail with AMBIGUOUS_MATCH', |
| 42 | + ], |
| 43 | + scripting: [ |
| 44 | + 'scroll down --until <selector>', |
| 45 | + 'add an explicit landmark wait', |
| 46 | + '--record-as PASSWORD', |
| 47 | + 'contain only ${PASSWORD}', |
| 48 | + 'refuses a terminal, empty input, invalid UTF-8', |
| 49 | + ], |
| 50 | + gestures: [ |
| 51 | + 'Use coordinates only as a fallback', |
| 52 | + 'Do not assume that requested values map exactly to recognizer output', |
| 53 | + 'then run diff snapshot -i', |
| 54 | + 'Coordinates are fallback-only. Document why you used them.', |
| 55 | + ], |
| 56 | + tv: [ |
| 57 | + 'Move focus with remote/D-pad buttons, then activate the focused control', |
| 58 | + 'not ADB', |
| 59 | + 'Physical Fire TV devices remain unsupported', |
| 60 | + 'The VVD never boots implicitly', |
| 61 | + 'remain unsupported until their Vega backends are implemented', |
| 62 | + ], |
| 63 | + debugging: [ |
| 64 | + 'Do not cat a full stale log into the agent context', |
| 65 | + 'Do not split it into logs stop, logs clear, logs start', |
| 66 | + '--launch-console is only for direct iOS simulator app launches, not URL opens', |
| 67 | + 'Do not write network log headers', |
| 68 | + ], |
| 69 | + 'react-devtools': [ |
| 70 | + 'React Native 0.74 or newer', |
| 71 | + 'After either event, run find again', |
| 72 | + 'fail with tree-unavailable', |
| 73 | + 'Run profile start immediately before the interaction', |
| 74 | + ], |
| 75 | + cdp: [ |
| 76 | + 'Do not use it as the default React Native', |
| 77 | + 'adb reverse tcp:8081 tcp:8081', |
| 78 | + 'agent-device cdp target select <target-id>', |
| 79 | + 'cdp memory snapshot leak-triplet', |
| 80 | + ], |
| 81 | + 'react-native': [ |
| 82 | + 'Do not run doctor as routine QA/dogfood prep', |
| 83 | + 'prefer open --relaunch before the first snapshot', |
| 84 | + 'Do not use agent-device reload', |
| 85 | + 'the sandbox probe is not authoritative', |
| 86 | + 'Do not invent app ids or URLs; ask instead', |
| 87 | + ], |
| 88 | + 'physical-device': [ |
| 89 | + 'have Developer Mode enabled', |
| 90 | + 'must be signed before commands can run on a physical device', |
| 91 | + 'devtools_security_developer_mode_disabled is the DevToolsSecurity setting of the Mac, not the Developer Mode of the device', |
| 92 | + 'device_developer_mode_disabled is read from the device before the runner builds', |
| 93 | + 'The two device reasons are never inferred from tool output or from each other', |
| 94 | + 'Enable USB debugging', |
| 95 | + ], |
| 96 | + 'ios-system-ui': [ |
| 97 | + 'agent-device open com.apple.springboard --platform ios', |
| 98 | + 'Do not rely on the literal strings', |
| 99 | + 'This is the one deliberate coordinate step', |
| 100 | + 'Do not hard-code Edit/Done/Add Widget', |
| 101 | + 'Reopen the app bundle under test', |
| 102 | + ], |
| 103 | + foldable: [ |
| 104 | + 'Do not add a screen flag to the normal loop', |
| 105 | + 'Never carry coordinates or refs across a pose change', |
| 106 | + 'After every fold, take a new snapshot', |
| 107 | + 'fold-pose-unverified', |
| 108 | + 'Report which poses the run covered', |
| 109 | + ], |
| 110 | + remote: [ |
| 111 | + 'connect -> install/open -> commands -> close -> disconnect', |
| 112 | + 'not the app artifact name, ARN, or lt:// id', |
| 113 | + 'run its printed reconnect command before open', |
| 114 | + 'Do not run devices as a pre-open catalog probe for direct providers', |
| 115 | + 'It does not expose connect/disconnect', |
| 116 | + ], |
| 117 | + macos: [ |
| 118 | + 'Do not write inspect as a command', |
| 119 | + '--button secondary', |
| 120 | + 'Do not let iOS simulator-set scoping hide macOS desktop targets', |
| 121 | + 'Do not guess coordinates', |
| 122 | + 'does not raise the app', |
| 123 | + ], |
| 124 | + web: [ |
| 125 | + 'Use --platform web only for the minimal browser command loop', |
| 126 | + 'agent-device web doctor', |
| 127 | + 'Web automation requires Node 24+', |
| 128 | + 'write only agent-device command lines', |
| 129 | + ], |
| 130 | + dogfood: [ |
| 131 | + 'Do not read app source or invent findings from code', |
| 132 | + 'Ask only if one of them is missing', |
| 133 | + 'Keep stateful commands serial within the same session', |
| 134 | + 'use react-native dismiss-overlay', |
| 135 | + ], |
| 136 | + validate: [ |
| 137 | + 'Do not validate against stale dist output', |
| 138 | + 'start with pnpm build', |
| 139 | + 'run pnpm build:android before pnpm clean:daemon', |
| 140 | + 'Do not build the Apple runner for TypeScript-only changes', |
| 141 | + 'Close sessions and release leases before you finish', |
| 142 | + ], |
| 143 | +}; |
| 144 | + |
| 145 | +async function renderHelp(id: string): Promise<string> { |
| 146 | + if (id === ROOT_HELP) return await usage(); |
| 147 | + const text = await usageForCommand(id); |
| 148 | + if (text === null) throw new Error(`Expected help text for topic ${id}`); |
| 149 | + return text; |
| 150 | +} |
| 151 | + |
| 152 | +test('every help topic and the root help declare first-screen rules', () => { |
| 153 | + assert.deepEqual(Object.keys(FIRST_SCREEN_RULES).sort(), [ROOT_HELP, ...helpTopicIds()].sort()); |
| 154 | +}); |
| 155 | + |
| 156 | +test.each(Object.entries(FIRST_SCREEN_RULES))( |
| 157 | + 'help %s keeps its key rules in the first 30 lines', |
| 158 | + async (id, rules) => { |
| 159 | + const lines = (await renderHelp(id)).split('\n'); |
| 160 | + const firstScreen = lines.slice(0, FIRST_SCREEN_LINES).join('\n'); |
| 161 | + for (const rule of rules) { |
| 162 | + const line = lines.findIndex((text) => text.includes(rule)) + 1; |
| 163 | + assert.ok( |
| 164 | + firstScreen.includes(rule), |
| 165 | + line > 0 |
| 166 | + ? `"${rule}" is on line ${line}, below the first ${FIRST_SCREEN_LINES} lines of help ${id}` |
| 167 | + : `"${rule}" is missing from help ${id}`, |
| 168 | + ); |
| 169 | + } |
| 170 | + }, |
| 171 | +); |
0 commit comments