Skip to content

Commit 9c7d260

Browse files
committed
test(help): pin each topic's key rules to its first 30 lines
Agents often read only the first screen of a help text, and the help benchmark caps the root help at 30 lines. Several rewrites moved safety rules below that screen. Declare, for the root help and for every topic id, the key rules from the base rendering that must stay in lines 1-30: prerequisites, refusals, verification steps, and recovery. A topic id without an entry fails the test.
1 parent b39a9ed commit 9c7d260

1 file changed

Lines changed: 171 additions & 0 deletions

File tree

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import { usage, usageForCommand } from '../../cli/parser/args.ts';
4+
import { helpTopicIds } from './cli-help.ts';
5+
6+
// Agents often read only the first screen of a help text, and the help benchmark caps the root
7+
// help at 30 lines (`--help:first30`). Each entry pins the rules that keep an agent safe on that
8+
// screen: prerequisites, refusals, verification steps, and recovery. A new topic must declare its
9+
// own entry.
10+
const FIRST_SCREEN_LINES = 30;
11+
const ROOT_HELP = '--help';
12+
13+
const FIRST_SCREEN_RULES: Record<string, readonly string[]> = {
14+
[ROOT_HELP]: [
15+
'agent-device open <app> --foreground',
16+
'Do not probe first with devices, apps, appstate, snapshot, or screenshot',
17+
'Never invent ids',
18+
'A bare screenshot is not verification',
19+
'End with: agent-device close',
20+
],
21+
commands: ['Use agent-device help <command> for exact flags and behavior', 'Commands:'],
22+
'manual-qa': [
23+
'Do not explore unrelated screens',
24+
'stop and report the ambiguity',
25+
'Use --relaunch only when the script needs fresh state',
26+
'If --settle prints not settled, follow its hint',
27+
'A bare screenshot/snapshot is not verification',
28+
],
29+
maestro: [
30+
'replay <flow.yaml> --maestro',
31+
'there is no silent all: allow default',
32+
'do not place secrets in diagnostic identifiers',
33+
'not a security sandbox',
34+
'Unsupported syntax fails with an explicit error',
35+
],
36+
workflow: [
37+
'never a placeholder (@ref, @eN, @Label_Name)',
38+
'every platform rejects a stale pinned ref',
39+
'Never open artifact paths or invent package ids',
40+
'help tv (Android TV, tvOS, Vega VVD remote)',
41+
'Matches in distinct subtrees fail with AMBIGUOUS_MATCH',
42+
],
43+
scripting: [
44+
'scroll down --until <selector>',
45+
'add an explicit landmark wait',
46+
'--record-as PASSWORD',
47+
'contain only ${PASSWORD}',
48+
'refuses a terminal, empty input, invalid UTF-8',
49+
],
50+
gestures: [
51+
'Use coordinates only as a fallback',
52+
'Do not assume that requested values map exactly to recognizer output',
53+
'then run diff snapshot -i',
54+
'Coordinates are fallback-only. Document why you used them.',
55+
],
56+
tv: [
57+
'Move focus with remote/D-pad buttons, then activate the focused control',
58+
'not ADB',
59+
'Physical Fire TV devices remain unsupported',
60+
'The VVD never boots implicitly',
61+
'remain unsupported until their Vega backends are implemented',
62+
],
63+
debugging: [
64+
'Do not cat a full stale log into the agent context',
65+
'Do not split it into logs stop, logs clear, logs start',
66+
'--launch-console is only for direct iOS simulator app launches, not URL opens',
67+
'Do not write network log headers',
68+
],
69+
'react-devtools': [
70+
'React Native 0.74 or newer',
71+
'After either event, run find again',
72+
'fail with tree-unavailable',
73+
'Run profile start immediately before the interaction',
74+
],
75+
cdp: [
76+
'Do not use it as the default React Native',
77+
'adb reverse tcp:8081 tcp:8081',
78+
'agent-device cdp target select <target-id>',
79+
'cdp memory snapshot leak-triplet',
80+
],
81+
'react-native': [
82+
'Do not run doctor as routine QA/dogfood prep',
83+
'prefer open --relaunch before the first snapshot',
84+
'Do not use agent-device reload',
85+
'the sandbox probe is not authoritative',
86+
'Do not invent app ids or URLs; ask instead',
87+
],
88+
'physical-device': [
89+
'have Developer Mode enabled',
90+
'must be signed before commands can run on a physical device',
91+
'devtools_security_developer_mode_disabled is the DevToolsSecurity setting of the Mac, not the Developer Mode of the device',
92+
'device_developer_mode_disabled is read from the device before the runner builds',
93+
'The two device reasons are never inferred from tool output or from each other',
94+
'Enable USB debugging',
95+
],
96+
'ios-system-ui': [
97+
'agent-device open com.apple.springboard --platform ios',
98+
'Do not rely on the literal strings',
99+
'This is the one deliberate coordinate step',
100+
'Do not hard-code Edit/Done/Add Widget',
101+
'Reopen the app bundle under test',
102+
],
103+
foldable: [
104+
'Do not add a screen flag to the normal loop',
105+
'Never carry coordinates or refs across a pose change',
106+
'After every fold, take a new snapshot',
107+
'fold-pose-unverified',
108+
'Report which poses the run covered',
109+
],
110+
remote: [
111+
'connect -> install/open -> commands -> close -> disconnect',
112+
'not the app artifact name, ARN, or lt:// id',
113+
'run its printed reconnect command before open',
114+
'Do not run devices as a pre-open catalog probe for direct providers',
115+
'It does not expose connect/disconnect',
116+
],
117+
macos: [
118+
'Do not write inspect as a command',
119+
'--button secondary',
120+
'Do not let iOS simulator-set scoping hide macOS desktop targets',
121+
'Do not guess coordinates',
122+
'does not raise the app',
123+
],
124+
web: [
125+
'Use --platform web only for the minimal browser command loop',
126+
'agent-device web doctor',
127+
'Web automation requires Node 24+',
128+
'write only agent-device command lines',
129+
],
130+
dogfood: [
131+
'Do not read app source or invent findings from code',
132+
'Ask only if one of them is missing',
133+
'Keep stateful commands serial within the same session',
134+
'use react-native dismiss-overlay',
135+
],
136+
validate: [
137+
'Do not validate against stale dist output',
138+
'start with pnpm build',
139+
'run pnpm build:android before pnpm clean:daemon',
140+
'Do not build the Apple runner for TypeScript-only changes',
141+
'Close sessions and release leases before you finish',
142+
],
143+
};
144+
145+
async function renderHelp(id: string): Promise<string> {
146+
if (id === ROOT_HELP) return await usage();
147+
const text = await usageForCommand(id);
148+
if (text === null) throw new Error(`Expected help text for topic ${id}`);
149+
return text;
150+
}
151+
152+
test('every help topic and the root help declare first-screen rules', () => {
153+
assert.deepEqual(Object.keys(FIRST_SCREEN_RULES).sort(), [ROOT_HELP, ...helpTopicIds()].sort());
154+
});
155+
156+
test.each(Object.entries(FIRST_SCREEN_RULES))(
157+
'help %s keeps its key rules in the first 30 lines',
158+
async (id, rules) => {
159+
const lines = (await renderHelp(id)).split('\n');
160+
const firstScreen = lines.slice(0, FIRST_SCREEN_LINES).join('\n');
161+
for (const rule of rules) {
162+
const line = lines.findIndex((text) => text.includes(rule)) + 1;
163+
assert.ok(
164+
firstScreen.includes(rule),
165+
line > 0
166+
? `"${rule}" is on line ${line}, below the first ${FIRST_SCREEN_LINES} lines of help ${id}`
167+
: `"${rule}" is missing from help ${id}`,
168+
);
169+
}
170+
},
171+
);

0 commit comments

Comments
 (0)