Skip to content

Commit d171a88

Browse files
feat(mobile): run the chrome e2e across poses on the device pool
`vp run mobile:e2e` now leases iOS and Android devices from the Simlock pool (SIMLOCK_AGENT_DEVICE_URL + AGENT_DEVICE_DAEMON_AUTH_TOKEN) instead of booting local simulators, and walks each of 12 poses (phone sizes, iPads, a fold, a tablet, dark mode, largest text) through the app: thread list, a thread, scrolling, typing, back, settings, then rotation or folding. Every screen gets generic layout checks that need no per-screen code: app in front, no error screen or launch dialog, no fixed control cut off by the screen edge, no sibling controls stacked, nothing under a system bar, no input under the keyboard. Transitions keep the video check for tabs lagging the screen, now measured against the recording's own first and last frames. Afterwards a headless Claude reviews each pose's screenshots for what the checks cannot see, and the findings land in the same report grid. `mobile:install --session` now has the session's host download the dev client from the shared cache with the read-only key, instead of uploading it through the gateway (13 s instead of 10+ minutes for an APK). Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent fb8fc1f commit d171a88

11 files changed

Lines changed: 1507 additions & 713 deletions

File tree

‎scripts/mobile-e2e.ts‎

Lines changed: 214 additions & 702 deletions
Large diffs are not rendered by default.

‎scripts/mobile-e2e/checks.test.ts‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
3+
import { layoutChecks, type Snapshot, type SnapshotNode } from "./checks.ts";
4+
5+
const APP = "com.t3tools.t3code.dev";
6+
const node = (index: number, fields: Omit<SnapshotNode, "index">): SnapshotNode => ({
7+
index,
8+
hittable: true,
9+
kind: "button",
10+
...fields,
11+
});
12+
13+
const failing = (snapshot: Snapshot) =>
14+
Object.fromEntries(
15+
layoutChecks(snapshot, APP)
16+
.filter((item) => !item.pass)
17+
.map((item) => [item.name, item.detail]),
18+
);
19+
20+
describe("layoutChecks", () => {
21+
it("flags broken chrome but not scrolled content, nested controls or duplicates", () => {
22+
const snapshot: Snapshot = {
23+
appBundleId: APP,
24+
viewport: { width: 400, height: 800 },
25+
nodes: [
26+
node(0, {
27+
kind: "application",
28+
hittable: false,
29+
rect: { x: 0, y: 0, width: 400, height: 800 },
30+
}),
31+
node(1, {
32+
kind: "scroll-area",
33+
parentIndex: 0,
34+
rect: { x: 0, y: 0, width: 400, height: 700 },
35+
}),
36+
node(2, {
37+
label: "Row past the edge",
38+
parentIndex: 1,
39+
rect: { x: 0, y: 680, width: 400, height: 60 },
40+
}),
41+
node(3, { label: "Send", parentIndex: 0, rect: { x: 340, y: 770, width: 80, height: 40 } }),
42+
node(4, {
43+
label: "Card",
44+
parentIndex: 0,
45+
rect: { x: 10, y: 100, width: 200, height: 100 },
46+
}),
47+
node(5, { label: "Inner", parentIndex: 4, rect: { x: 20, y: 110, width: 50, height: 40 } }),
48+
node(6, { label: "Twin", parentIndex: 0, rect: { x: 10, y: 300, width: 80, height: 40 } }),
49+
node(7, { label: "Twin", parentIndex: 0, rect: { x: 10, y: 300, width: 80, height: 40 } }),
50+
node(8, {
51+
label: "Plan",
52+
parentIndex: 0,
53+
rect: { x: 100, y: 400, width: 100, height: 40 },
54+
}),
55+
node(9, {
56+
label: "Stop",
57+
parentIndex: 0,
58+
rect: { x: 120, y: 405, width: 100, height: 40 },
59+
}),
60+
node(10, {
61+
kind: "text-field",
62+
label: "Message",
63+
parentIndex: 0,
64+
rect: { x: 0, y: 600, width: 300, height: 40 },
65+
}),
66+
node(12, {
67+
label: "Row of a covered screen",
68+
parentIndex: 1,
69+
rect: { x: 100, y: 400, width: 100, height: 40 },
70+
}),
71+
node(11, {
72+
kind: "keyboard",
73+
hittable: false,
74+
rect: { x: 0, y: 500, width: 400, height: 300 },
75+
}),
76+
],
77+
};
78+
79+
expect(failing(snapshot)).toEqual({
80+
"no control cut off by the screen edge": '"Send" at 340,770 80×40',
81+
"no controls stacked on each other": '"Plan" and "Stop"',
82+
"no input under the keyboard": '"Message" at 0,600 300×40',
83+
});
84+
});
85+
86+
it("flags an error screen and controls under Android system bars", () => {
87+
const snapshot: Snapshot = {
88+
appBundleId: APP,
89+
viewport: { width: 1080, height: 2400 },
90+
nodes: [
91+
node(0, {
92+
kind: "text",
93+
label: "Render Error",
94+
rect: { x: 0, y: 200, width: 600, height: 60 },
95+
}),
96+
node(1, {
97+
kind: "text",
98+
systemBar: "status",
99+
bundleId: "com.android.systemui",
100+
rect: { x: 0, y: 0, width: 1080, height: 60 },
101+
}),
102+
node(2, {
103+
label: "Search",
104+
bundleId: APP,
105+
rect: { x: 900, y: 10, width: 120, height: 80 },
106+
}),
107+
],
108+
};
109+
110+
expect(Object.keys(failing(snapshot))).toEqual([
111+
"no error screen",
112+
"no control under a system bar",
113+
]);
114+
});
115+
});

‎scripts/mobile-e2e/checks.ts‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
export interface Rect {
2+
readonly x: number;
3+
readonly y: number;
4+
readonly width: number;
5+
readonly height: number;
6+
}
7+
8+
/** One node of an agent-device `snapshot --json`, on iOS or Android. */
9+
export interface SnapshotNode {
10+
readonly index: number;
11+
readonly parentIndex?: number;
12+
/** Valid for commands until the next snapshot. */
13+
readonly ref?: string;
14+
readonly kind?: string;
15+
readonly type?: string;
16+
readonly label?: string;
17+
readonly identifier?: string;
18+
readonly rect?: Rect;
19+
readonly hittable?: boolean;
20+
readonly editable?: boolean;
21+
readonly bundleId?: string;
22+
readonly systemBar?: string;
23+
}
24+
25+
export interface Snapshot {
26+
readonly nodes: ReadonlyArray<SnapshotNode>;
27+
readonly viewport: { readonly width: number; readonly height: number };
28+
readonly appBundleId?: string;
29+
readonly keyboard?: { readonly kind?: string; readonly rect?: Rect };
30+
}
31+
32+
export interface Check {
33+
readonly name: string;
34+
readonly pass: boolean;
35+
readonly detail: string;
36+
}
37+
38+
export const check = (name: string, pass: boolean, detail: string): Check => ({
39+
name,
40+
pass,
41+
detail,
42+
});
43+
44+
// Kinds that are never controls a person taps; iOS also marks plain text hittable.
45+
const NOT_CONTROLS = new Set([
46+
"application",
47+
"text",
48+
"image",
49+
"scroll-area",
50+
"other",
51+
"toolbar",
52+
"tab-bar",
53+
"activity-indicator",
54+
"composeview",
55+
"keyboard",
56+
]);
57+
58+
const area = (rect: Rect) => Math.max(0, rect.width) * Math.max(0, rect.height);
59+
60+
const intersection = (a: Rect, b: Rect) =>
61+
Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x)) *
62+
Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y));
63+
64+
export const describeRect = (rect: Rect | undefined) =>
65+
rect
66+
? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}`
67+
: "absent";
68+
69+
const name = (node: SnapshotNode) => node.label || node.identifier || node.type || "unlabelled";
70+
71+
const appOwned = (node: SnapshotNode, snapshot: Snapshot) =>
72+
!node.systemBar && (node.bundleId === undefined || node.bundleId === snapshot.appBundleId);
73+
74+
const isControl = (node: SnapshotNode, snapshot: Snapshot) =>
75+
node.hittable === true &&
76+
node.rect !== undefined &&
77+
area(node.rect) > 0 &&
78+
!NOT_CONTROLS.has(node.kind ?? "") &&
79+
appOwned(node, snapshot);
80+
81+
function ancestors(node: SnapshotNode, snapshot: Snapshot): SnapshotNode[] {
82+
const byIndex = new Map(snapshot.nodes.map((candidate) => [candidate.index, candidate]));
83+
const chain: SnapshotNode[] = [];
84+
for (let parent = node.parentIndex; parent !== undefined;) {
85+
const next = byIndex.get(parent);
86+
if (!next || chain.includes(next)) break;
87+
chain.push(next);
88+
parent = next.parentIndex;
89+
}
90+
return chain;
91+
}
92+
93+
const viewportRect = (snapshot: Snapshot): Rect => ({ x: 0, y: 0, ...snapshot.viewport });
94+
95+
export function keyboardRect(snapshot: Snapshot): Rect | undefined {
96+
if (snapshot.keyboard?.rect) return snapshot.keyboard.rect;
97+
return snapshot.nodes.find((node) => node.type === "Keyboard" || node.kind === "keyboard")?.rect;
98+
}
99+
100+
const TAB_LABELS = new Set(["Chats", "Pulls", "Usage", "Settings"]);
101+
102+
/** The bottom tab bar, as a native tab bar node or the union of its tab buttons. */
103+
export function tabBarRect(snapshot: Snapshot): Rect | undefined {
104+
const native = snapshot.nodes.find((node) => node.kind === "tab-bar" && node.rect)?.rect;
105+
if (native) return native;
106+
const tabs = snapshot.nodes.filter(
107+
(node) =>
108+
node.rect &&
109+
TAB_LABELS.has(node.label ?? "") &&
110+
node.rect.y > snapshot.viewport.height * 0.7 &&
111+
appOwned(node, snapshot),
112+
);
113+
if (tabs.length < 2) return undefined;
114+
const left = Math.min(...tabs.map((tab) => tab.rect!.x));
115+
const top = Math.min(...tabs.map((tab) => tab.rect!.y));
116+
const right = Math.max(...tabs.map((tab) => tab.rect!.x + tab.rect!.width));
117+
const bottom = Math.max(...tabs.map((tab) => tab.rect!.y + tab.rect!.height));
118+
return { x: left, y: top, width: right - left, height: bottom - top };
119+
}
120+
121+
const ERROR_TEXT =
122+
/\b(Render Error|Uncaught Error|Unhandled JS Exception|Syntax Error|keeps stopping|isn't responding)\b/i;
123+
124+
/**
125+
* Checks every screen should pass, whatever it shows: the app is in front without an error
126+
* screen, and no control is clipped by the screen edge, stacked on another control, under a
127+
* system bar, or an input under the keyboard.
128+
*/
129+
export function layoutChecks(snapshot: Snapshot, bundleId: string): Check[] {
130+
const viewport = viewportRect(snapshot);
131+
const controls = snapshot.nodes.filter((node) => isControl(node, snapshot));
132+
const errors = snapshot.nodes.filter((node) => ERROR_TEXT.test(node.label ?? ""));
133+
134+
const clipped = controls.filter((node) => {
135+
const visible = intersection(node.rect!, viewport);
136+
if (visible === 0 || visible >= area(node.rect!) * 0.98) return false;
137+
// Content scrolled past the edge is expected; fixed chrome cut off by it is not.
138+
return !ancestors(node, snapshot).some((parent) => parent.kind === "scroll-area");
139+
});
140+
141+
// Only siblings: a screen covered by another stays in the tree, so cousins overlap
142+
// legitimately. Native tab items report overlapping frames.
143+
const siblings = controls.filter(
144+
(node) => !ancestors(node, snapshot).some((parent) => parent.kind === "tab-bar"),
145+
);
146+
const stacked: string[] = [];
147+
for (const [index, a] of siblings.entries()) {
148+
for (const b of siblings.slice(index + 1)) {
149+
if (a.parentIndex !== b.parentIndex) continue;
150+
const shared = intersection(a.rect!, b.rect!);
151+
if (shared < Math.min(area(a.rect!), area(b.rect!)) * 0.25) continue;
152+
// iOS exposes some elements twice, at the same frame.
153+
const same =
154+
describeRect(a.rect) === describeRect(b.rect) &&
155+
(!a.label || !b.label || a.label === b.label);
156+
if (!same) stacked.push(`"${name(a)}" and "${name(b)}"`);
157+
}
158+
}
159+
160+
const bars = snapshot.nodes.filter((node) => node.systemBar && node.rect);
161+
const underBars = controls.filter((node) =>
162+
bars.some((bar) => intersection(node.rect!, bar.rect!) > area(node.rect!) * 0.3),
163+
);
164+
165+
const keyboard = keyboardRect(snapshot);
166+
const coveredInputs = keyboard
167+
? snapshot.nodes.filter(
168+
(node) =>
169+
(node.kind === "text-field" || node.editable === true) &&
170+
node.rect &&
171+
appOwned(node, snapshot) &&
172+
intersection(node.rect, keyboard) > area(node.rect) * 0.3,
173+
)
174+
: [];
175+
176+
const list = (nodes: ReadonlyArray<SnapshotNode>) =>
177+
nodes.map((node) => `"${name(node)}" at ${describeRect(node.rect)}`).join(", ");
178+
return [
179+
check(
180+
"app in front",
181+
snapshot.appBundleId === undefined || snapshot.appBundleId === bundleId,
182+
snapshot.appBundleId ?? "unknown",
183+
),
184+
check("no error screen", errors.length === 0, errors.map(name).join(", ") || "none"),
185+
check("no control cut off by the screen edge", clipped.length === 0, list(clipped) || "none"),
186+
check("no controls stacked on each other", stacked.length === 0, stacked.join("; ") || "none"),
187+
check("no control under a system bar", underBars.length === 0, list(underBars) || "none"),
188+
check(
189+
"no input under the keyboard",
190+
coveredInputs.length === 0,
191+
keyboard ? list(coveredInputs) || "none" : "no keyboard",
192+
),
193+
];
194+
}
195+
196+
export const findLabel = (snapshot: Snapshot, pattern: RegExp) =>
197+
snapshot.nodes.find(
198+
(node) =>
199+
node.rect &&
200+
pattern.test(node.label ?? "") &&
201+
intersection(node.rect, viewportRect(snapshot)) > 0 &&
202+
appOwned(node, snapshot),
203+
);

0 commit comments

Comments
 (0)