|
| 1 | +export interface Rect { |
| 2 | + readonly x: number; |
| 3 | + readonly y: number; |
| 4 | + readonly width: number; |
| 5 | + readonly height: number; |
| 6 | +} |
| 7 | + |
| 8 | +/** One node of an agent-device `snapshot --json`, on iOS or Android. */ |
| 9 | +export interface SnapshotNode { |
| 10 | + readonly index: number; |
| 11 | + readonly parentIndex?: number; |
| 12 | + /** Valid for commands until the next snapshot. */ |
| 13 | + readonly ref?: string; |
| 14 | + readonly kind?: string; |
| 15 | + readonly type?: string; |
| 16 | + readonly label?: string; |
| 17 | + readonly identifier?: string; |
| 18 | + readonly rect?: Rect; |
| 19 | + readonly hittable?: boolean; |
| 20 | + readonly editable?: boolean; |
| 21 | + readonly bundleId?: string; |
| 22 | + readonly systemBar?: string; |
| 23 | +} |
| 24 | + |
| 25 | +export interface Snapshot { |
| 26 | + readonly nodes: ReadonlyArray<SnapshotNode>; |
| 27 | + readonly viewport: { readonly width: number; readonly height: number }; |
| 28 | + readonly appBundleId?: string; |
| 29 | + readonly keyboard?: { readonly kind?: string; readonly rect?: Rect }; |
| 30 | +} |
| 31 | + |
| 32 | +export interface Check { |
| 33 | + readonly name: string; |
| 34 | + readonly pass: boolean; |
| 35 | + readonly detail: string; |
| 36 | +} |
| 37 | + |
| 38 | +export const check = (name: string, pass: boolean, detail: string): Check => ({ |
| 39 | + name, |
| 40 | + pass, |
| 41 | + detail, |
| 42 | +}); |
| 43 | + |
| 44 | +// Kinds that are never controls a person taps; iOS also marks plain text hittable. |
| 45 | +const NOT_CONTROLS = new Set([ |
| 46 | + "application", |
| 47 | + "text", |
| 48 | + "image", |
| 49 | + "scroll-area", |
| 50 | + "other", |
| 51 | + "toolbar", |
| 52 | + "tab-bar", |
| 53 | + "activity-indicator", |
| 54 | + "composeview", |
| 55 | + "keyboard", |
| 56 | +]); |
| 57 | + |
| 58 | +const area = (rect: Rect) => Math.max(0, rect.width) * Math.max(0, rect.height); |
| 59 | + |
| 60 | +const intersection = (a: Rect, b: Rect) => |
| 61 | + Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x)) * |
| 62 | + Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y)); |
| 63 | + |
| 64 | +export const describeRect = (rect: Rect | undefined) => |
| 65 | + rect |
| 66 | + ? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}` |
| 67 | + : "absent"; |
| 68 | + |
| 69 | +const name = (node: SnapshotNode) => node.label || node.identifier || node.type || "unlabelled"; |
| 70 | + |
| 71 | +const appOwned = (node: SnapshotNode, snapshot: Snapshot) => |
| 72 | + !node.systemBar && (node.bundleId === undefined || node.bundleId === snapshot.appBundleId); |
| 73 | + |
| 74 | +const isControl = (node: SnapshotNode, snapshot: Snapshot) => |
| 75 | + node.hittable === true && |
| 76 | + node.rect !== undefined && |
| 77 | + area(node.rect) > 0 && |
| 78 | + !NOT_CONTROLS.has(node.kind ?? "") && |
| 79 | + appOwned(node, snapshot); |
| 80 | + |
| 81 | +function ancestors(node: SnapshotNode, snapshot: Snapshot): SnapshotNode[] { |
| 82 | + const byIndex = new Map(snapshot.nodes.map((candidate) => [candidate.index, candidate])); |
| 83 | + const chain: SnapshotNode[] = []; |
| 84 | + for (let parent = node.parentIndex; parent !== undefined;) { |
| 85 | + const next = byIndex.get(parent); |
| 86 | + if (!next || chain.includes(next)) break; |
| 87 | + chain.push(next); |
| 88 | + parent = next.parentIndex; |
| 89 | + } |
| 90 | + return chain; |
| 91 | +} |
| 92 | + |
| 93 | +const viewportRect = (snapshot: Snapshot): Rect => ({ x: 0, y: 0, ...snapshot.viewport }); |
| 94 | + |
| 95 | +export function keyboardRect(snapshot: Snapshot): Rect | undefined { |
| 96 | + if (snapshot.keyboard?.rect) return snapshot.keyboard.rect; |
| 97 | + return snapshot.nodes.find((node) => node.type === "Keyboard" || node.kind === "keyboard")?.rect; |
| 98 | +} |
| 99 | + |
| 100 | +const TAB_LABELS = new Set(["Chats", "Pulls", "Usage", "Settings"]); |
| 101 | + |
| 102 | +/** The bottom tab bar, as a native tab bar node or the union of its tab buttons. */ |
| 103 | +export function tabBarRect(snapshot: Snapshot): Rect | undefined { |
| 104 | + const native = snapshot.nodes.find((node) => node.kind === "tab-bar" && node.rect)?.rect; |
| 105 | + if (native) return native; |
| 106 | + const tabs = snapshot.nodes.filter( |
| 107 | + (node) => |
| 108 | + node.rect && |
| 109 | + TAB_LABELS.has(node.label ?? "") && |
| 110 | + node.rect.y > snapshot.viewport.height * 0.7 && |
| 111 | + appOwned(node, snapshot), |
| 112 | + ); |
| 113 | + if (tabs.length < 2) return undefined; |
| 114 | + const left = Math.min(...tabs.map((tab) => tab.rect!.x)); |
| 115 | + const top = Math.min(...tabs.map((tab) => tab.rect!.y)); |
| 116 | + const right = Math.max(...tabs.map((tab) => tab.rect!.x + tab.rect!.width)); |
| 117 | + const bottom = Math.max(...tabs.map((tab) => tab.rect!.y + tab.rect!.height)); |
| 118 | + return { x: left, y: top, width: right - left, height: bottom - top }; |
| 119 | +} |
| 120 | + |
| 121 | +const ERROR_TEXT = |
| 122 | + /\b(Render Error|Uncaught Error|Unhandled JS Exception|Syntax Error|keeps stopping|isn't responding)\b/i; |
| 123 | + |
| 124 | +/** |
| 125 | + * Checks every screen should pass, whatever it shows: the app is in front without an error |
| 126 | + * screen, and no control is clipped by the screen edge, stacked on another control, under a |
| 127 | + * system bar, or an input under the keyboard. |
| 128 | + */ |
| 129 | +export function layoutChecks(snapshot: Snapshot, bundleId: string): Check[] { |
| 130 | + const viewport = viewportRect(snapshot); |
| 131 | + const controls = snapshot.nodes.filter((node) => isControl(node, snapshot)); |
| 132 | + const errors = snapshot.nodes.filter((node) => ERROR_TEXT.test(node.label ?? "")); |
| 133 | + |
| 134 | + const clipped = controls.filter((node) => { |
| 135 | + const visible = intersection(node.rect!, viewport); |
| 136 | + if (visible === 0 || visible >= area(node.rect!) * 0.98) return false; |
| 137 | + // Content scrolled past the edge is expected; fixed chrome cut off by it is not. |
| 138 | + return !ancestors(node, snapshot).some((parent) => parent.kind === "scroll-area"); |
| 139 | + }); |
| 140 | + |
| 141 | + // Only siblings: a screen covered by another stays in the tree, so cousins overlap |
| 142 | + // legitimately. Native tab items report overlapping frames. |
| 143 | + const siblings = controls.filter( |
| 144 | + (node) => !ancestors(node, snapshot).some((parent) => parent.kind === "tab-bar"), |
| 145 | + ); |
| 146 | + const stacked: string[] = []; |
| 147 | + for (const [index, a] of siblings.entries()) { |
| 148 | + for (const b of siblings.slice(index + 1)) { |
| 149 | + if (a.parentIndex !== b.parentIndex) continue; |
| 150 | + const shared = intersection(a.rect!, b.rect!); |
| 151 | + if (shared < Math.min(area(a.rect!), area(b.rect!)) * 0.25) continue; |
| 152 | + // iOS exposes some elements twice, at the same frame. |
| 153 | + const same = |
| 154 | + describeRect(a.rect) === describeRect(b.rect) && |
| 155 | + (!a.label || !b.label || a.label === b.label); |
| 156 | + if (!same) stacked.push(`"${name(a)}" and "${name(b)}"`); |
| 157 | + } |
| 158 | + } |
| 159 | + |
| 160 | + const bars = snapshot.nodes.filter((node) => node.systemBar && node.rect); |
| 161 | + const underBars = controls.filter((node) => |
| 162 | + bars.some((bar) => intersection(node.rect!, bar.rect!) > area(node.rect!) * 0.3), |
| 163 | + ); |
| 164 | + |
| 165 | + const keyboard = keyboardRect(snapshot); |
| 166 | + const coveredInputs = keyboard |
| 167 | + ? snapshot.nodes.filter( |
| 168 | + (node) => |
| 169 | + (node.kind === "text-field" || node.editable === true) && |
| 170 | + node.rect && |
| 171 | + appOwned(node, snapshot) && |
| 172 | + intersection(node.rect, keyboard) > area(node.rect) * 0.3, |
| 173 | + ) |
| 174 | + : []; |
| 175 | + |
| 176 | + const list = (nodes: ReadonlyArray<SnapshotNode>) => |
| 177 | + nodes.map((node) => `"${name(node)}" at ${describeRect(node.rect)}`).join(", "); |
| 178 | + return [ |
| 179 | + check( |
| 180 | + "app in front", |
| 181 | + snapshot.appBundleId === undefined || snapshot.appBundleId === bundleId, |
| 182 | + snapshot.appBundleId ?? "unknown", |
| 183 | + ), |
| 184 | + check("no error screen", errors.length === 0, errors.map(name).join(", ") || "none"), |
| 185 | + check("no control cut off by the screen edge", clipped.length === 0, list(clipped) || "none"), |
| 186 | + check("no controls stacked on each other", stacked.length === 0, stacked.join("; ") || "none"), |
| 187 | + check("no control under a system bar", underBars.length === 0, list(underBars) || "none"), |
| 188 | + check( |
| 189 | + "no input under the keyboard", |
| 190 | + coveredInputs.length === 0, |
| 191 | + keyboard ? list(coveredInputs) || "none" : "no keyboard", |
| 192 | + ), |
| 193 | + ]; |
| 194 | +} |
| 195 | + |
| 196 | +export const findLabel = (snapshot: Snapshot, pattern: RegExp) => |
| 197 | + snapshot.nodes.find( |
| 198 | + (node) => |
| 199 | + node.rect && |
| 200 | + pattern.test(node.label ?? "") && |
| 201 | + intersection(node.rect, viewportRect(snapshot)) > 0 && |
| 202 | + appOwned(node, snapshot), |
| 203 | + ); |
0 commit comments