Skip to content

Commit 9392f8a

Browse files
fix(devices): flatten the nested iOS accessibility tree for the overlay
Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent 91be5c6 commit 9392f8a

1 file changed

Lines changed: 42 additions & 22 deletions

File tree

‎apps/web/src/components/device/deviceHubApi.ts‎

Lines changed: 42 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,44 @@ const isRecord = (value: unknown): value is Record<string, unknown> =>
6868
const numberOr = (value: unknown, fallback: number) =>
6969
typeof value === "number" && Number.isFinite(value) ? value : fallback;
7070

71+
const AX_ELEMENT_LIMIT = 500;
72+
73+
/**
74+
* serve-sim's helper returns the native nested tree; the root node is the
75+
* application covering the whole screen. Flatten it the way serve-sim's own
76+
* overlay does: skip nodes with the root's frame, cap the count.
77+
*/
78+
const flattenIosAxTree = (roots: ReadonlyArray<unknown>): ReadonlyArray<DeviceAxElement> => {
79+
const first = roots[0];
80+
const rootFrame = isRecord(first) && isRecord(first.frame) ? first.frame : null;
81+
const screenWidth = Math.max(1, numberOr(rootFrame?.width, 1));
82+
const screenHeight = Math.max(1, numberOr(rootFrame?.height, 1));
83+
const elements: DeviceAxElement[] = [];
84+
const visit = (node: unknown, path: string) => {
85+
if (elements.length >= AX_ELEMENT_LIMIT || !isRecord(node) || !isRecord(node.frame)) return;
86+
const frame = node.frame;
87+
const width = numberOr(frame.width, 0);
88+
const height = numberOr(frame.height, 0);
89+
const coversScreen =
90+
Math.abs(width - screenWidth) < 0.5 && Math.abs(height - screenHeight) < 0.5;
91+
if (!coversScreen && width > 0 && height > 0) {
92+
elements.push({
93+
id: typeof node.AXUniqueId === "string" ? node.AXUniqueId : path,
94+
label: typeof node.AXLabel === "string" ? node.AXLabel : "",
95+
role: typeof node.type === "string" ? node.type : "",
96+
x: numberOr(frame.x, 0) / screenWidth,
97+
y: numberOr(frame.y, 0) / screenHeight,
98+
width: width / screenWidth,
99+
height: height / screenHeight,
100+
});
101+
}
102+
const children = Array.isArray(node.children) ? node.children : [];
103+
children.forEach((child, index) => visit(child, `${path}.${index}`));
104+
};
105+
roots.forEach((root, index) => visit(root, String(index)));
106+
return elements;
107+
};
108+
71109
export async function fetchDeviceAxTree(
72110
target: Target,
73111
signal?: AbortSignal,
@@ -78,29 +116,11 @@ export async function fetchDeviceAxTree(
78116
hubUrl(target, `/helper/${encodeURIComponent(target.deviceId)}/ax`),
79117
signal,
80118
);
81-
if (!isRecord(payload) || !Array.isArray(payload.elements)) {
82-
return { elements: [], errors: ["Unexpected accessibility payload."] };
119+
if (!Array.isArray(payload)) {
120+
const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null;
121+
return { elements: [], errors: [error ?? "Unexpected accessibility payload."] };
83122
}
84-
const screen = isRecord(payload.screen) ? payload.screen : {};
85-
const screenWidth = Math.max(1, numberOr(screen.width, 1));
86-
const screenHeight = Math.max(1, numberOr(screen.height, 1));
87-
const elements = payload.elements.flatMap((raw): DeviceAxElement[] => {
88-
if (!isRecord(raw) || !isRecord(raw.frame)) return [];
89-
const frame = raw.frame;
90-
return [
91-
{
92-
id: String(raw.id ?? raw.path ?? ""),
93-
label: typeof raw.label === "string" ? raw.label : "",
94-
role: typeof raw.role === "string" ? raw.role : "",
95-
x: numberOr(frame.x, 0) / screenWidth,
96-
y: numberOr(frame.y, 0) / screenHeight,
97-
width: numberOr(frame.width, 0) / screenWidth,
98-
height: numberOr(frame.height, 0) / screenHeight,
99-
},
100-
];
101-
});
102-
const errors = Array.isArray(payload.errors) ? payload.errors.map(String) : [];
103-
return { elements, errors };
123+
return { elements: flattenIosAxTree(payload), errors: [] };
104124
}
105125
const payload = await fetchJson(
106126
target,

0 commit comments

Comments
 (0)