Skip to content

Commit b67bb2b

Browse files
committed
fix(desktop): update xa11y to 0.15.4 so App Shots read window text faster
xa11y 0.15 reads each element in fewer cross-process calls on Windows, but changes two things App Shots relied on: App.foreground now returns the process with its windows as children, and Windows bounds, screenshot regions and element positions are physical desktop pixels instead of DIPs. Look up the captured window among the foreground process's windows, pass physical window bounds to capture and window matching, and convert the focus target's Electron bounds to physical pixels.
1 parent d456cd7 commit b67bb2b

9 files changed

Lines changed: 60 additions & 64 deletions

‎apps/desktop/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"dependencies": {
1313
"@clerk/electron": "catalog:",
1414
"@clerk/electron-passkeys": "catalog:",
15-
"@crowecawcaw/xa11y": "0.13.0",
15+
"@crowecawcaw/xa11y": "0.15.4",
1616
"@effect/platform-node": "catalog:",
1717
"@napi-rs/keyring": "^1.3.0",
1818
"@t3tools/client-runtime": "workspace:*",

‎apps/desktop/src/electron/ElectronWindow.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ vi.mock("electron", () => ({
5353
getAllWindows: getAllWindowsMock,
5454
getFocusedWindow: getFocusedWindowMock,
5555
}),
56+
screen: { dipToScreenRect: (_window: unknown, rect: unknown) => rect },
5657
}));
5758

5859
import * as ElectronWindow from "./ElectronWindow.ts";

‎apps/desktop/src/electron/ElectronWindow.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ function windowsForegroundFocusTarget(window: Electron.BrowserWindow) {
2626
windowId: window.id,
2727
processId: process.pid,
2828
title: window.getTitle(),
29-
bounds: window.getBounds(),
30-
contentBounds: window.getContentBounds(),
29+
// xa11y reports physical desktop pixels on Windows; Electron bounds are DIPs.
30+
bounds: Electron.screen.dipToScreenRect(window, window.getBounds()),
31+
contentBounds: Electron.screen.dipToScreenRect(window, window.getContentBounds()),
3132
};
3233
}
3334

‎apps/desktop/src/electron/WindowsForegroundFocusWorker.ts‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,7 @@ async function findTarget(
3232
const children = await App.byPid(target.processId, { timeout: 0 })
3333
.then((app) => app.children())
3434
.catch(() => []);
35-
return (
36-
children.find((candidate) => matchesTarget(candidate, target)) ??
37-
(await App.list())
38-
.filter((candidate) => candidate.pid === target.processId)
39-
.map((candidate) => candidate.asElement())
40-
.find((candidate) => matchesTarget(candidate, target))
41-
);
35+
return children.find((candidate) => matchesTarget(candidate, target));
4236
}
4337

4438
const cachedElements = new Map<number, Element>();

‎apps/desktop/src/snapShot/DesktopSnapShot.test.ts‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,7 @@ it.effect.each(["succeeds", "fails"] as const)(
978978
},
979979
);
980980

981-
it.effect("matches Windows accessibility windows on a scaled display", () => {
981+
it.effect("captures and reads Windows windows in physical pixels on a scaled display", () => {
982982
const png = Buffer.from([1, 2, 3]);
983983
const active = {
984984
platform: "windows",
@@ -987,26 +987,26 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
987987
owner: { name: "Editor", processId: 123 },
988988
bounds: { x: 10, y: 20, width: 800, height: 600 },
989989
} as const;
990-
const dipBounds = { x: 5, y: 10, width: 400, height: 300 };
990+
const window = (name: string, value: string) => ({
991+
role: "window",
992+
name,
993+
bounds: active.bounds,
994+
children: async () => [
995+
{
996+
role: "button",
997+
name: "Save",
998+
bounds: { x: 210, y: 220, width: 200, height: 100 },
999+
children: async () => [],
1000+
},
1001+
],
1002+
tree: async () => ({ name, value, children: [] }),
1003+
});
9911004
activeWindowMock.mockReset().mockResolvedValue(active);
992-
screenToDipRectMock.mockImplementation(() => dipBounds);
1005+
screenToDipRectMock.mockImplementation(() => ({ x: 5, y: 10, width: 400, height: 300 }));
9931006
regionCaptureMock.mockReset().mockResolvedValue({ width: 800, height: 600, png });
9941007
accessibilityForegroundMock.mockReset().mockResolvedValue({
9951008
pid: 123,
996-
asElement: () => ({
997-
role: "window",
998-
name: "Editor",
999-
bounds: dipBounds,
1000-
children: async () => [
1001-
{
1002-
role: "button",
1003-
name: "Save",
1004-
bounds: { x: 105, y: 110, width: 100, height: 50 },
1005-
children: async () => [],
1006-
},
1007-
],
1008-
tree: async () => ({ name: "Editor", value: "Scaled text", children: [] }),
1009-
}),
1009+
children: async () => [window("Settings", "Wrong window"), window("Editor", "Scaled text")],
10101010
});
10111011
let metadata = "";
10121012
return Effect.scoped(
@@ -1015,6 +1015,7 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
10151015
yield* service.configure(enabledSettings());
10161016
yield* service.capture;
10171017
const saved = yield* decodePendingMetadata(metadata);
1018+
assert.deepEqual(regionCaptureMock.mock.calls[0]?.[0], active.bounds);
10181019
assert.include(saved.source.accessibleText, "Scaled text");
10191020
assert.deepEqual(
10201021
saved.source.accessibility?.format === "element-tree"
@@ -1402,7 +1403,7 @@ it.each(["darwin", "win32", "linux"] as const)(
14021403
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
14031404
accessibilityForegroundMock
14041405
.mockReset()
1405-
.mockResolvedValue({ pid: 123, asElement: () => window });
1406+
.mockResolvedValue({ pid: 123, children: async () => [window] });
14061407

14071408
const result = await readAccessibleWindowContext(
14081409
{ title: "Editor", bounds, owner: { processId: 123 } },
@@ -1442,7 +1443,7 @@ it.each(["darwin", "win32"] as const)(
14421443
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
14431444
accessibilityForegroundMock
14441445
.mockReset()
1445-
.mockResolvedValue({ pid: 123, asElement: () => window });
1446+
.mockResolvedValue({ pid: 123, children: async () => [window] });
14461447
try {
14471448
assert.isUndefined(
14481449
await readAccessibleWindowText(
@@ -1484,7 +1485,7 @@ it.each([
14841485
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
14851486
accessibilityForegroundMock
14861487
.mockReset()
1487-
.mockResolvedValue({ pid: 123, asElement: () => window });
1488+
.mockResolvedValue({ pid: 123, children: async () => [window] });
14881489
try {
14891490
const result = await readAccessibleWindowContext(
14901491
{ title: "Editor", bounds, owner: { processId: 123 } },

‎apps/desktop/src/snapShot/DesktopSnapShot.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -508,7 +508,6 @@ async function captureSource({
508508
({ source, png } = await captureRegionWindowSnapshot(
509509
regionSnapShotPool,
510510
active,
511-
snapShotFlashBounds(active, platform),
512511
snapShotThumbnailSize(active),
513512
));
514513
} else {
@@ -547,7 +546,7 @@ async function captureSource({
547546
}
548547
}
549548
const accessibleIdentity = active
550-
? { ...active, bounds: snapShotFlashBounds(active, platform) }
549+
? active
551550
: linuxWindow?.processId
552551
? {
553552
title: linuxWindow.title,

‎apps/desktop/src/snapShot/RegionSnapShot.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,10 +175,9 @@ export function makeRegionSnapShotPool(
175175
export async function captureRegionWindowSnapshot(
176176
pool: Pick<RegionSnapShotPool, "capture">,
177177
active: ActiveWindow,
178-
region: Electron.Rectangle,
179178
maxSize: Electron.Size,
180179
): Promise<{ readonly source: RegionSnapShotSource; readonly png: Buffer }> {
181-
const shot = await pool.capture(region);
180+
const shot = await pool.capture(active.bounds);
182181
const scale = Math.min(maxSize.width / shot.width, maxSize.height / shot.height, 1);
183182
const png =
184183
scale < 1

‎apps/desktop/src/snapShot/SnapShotAccessibility.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ async function readCapturedWindowAccessibility(
125125
const pidWindows =
126126
foreground !== undefined
127127
? foreground.pid === active.owner.processId
128-
? [foreground.asElement()]
128+
? await foreground.children().catch(() => [])
129129
: []
130130
: await windowsForPid(App, active.owner.processId);
131131
const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";

‎pnpm-lock.yaml‎

Lines changed: 30 additions & 29 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)