Skip to content

Commit c4ce7b0

Browse files
authored
fix(desktop): update xa11y to 0.15.4 so App Shots read window text faster (#33135)
1 parent 06f8b99 commit c4ce7b0

9 files changed

Lines changed: 75 additions & 64 deletions

‎apps/desktop/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"dependencies": {
1313
"@clerk/electron": "catalog:",
1414
"@clerk/electron-passkeys": "catalog:",
15-
"@crowecawcaw/xa11y": "0.13.0",
15+
"@crowecawcaw/xa11y": "0.15.4",
1616
"@effect/platform-node": "catalog:",
1717
"@napi-rs/keyring": "^1.3.0",
1818
"@t3tools/client-runtime": "workspace:*",

‎apps/desktop/src/electron/ElectronWindow.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,15 @@ vi.mock("electron", () => ({
5353
getAllWindows: getAllWindowsMock,
5454
getFocusedWindow: getFocusedWindowMock,
5555
}),
56+
screen: {
57+
// A 200% display, so the focus target must carry converted bounds.
58+
dipToScreenRect: (_window: unknown, rect: Electron.Rectangle) => ({
59+
x: rect.x * 2,
60+
y: rect.y * 2,
61+
width: rect.width * 2,
62+
height: rect.height * 2,
63+
}),
64+
},
5665
}));
5766

5867
import * as ElectronWindow from "./ElectronWindow.ts";
@@ -116,6 +125,13 @@ describe("ElectronWindow", () => {
116125
});
117126
const electronWindow = yield* ElectronWindow.ElectronWindow;
118127
yield* electronWindow.prepareReveal(window as unknown as Electron.BrowserWindow);
128+
assert.deepEqual(windowsForegroundPrepareMock.mock.calls[0]?.[0], {
129+
windowId: 41,
130+
processId: process.pid,
131+
title: "T3 Code (Dev)",
132+
bounds: { x: 200, y: 100, width: 2_400, height: 1_600 },
133+
contentBounds: { x: 216, y: 100, width: 2_368, height: 1_584 },
134+
});
119135
window.getTitle.mockClear();
120136

121137
const revealFiber = yield* electronWindow

‎apps/desktop/src/electron/ElectronWindow.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ function windowsForegroundFocusTarget(window: Electron.BrowserWindow) {
2626
windowId: window.id,
2727
processId: process.pid,
2828
title: window.getTitle(),
29-
bounds: window.getBounds(),
30-
contentBounds: window.getContentBounds(),
29+
// xa11y reports physical desktop pixels on Windows; Electron bounds are DIPs.
30+
bounds: Electron.screen.dipToScreenRect(window, window.getBounds()),
31+
contentBounds: Electron.screen.dipToScreenRect(window, window.getContentBounds()),
3132
};
3233
}
3334

‎apps/desktop/src/electron/WindowsForegroundFocusWorker.ts‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,7 @@ async function findTarget(
3232
const children = await App.byPid(target.processId, { timeout: 0 })
3333
.then((app) => app.children())
3434
.catch(() => []);
35-
return (
36-
children.find((candidate) => matchesTarget(candidate, target)) ??
37-
(await App.list())
38-
.filter((candidate) => candidate.pid === target.processId)
39-
.map((candidate) => candidate.asElement())
40-
.find((candidate) => matchesTarget(candidate, target))
41-
);
35+
return children.find((candidate) => matchesTarget(candidate, target));
4236
}
4337

4438
const cachedElements = new Map<number, Element>();

‎apps/desktop/src/snapShot/DesktopSnapShot.test.ts‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -978,7 +978,7 @@ it.effect.each(["succeeds", "fails"] as const)(
978978
},
979979
);
980980

981-
it.effect("matches Windows accessibility windows on a scaled display", () => {
981+
it.effect("captures and reads Windows windows in physical pixels on a scaled display", () => {
982982
const png = Buffer.from([1, 2, 3]);
983983
const active = {
984984
platform: "windows",
@@ -987,26 +987,26 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
987987
owner: { name: "Editor", processId: 123 },
988988
bounds: { x: 10, y: 20, width: 800, height: 600 },
989989
} as const;
990-
const dipBounds = { x: 5, y: 10, width: 400, height: 300 };
990+
const window = (name: string, value: string) => ({
991+
role: "window",
992+
name,
993+
bounds: active.bounds,
994+
children: async () => [
995+
{
996+
role: "button",
997+
name: "Save",
998+
bounds: { x: 210, y: 220, width: 200, height: 100 },
999+
children: async () => [],
1000+
},
1001+
],
1002+
tree: async () => ({ name, value, children: [] }),
1003+
});
9911004
activeWindowMock.mockReset().mockResolvedValue(active);
992-
screenToDipRectMock.mockImplementation(() => dipBounds);
1005+
screenToDipRectMock.mockImplementation(() => ({ x: 5, y: 10, width: 400, height: 300 }));
9931006
regionCaptureMock.mockReset().mockResolvedValue({ width: 800, height: 600, png });
9941007
accessibilityForegroundMock.mockReset().mockResolvedValue({
9951008
pid: 123,
996-
asElement: () => ({
997-
role: "window",
998-
name: "Editor",
999-
bounds: dipBounds,
1000-
children: async () => [
1001-
{
1002-
role: "button",
1003-
name: "Save",
1004-
bounds: { x: 105, y: 110, width: 100, height: 50 },
1005-
children: async () => [],
1006-
},
1007-
],
1008-
tree: async () => ({ name: "Editor", value: "Scaled text", children: [] }),
1009-
}),
1009+
children: async () => [window("Settings", "Wrong window"), window("Editor", "Scaled text")],
10101010
});
10111011
let metadata = "";
10121012
return Effect.scoped(
@@ -1015,6 +1015,7 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
10151015
yield* service.configure(enabledSettings());
10161016
yield* service.capture;
10171017
const saved = yield* decodePendingMetadata(metadata);
1018+
assert.deepEqual(regionCaptureMock.mock.calls[0]?.[0], active.bounds);
10181019
assert.include(saved.source.accessibleText, "Scaled text");
10191020
assert.deepEqual(
10201021
saved.source.accessibility?.format === "element-tree"
@@ -1402,7 +1403,7 @@ it.each(["darwin", "win32", "linux"] as const)(
14021403
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
14031404
accessibilityForegroundMock
14041405
.mockReset()
1405-
.mockResolvedValue({ pid: 123, asElement: () => window });
1406+
.mockResolvedValue({ pid: 123, children: async () => [window] });
14061407

14071408
const result = await readAccessibleWindowContext(
14081409
{ title: "Editor", bounds, owner: { processId: 123 } },
@@ -1442,7 +1443,7 @@ it.each(["darwin", "win32"] as const)(
14421443
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
14431444
accessibilityForegroundMock
14441445
.mockReset()
1445-
.mockResolvedValue({ pid: 123, asElement: () => window });
1446+
.mockResolvedValue({ pid: 123, children: async () => [window] });
14461447
try {
14471448
assert.isUndefined(
14481449
await readAccessibleWindowText(
@@ -1484,7 +1485,7 @@ it.each([
14841485
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
14851486
accessibilityForegroundMock
14861487
.mockReset()
1487-
.mockResolvedValue({ pid: 123, asElement: () => window });
1488+
.mockResolvedValue({ pid: 123, children: async () => [window] });
14881489
try {
14891490
const result = await readAccessibleWindowContext(
14901491
{ title: "Editor", bounds, owner: { processId: 123 } },

‎apps/desktop/src/snapShot/DesktopSnapShot.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -508,7 +508,6 @@ async function captureSource({
508508
({ source, png } = await captureRegionWindowSnapshot(
509509
regionSnapShotPool,
510510
active,
511-
snapShotFlashBounds(active, platform),
512511
snapShotThumbnailSize(active),
513512
));
514513
} else {
@@ -547,7 +546,7 @@ async function captureSource({
547546
}
548547
}
549548
const accessibleIdentity = active
550-
? { ...active, bounds: snapShotFlashBounds(active, platform) }
549+
? active
551550
: linuxWindow?.processId
552551
? {
553552
title: linuxWindow.title,

‎apps/desktop/src/snapShot/RegionSnapShot.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,10 +175,9 @@ export function makeRegionSnapShotPool(
175175
export async function captureRegionWindowSnapshot(
176176
pool: Pick<RegionSnapShotPool, "capture">,
177177
active: ActiveWindow,
178-
region: Electron.Rectangle,
179178
maxSize: Electron.Size,
180179
): Promise<{ readonly source: RegionSnapShotSource; readonly png: Buffer }> {
181-
const shot = await pool.capture(region);
180+
const shot = await pool.capture(active.bounds);
182181
const scale = Math.min(maxSize.width / shot.width, maxSize.height / shot.height, 1);
183182
const png =
184183
scale < 1

‎apps/desktop/src/snapShot/SnapShotAccessibility.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ async function readCapturedWindowAccessibility(
125125
const pidWindows =
126126
foreground !== undefined
127127
? foreground.pid === active.owner.processId
128-
? [foreground.asElement()]
128+
? await foreground.children().catch(() => [])
129129
: []
130130
: await windowsForPid(App, active.owner.processId);
131131
const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";

‎pnpm-lock.yaml‎

Lines changed: 30 additions & 29 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)