Skip to content

Commit 1bbfe8a

Browse files
committed
fix(desktop): preserve long offscreen text in SnapShots
1 parent 26894dd commit 1bbfe8a

4 files changed

Lines changed: 136 additions & 8 deletions

File tree

‎apps/desktop/src/snapShot/DesktopSnapShot.test.ts‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2479,22 +2479,35 @@ it.each(["client", "frame"] as const)(
24792479
},
24802480
);
24812481

2482-
it.each(["darwin", "win32"] as const)(
2482+
it.each(["darwin", "win32", "linux"] as const)(
24832483
"extracts the same structured accessibility tree on %s",
24842484
async (platform) => {
24852485
const bounds = { x: 100, y: 200, width: 800, height: 600 };
24862486
const window = {
24872487
role: "window",
24882488
name: "Editor",
24892489
bounds,
2490-
tree: async () => ({ name: "Editor", children: [{ name: "Save", children: [] }] }),
2490+
tree: async () => ({
2491+
name: "Editor",
2492+
children: [
2493+
{ name: "Save", children: [] },
2494+
{ name: "Below scroll view", children: [] },
2495+
],
2496+
}),
24912497
children: async () => [
24922498
{
24932499
role: "button",
24942500
name: "Save",
24952501
bounds: { x: 300, y: 350, width: 100, height: 50 },
24962502
children: async () => [],
24972503
},
2504+
{
2505+
role: "static_text",
2506+
name: "Below scroll view",
2507+
bounds: { x: 300, y: 1_000, width: 100, height: 50 },
2508+
visible: false,
2509+
children: async () => [],
2510+
},
24982511
],
24992512
};
25002513
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
@@ -2516,6 +2529,12 @@ it.each(["darwin", "win32"] as const)(
25162529
: undefined,
25172530
"Save",
25182531
);
2532+
assert.deepInclude(
2533+
result?.accessibility?.format === "element-tree"
2534+
? result.accessibility.root.children[1]
2535+
: undefined,
2536+
{ name: "Below scroll view", bounds: null, state: { visible: false } },
2537+
);
25192538
assert.lengthOf(accessibilityByPidMock.mock.calls, platform === "win32" ? 0 : 1);
25202539
assert.lengthOf(accessibilityForegroundMock.mock.calls, platform === "win32" ? 1 : 0);
25212540
},
@@ -2554,6 +2573,48 @@ it.each(["darwin", "win32"] as const)(
25542573
},
25552574
);
25562575

2576+
it.each([
2577+
["darwin", "", "value", 650],
2578+
["win32", "", "value", 650],
2579+
["linux", "x11", "value", 650],
2580+
["linux", "wayland", "value", 650],
2581+
["darwin", "", "name", 100],
2582+
] as const)(
2583+
"preserves long text outside the scroll view on %s %s (%s)",
2584+
async (platform, session, field, lines) => {
2585+
const text = `${"Document line\n".repeat(lines)}End of document`;
2586+
vi.stubEnv("XDG_SESSION_TYPE", session);
2587+
const bounds = { x: 0, y: 0, width: 800, height: 600 };
2588+
const window = {
2589+
role: "window",
2590+
name: "Editor",
2591+
bounds,
2592+
tree: async () => ({ name: "Editor", children: [{ [field]: text, children: [] }] }),
2593+
children: async () => [
2594+
{ role: "text_area", [field]: text, bounds, children: async () => [] },
2595+
],
2596+
};
2597+
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
2598+
accessibilityForegroundMock
2599+
.mockReset()
2600+
.mockResolvedValue({ pid: 123, asElement: () => window });
2601+
try {
2602+
const result = await readAccessibleWindowContext(
2603+
{ title: "Editor", bounds, owner: { processId: 123 } },
2604+
platform,
2605+
"Editor",
2606+
);
2607+
assert.deepEqual(result?.accessibility, {
2608+
format: "flat-text",
2609+
text: `Editor\n${text}`,
2610+
truncated: false,
2611+
});
2612+
} finally {
2613+
vi.unstubAllEnvs();
2614+
}
2615+
},
2616+
);
2617+
25572618
it.each([
25582619
{ names: ["⠙ t3code"], expected: "Verified text" },
25592620
{ names: ["⠋ t3code", "⠙ t3code"], expected: undefined },
@@ -2681,6 +2742,44 @@ it("falls back to completed flat text when rich traversal reaches the deadline",
26812742
}
26822743
});
26832744

2745+
it("keeps a truncated element tree when the flat text read fails", async () => {
2746+
const bounds = { x: 0, y: 0, width: 800, height: 600 };
2747+
accessibilityByPidMock.mockReset().mockResolvedValue({
2748+
children: async () => [
2749+
{
2750+
role: "window",
2751+
name: "Editor",
2752+
bounds,
2753+
tree: async () => {
2754+
throw new Error("Text read failed");
2755+
},
2756+
children: async () => [
2757+
{
2758+
role: "text_area",
2759+
value: "x".repeat(8_001),
2760+
bounds: { x: 10, y: 50, width: 700, height: 500 },
2761+
children: async () => [],
2762+
},
2763+
],
2764+
},
2765+
],
2766+
});
2767+
2768+
const result = await readAccessibleWindowContext(
2769+
{ title: "Editor", bounds, owner: { processId: 123 } },
2770+
"darwin",
2771+
"Editor",
2772+
);
2773+
assert.equal(result?.accessibility?.format, "element-tree");
2774+
assert.isTrue(result?.accessibility?.truncated);
2775+
assert.deepEqual(
2776+
result?.accessibility?.format === "element-tree"
2777+
? result.accessibility.root.children[0]?.bounds
2778+
: undefined,
2779+
{ x: 10, y: 50, width: 700, height: 500 },
2780+
);
2781+
});
2782+
26842783
it("times out after three seconds without overlapping the outstanding accessibility read", async () => {
26852784
vi.useFakeTimers();
26862785
accessibilityByPidMock.mockReset();

‎apps/desktop/src/snapShot/SnapShotAccessibility.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,19 +63,19 @@ function accessibilityReadSnapshot(
6363
? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
6464
: undefined);
6565
const accessibility: SnapShotAccessibility | undefined =
66-
progress.richComplete && richTree
66+
progress.richComplete && richTree && !progress.richTruncated && !richTree.truncated
6767
? {
6868
format: "element-tree",
6969
coordinateSpace: "captured-image",
7070
imageSize,
71-
truncated: progress.richTruncated || richTree.truncated,
71+
truncated: false,
7272
root: richTree.root,
7373
}
74-
: progress.flatComplete && accessibleText
74+
: progress.flatComplete && progress.accessibleText
7575
? {
7676
format: "flat-text",
77-
text: accessibleText,
78-
truncated: accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
77+
text: progress.accessibleText,
78+
truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
7979
}
8080
: richTree
8181
? {

‎apps/desktop/src/snapShot/snapShot.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,23 @@ describe("accessibleWindowElementTree", () => {
263263
expect(tree?.root.children.length).toBeLessThan(10);
264264
expect(JSON.stringify(tree).length).toBeLessThanOrEqual(32_000);
265265
});
266+
267+
it.each(["name", "value", "description"] as const)(
268+
"reports clipped %s text in partial trees",
269+
async (field) => {
270+
const tree = await accessibleWindowElementTree(
271+
{
272+
role: "window",
273+
children: async () => [{ role: "text_area", [field]: "x".repeat(8_001) }],
274+
},
275+
{ x: 0, y: 0, width: 800, height: 600 },
276+
{ width: 800, height: 600 },
277+
);
278+
279+
expect(tree?.truncated).toBe(true);
280+
expect(tree?.root.children[0]?.[field]?.length).toBeLessThan(8_001);
281+
},
282+
);
266283
});
267284

268285
describe("compactAccessibilityTree", () => {

‎apps/desktop/src/snapShot/snapShot.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,11 +155,16 @@ function safeProperty<T>(read: () => T): T | undefined {
155155
}
156156
}
157157

158-
export function boundedSnapShotString(value: unknown, maxChars: number): string | undefined {
158+
export function boundedSnapShotString(
159+
value: unknown,
160+
maxChars: number,
161+
onTruncated?: () => void,
162+
): string | undefined {
159163
if (typeof value !== "string") return undefined;
160164
const candidate = value.replaceAll("\0", "").trim();
161165
if (!candidate) return undefined;
162166
if (candidate.length <= maxChars) return candidate;
167+
onTruncated?.();
163168
const end = /[\uD800-\uDBFF]/.test(candidate[maxChars - 1] ?? "") ? maxChars - 1 : maxChars;
164169
return candidate.slice(0, end).trimEnd();
165170
}
@@ -204,18 +209,22 @@ function accessibilityNode(
204209
imageSize: CapturedImageSize,
205210
isRoot: boolean,
206211
locationsReliable: boolean,
212+
onTruncated: () => void,
207213
): MutableAccessibilityNode {
208214
const name = boundedSnapShotString(
209215
safeProperty(() => element.name),
210216
1_000,
217+
onTruncated,
211218
);
212219
const value = boundedSnapShotString(
213220
safeProperty(() => element.value),
214221
8_000,
222+
onTruncated,
215223
);
216224
const description = boundedSnapShotString(
217225
safeProperty(() => element.description),
218226
2_000,
227+
onTruncated,
219228
);
220229
const checked = safeProperty(() => element.checked);
221230
const expanded = safeProperty(() => element.expanded);
@@ -389,6 +398,9 @@ export async function accessibleWindowElementTree(
389398
imageSize,
390399
required,
391400
options.locationsReliable !== false,
401+
() => {
402+
truncated = true;
403+
},
392404
);
393405
nodes += 1;
394406
root ??= node;

0 commit comments

Comments
 (0)