Skip to content

Commit 1bd798f

Browse files
thymikeethymikee
committed
fix(android): return meaningful occluding system surfaces instead of failing the snapshot (#1253)
The notification shade and quick settings legitimately own the whole screen: the helper faithfully captures the active system window, but the content classifier treated the missing application window as a helper failure. Add a carve-out: an active/focused non-application window carrying meaningful content is returned as the snapshot, flagged systemSurfaceOnly, with an agent-facing warning explaining how to reach app content. Sparse or inactive system windows keep failing with the structured retriable error.
1 parent 117f781 commit 1bd798f

5 files changed

Lines changed: 232 additions & 39 deletions

File tree

‎src/commands/capture/runtime/snapshot.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,13 +243,23 @@ function buildSnapshotWarnings(params: {
243243
const reactNativeOverlayWarning = formatReactNativeOverlayWarning(params.snapshot.nodes);
244244
if (reactNativeOverlayWarning) warnings.push(reactNativeOverlayWarning);
245245

246+
const systemSurfaceWarning = formatAndroidSystemSurfaceWarning(params.annotations);
247+
if (systemSurfaceWarning) warnings.push(systemSurfaceWarning);
248+
246249
const recentDropWarning = formatRecentSnapshotDropWarning(params);
247250
if (recentDropWarning) warnings.push(recentDropWarning);
248251

249252
warnings.push(...formatFreshnessWarnings(params.annotations.freshness, params.snapshot.backend));
250253
return Array.from(new Set(warnings));
251254
}
252255

256+
function formatAndroidSystemSurfaceWarning(
257+
annotations: SnapshotCaptureAnnotations,
258+
): string | undefined {
259+
if (annotations.androidSnapshot?.systemSurfaceOnly !== true) return undefined;
260+
return 'A system surface (notification shade, quick settings, or another system overlay) covers the app, so this snapshot shows that surface. Interact with it or dismiss it (press back or swipe up) to reach app content.';
261+
}
262+
253263
function buildSparseIosInteractiveWarnings(params: {
254264
snapshot: SnapshotState;
255265
options: SnapshotCommandOptions;

‎src/platforms/android/__tests__/snapshot-content-recovery.test.ts‎

Lines changed: 134 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
11
import { test, expect } from 'vitest';
2-
import { classifyAndroidHelperContentRecovery } from '../snapshot-content-recovery.ts';
2+
import {
3+
classifyAndroidHelperContent,
4+
type AndroidHelperContentRecoveryDecision,
5+
} from '../snapshot-content-recovery.ts';
6+
7+
// Legacy assertions below check the unusable decision (or its absence); outcome-specific tests
8+
// call classifyAndroidHelperContent directly.
9+
function unusableDecisionFor(
10+
...args: Parameters<typeof classifyAndroidHelperContent>
11+
): AndroidHelperContentRecoveryDecision | undefined {
12+
const result = classifyAndroidHelperContent(...args);
13+
return result.outcome === 'unusable' ? result.decision : undefined;
14+
}
315

416
test('keeps known IME blocking windows instead of falling back to covered app content', () => {
5-
const decision = classifyAndroidHelperContentRecovery(
17+
const decision = unusableDecisionFor(
618
helperXml([
719
node({
820
windowType: 1,
@@ -40,7 +52,7 @@ test('keeps known IME blocking windows instead of falling back to covered app co
4052
});
4153

4254
test('rejects helper output with only one meaningful IME node', () => {
43-
const decision = classifyAndroidHelperContentRecovery(
55+
const decision = unusableDecisionFor(
4456
helperXml([
4557
node({
4658
windowType: 1,
@@ -68,7 +80,7 @@ test('rejects helper output with only one meaningful IME node', () => {
6880
});
6981

7082
test('rejects helper output with no foreground app or IME content', () => {
71-
const decision = classifyAndroidHelperContentRecovery(
83+
const decision = unusableDecisionFor(
7284
helperXml([
7385
node({
7486
windowType: 1,
@@ -96,7 +108,7 @@ test('rejects helper output with no foreground app or IME content', () => {
96108
});
97109

98110
test('keeps a meaningful foreground application surface owned by another package', () => {
99-
const decision = classifyAndroidHelperContentRecovery(
111+
const decision = unusableDecisionFor(
100112
helperXml([
101113
node({
102114
windowType: 1,
@@ -129,7 +141,7 @@ test('keeps a meaningful foreground application surface owned by another package
129141
});
130142

131143
test('rejects a status-bar-only capture without window metadata', () => {
132-
const decision = classifyAndroidHelperContentRecovery(
144+
const decision = unusableDecisionFor(
133145
helperXml([
134146
node({
135147
text: '7:52',
@@ -160,7 +172,7 @@ test('rejects a status-bar-only capture without window metadata', () => {
160172
});
161173

162174
test('rejects framework-owned content without window metadata', () => {
163-
const decision = classifyAndroidHelperContentRecovery(
175+
const decision = unusableDecisionFor(
164176
helperXml([
165177
node({
166178
text: 'System dialog',
@@ -184,7 +196,7 @@ test('rejects framework-owned content without window metadata', () => {
184196
});
185197

186198
test('keeps a recognized system dialog without window metadata', () => {
187-
const decision = classifyAndroidHelperContentRecovery(
199+
const decision = unusableDecisionFor(
188200
helperXml([
189201
node({
190202
text: "Demo isn't responding",
@@ -225,7 +237,7 @@ test('keeps a recognized system dialog without window metadata', () => {
225237
});
226238

227239
test('rejects system chrome inherited under an empty application window', () => {
228-
const decision = classifyAndroidHelperContentRecovery(
240+
const decision = unusableDecisionFor(
229241
helperXml([
230242
node({
231243
windowType: 1,
@@ -258,6 +270,114 @@ test('rejects system chrome inherited under an empty application window', () =>
258270
expect(decision?.diagnostics.helperNonSystemMeaningfulNodeCount).toBe(0);
259271
});
260272

273+
test('returns a meaningful active system surface (notification shade) faithfully', () => {
274+
const result = classifyAndroidHelperContent(
275+
helperXml([
276+
node({
277+
windowType: 3,
278+
windowActive: true,
279+
packageName: 'com.android.systemui',
280+
className: 'android.widget.FrameLayout',
281+
}),
282+
node({
283+
text: 'Wed, Jul 16',
284+
packageName: 'com.android.systemui',
285+
className: 'android.widget.TextView',
286+
}),
287+
node({
288+
text: 'Internet',
289+
resourceId: 'com.android.systemui:id/qs_tile_internet',
290+
packageName: 'com.android.systemui',
291+
className: 'android.widget.Switch',
292+
}),
293+
node({
294+
text: 'Manage',
295+
resourceId: 'com.android.systemui:id/manage_settings',
296+
packageName: 'com.android.systemui',
297+
className: 'android.widget.Button',
298+
}),
299+
]),
300+
{
301+
backend: 'android-helper',
302+
nodeCount: 4,
303+
rootPresent: true,
304+
windowCount: 1,
305+
captureMode: 'interactive-windows',
306+
},
307+
{ foregroundAppPackage: 'com.android.settings' },
308+
);
309+
310+
expect(result.outcome).toBe('system-surface-only');
311+
});
312+
313+
test('rejects a sparse active system surface below the meaningful-content floor', () => {
314+
const result = classifyAndroidHelperContent(
315+
helperXml([
316+
node({
317+
windowType: 3,
318+
windowActive: true,
319+
packageName: 'com.android.systemui',
320+
className: 'android.widget.FrameLayout',
321+
}),
322+
node({
323+
text: '7:52',
324+
resourceId: 'com.android.systemui:id/clock',
325+
packageName: 'com.android.systemui',
326+
className: 'android.widget.TextView',
327+
}),
328+
]),
329+
{
330+
backend: 'android-helper',
331+
nodeCount: 2,
332+
rootPresent: true,
333+
windowCount: 1,
334+
captureMode: 'interactive-windows',
335+
},
336+
{ foregroundAppPackage: 'com.android.settings' },
337+
);
338+
339+
expect(result.outcome).toBe('unusable');
340+
});
341+
342+
test('rejects a content-rich system window that is neither active nor focused', () => {
343+
const result = classifyAndroidHelperContent(
344+
helperXml([
345+
node({
346+
windowType: 3,
347+
packageName: 'com.android.systemui',
348+
className: 'android.widget.FrameLayout',
349+
}),
350+
node({
351+
text: 'Wed, Jul 16',
352+
packageName: 'com.android.systemui',
353+
className: 'android.widget.TextView',
354+
}),
355+
node({
356+
text: 'Internet',
357+
resourceId: 'com.android.systemui:id/qs_tile_internet',
358+
packageName: 'com.android.systemui',
359+
className: 'android.widget.Switch',
360+
}),
361+
node({
362+
text: 'Manage',
363+
resourceId: 'com.android.systemui:id/manage_settings',
364+
packageName: 'com.android.systemui',
365+
className: 'android.widget.Button',
366+
}),
367+
]),
368+
{
369+
backend: 'android-helper',
370+
nodeCount: 4,
371+
rootPresent: true,
372+
windowCount: 1,
373+
captureMode: 'interactive-windows',
374+
},
375+
{ foregroundAppPackage: 'com.android.settings' },
376+
);
377+
378+
expect(result.outcome).toBe('unusable');
379+
});
380+
261381
function helperXml(nodes: string[]): string {
262382
return `<?xml version='1.0' encoding='UTF-8' standalone='yes' ?><hierarchy>${nodes.join('')}</hierarchy>`;
263383
}
@@ -268,6 +388,10 @@ function node(options: {
268388
packageName: string;
269389
className: string;
270390
windowType?: number;
391+
windowActive?: boolean;
271392
}): string {
272-
return `<node index="0"${options.windowType === undefined ? '' : ` window-type="${options.windowType}"`} text="${options.text ?? ''}" resource-id="${options.resourceId ?? ''}" class="${options.className}" package="${options.packageName}" visible-to-user="true" enabled="true" bounds="[0,0][100,100]" />`;
393+
const windowAttrs =
394+
(options.windowType === undefined ? '' : ` window-type="${options.windowType}"`) +
395+
(options.windowActive === undefined ? '' : ` window-active="${options.windowActive}"`);
396+
return `<node index="0"${windowAttrs} text="${options.text ?? ''}" resource-id="${options.resourceId ?? ''}" class="${options.className}" package="${options.packageName}" visible-to-user="true" enabled="true" bounds="[0,0][100,100]" />`;
273397
}

0 commit comments

Comments
 (0)