Skip to content

Commit b0a0820

Browse files
committed
fix(android): back off a timed-out snapshot helper session and bound content re-captures
A session capture that failed is torn down and its helper build stays one-shot for the fallback budget plus the start backoff, instead of being respawned and timing out again on every attempt and command. Content re-captures stop after a 10s window so slow attempts cannot run a snapshot past the 90s request envelope. A helper failure reported under am exit 0 keeps its own reason instead of 'output could not be parsed'.
1 parent 9a245d0 commit b0a0820

8 files changed

Lines changed: 224 additions & 38 deletions
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import { afterEach, beforeEach, test, vi } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
4+
vi.mock('../adb.ts', async (importOriginal) => {
5+
const actual = await importOriginal<typeof import('../adb.ts')>();
6+
return { ...actual, sleep: vi.fn(async () => {}) };
7+
});
8+
9+
import type { DeviceInfo } from '@agent-device/kernel/device';
10+
import { AppError } from '@agent-device/kernel/errors';
11+
import { snapshotAndroid } from '../snapshot.ts';
12+
import { resetAndroidSnapshotHelperInstallCache } from '../snapshot-helper-install.ts';
13+
import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts';
14+
import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts';
15+
import {
16+
androidSystemWindowOnlyXml,
17+
createPersistentSnapshotHelperProvider,
18+
type FakeAndroidProcess,
19+
} from './snapshot-helper-session.fixtures.ts';
20+
21+
const device: DeviceInfo = {
22+
platform: 'android',
23+
id: 'emulator-5554',
24+
name: 'Pixel',
25+
kind: 'emulator',
26+
booted: true,
27+
};
28+
29+
beforeEach(async () => {
30+
await resetAndroidSnapshotHelperSessions();
31+
resetAndroidSnapshotHelperInstallCache();
32+
});
33+
34+
afterEach(async () => {
35+
vi.restoreAllMocks();
36+
await resetAndroidSnapshotHelperSessions();
37+
});
38+
39+
/** Captures of a busy screen that each answer with system chrome only, after `captureCostMs`. */
40+
async function captureBusyScreen(captureCostMs: number): Promise<{
41+
error: unknown;
42+
sessionCaptures: number;
43+
}> {
44+
let clockOffsetMs = 0;
45+
const realNow = Date.now.bind(Date);
46+
vi.spyOn(Date, 'now').mockImplementation(() => realNow() + clockOffsetMs);
47+
let sessionCaptures = 0;
48+
const provider = createPersistentSnapshotHelperProvider({
49+
calls: [],
50+
spawnArgs: [],
51+
processes: [] as FakeAndroidProcess[],
52+
sessionXml: () => {
53+
sessionCaptures += 1;
54+
clockOffsetMs += captureCostMs;
55+
return androidSystemWindowOnlyXml();
56+
},
57+
});
58+
const error = await snapshotAndroid(device, {
59+
helperAdb: provider,
60+
helperArtifact: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT,
61+
appBundleId: 'com.example.app',
62+
}).then(
63+
() => undefined,
64+
(error: unknown) => error,
65+
);
66+
return { error, sessionCaptures };
67+
}
68+
69+
test('a busy screen whose captures are cheap is re-captured before its verdict', async () => {
70+
const { error, sessionCaptures } = await captureBusyScreen(10);
71+
72+
assert.ok(error instanceof AppError);
73+
assert.equal(error.details?.attempts, 3);
74+
assert.equal(sessionCaptures, 3);
75+
});
76+
77+
test('a busy screen whose captures are slow is not re-captured past the window', async () => {
78+
// One attempt that spends a whole helper command budget leaves no room in the daemon request
79+
// envelope for two more of the same.
80+
const { error, sessionCaptures } = await captureBusyScreen(30_000);
81+
82+
assert.ok(error instanceof AppError);
83+
assert.equal(error.details?.attempts, 1);
84+
assert.equal(sessionCaptures, 1);
85+
});

‎packages/platform-android/src/__tests__/snapshot-helper-capture.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import assert from 'node:assert/strict';
22
import { beforeEach, test } from 'vitest';
3+
import { AppError } from '@agent-device/kernel/errors';
34
import { captureAndroidSnapshotWithHelper } from '../snapshot-helper-capture.ts';
45
import { resetAndroidSnapshotHelperRetirements } from '../snapshot-helper-retirement.ts';
56
import type { AndroidAdbExecutor } from '../snapshot-helper-types.ts';
@@ -130,6 +131,33 @@ test('canceled one-shot capture reports the cancellation and the next capture re
130131
]);
131132
});
132133

134+
test('a helper failure reported under a zero am exit status keeps its own reason', async () => {
135+
// `am instrument` exits 0 after the helper finished with ok=false, as an emulator answered a
136+
// capture of an app whose main thread never served its window.
137+
const adb: AndroidAdbExecutor = async (args) => {
138+
if (isAndroidHelperRuntimeProbe(args)) return androidHelperRuntimeProbeResult();
139+
return {
140+
exitCode: 0,
141+
stdout: [
142+
'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1',
143+
'INSTRUMENTATION_RESULT: errorType=com.callstack.agentdevice.snapshothelper.AccessibilityCaptureStabilizer$IncompleteCaptureException',
144+
'INSTRUMENTATION_RESULT: message=Android accessibility capture remained incomplete after 500 ms',
145+
'INSTRUMENTATION_RESULT: ok=false',
146+
'INSTRUMENTATION_CODE: 1',
147+
].join('\n'),
148+
stderr: '',
149+
};
150+
};
151+
152+
await assert.rejects(
153+
captureAndroidSnapshotWithHelper({ adb, deviceKey: 'android:emulator-5554' }),
154+
(error: unknown) =>
155+
error instanceof AppError &&
156+
error.message === 'Android accessibility capture remained incomplete after 500 ms' &&
157+
typeof error.details?.helper === 'object',
158+
);
159+
});
160+
133161
function helperOutput(xml: string): string {
134162
return [
135163
'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1',

‎packages/platform-android/src/__tests__/snapshot-helper-session-lifecycle.test.ts‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,27 +137,28 @@ test('a generous caller budget buys a slow start, and never more than the caller
137137
test('a session that reaches ready settles a release the device could not confirm', async () => {
138138
const calls: (readonly string[])[] = [];
139139
const spawnArgs: (readonly string[])[] = [];
140+
const deviceKey = 'android:emulator-5554';
140141
// The device answers every process read with an adb error no classifier lists, and the first
141-
// command's session stalls, so that command's teardown records a release nothing could prove.
142+
// session ignores its quit, so that teardown records a release nothing could prove.
142143
const provider = createSessionProvider({
143144
calls,
144145
spawnArgs,
145-
stalledSnapshots: 1,
146+
quitResponseMode: 'malformed',
146147
runtimeRelease: 'closed',
147148
});
148149

149-
const stalled = await captureAndroidSnapshotWithHelperSession({
150+
await captureAndroidSnapshotWithHelperSession({
150151
adb: provider.exec,
151152
adbProvider: provider,
152-
deviceKey: 'android:emulator-5554',
153+
deviceKey,
153154
commandTimeoutMs: 400,
154155
});
155-
assert.equal(stalled, undefined);
156+
await stopAndroidSnapshotHelperSession(deviceKey);
156157

157158
const started = await captureAndroidSnapshotWithHelperSession({
158159
adb: provider.exec,
159160
adbProvider: provider,
160-
deviceKey: 'android:emulator-5554',
161+
deviceKey,
161162
commandTimeoutMs: 400,
162163
});
163164
assert.equal(started?.metadata.sessionReused, false);
@@ -169,7 +170,7 @@ test('a session that reaches ready settles a release the device could not confir
169170
const reused = await captureAndroidSnapshotWithHelperSession({
170171
adb: provider.exec,
171172
adbProvider: provider,
172-
deviceKey: 'android:emulator-5554',
173+
deviceKey,
173174
commandTimeoutMs: 400,
174175
});
175176

@@ -181,6 +182,28 @@ test('a session that reaches ready settles a release the device could not confir
181182
);
182183
});
183184

185+
test('a session whose capture timed out is not restarted by the next command', async () => {
186+
const calls: (readonly string[])[] = [];
187+
const spawnArgs: (readonly string[])[] = [];
188+
const provider = createSessionProvider({ calls, spawnArgs, stalledSnapshots: 1 });
189+
const capture = () =>
190+
captureAndroidSnapshotWithHelperSession({
191+
adb: provider.exec,
192+
adbProvider: provider,
193+
deviceKey: 'android:emulator-5554',
194+
commandTimeoutMs: 400,
195+
});
196+
197+
assert.equal(await capture(), undefined, 'the timed-out session falls back to one-shot');
198+
assert.equal(await capture(), undefined, 'the next command answers one-shot as well');
199+
assert.equal(spawnArgs.length, 1, 'a timed-out session earns a backoff, not another spawn');
200+
assert.equal(
201+
calls.some(isAndroidHelperRuntimeForceStop),
202+
true,
203+
'the timed-out helper is stopped',
204+
);
205+
});
206+
184207
test('starts and reuses a persistent Android snapshot helper session', async () => {
185208
const calls: (readonly string[])[] = [];
186209
const spawnArgs: (readonly string[])[] = [];

‎packages/platform-android/src/__tests__/snapshot-helper-session.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ test('release the transport could not confirm falls back instead of failing the
147147
});
148148
assert.equal(output, undefined, `attempt ${attempt}`);
149149
}
150-
assert.equal(processes.length, 2);
150+
// The malformed answer backs the session off, so the second command goes straight to one-shot.
151+
assert.equal(processes.length, 1);
151152
});
152153

153154
test('capture refuses a device the previous teardown found the helper still running', async () => {

‎packages/platform-android/src/snapshot-helper-capture.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,9 +228,13 @@ async function readFallbackHelperOutputOrThrow(
228228
result: Awaited<ReturnType<AndroidSnapshotHelperCaptureOptions['adb']>>,
229229
error: unknown,
230230
): Promise<AndroidSnapshotHelperReadResult> {
231-
if (error instanceof AppError && result.exitCode !== 0 && error.details?.helper) throw error;
231+
const helperFailure = error instanceof AppError && error.details?.helper ? error : undefined;
232+
if (helperFailure && result.exitCode !== 0) throw helperFailure;
232233
const fileOutput = await readFallbackHelperOutputFile(options, resolved, result);
233234
if (fileOutput) return { output: fileOutput, cleanupDone: true };
235+
// `am instrument` exits 0 after a helper that reported its own failure, so that report is the
236+
// answer whenever no output file stands in for it.
237+
if (helperFailure) throw helperFailure;
234238
// exec-guard-allow: reachable at exit 0 (helper output unparseable); the
235239
// message already branches on the exit code.
236240
throw new AppError(

‎packages/platform-android/src/snapshot-helper-session-lifecycle.ts‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -58,13 +58,13 @@ const SESSION_REQUEST_OVERHEAD_MS = 3_000;
5858
const FORWARD_TIMEOUT_MS = 5_000;
5959
// A helper that cannot start spends its whole start budget failing, and the one-shot transport that
6060
// answers afterwards still has to run. Retrying that on the very next command is what made commands
61-
// on the slow hosts of #2553 take roughly twice as long, so a failed start keeps the persistent path
62-
// away for at least this long.
63-
const SESSION_START_RETRY_FLOOR_MS = 10_000;
64-
// …and for no longer than this, however long the start took. The floor keeps a burst of commands
61+
// on the slow hosts of #2553 take roughly twice as long, so a failed start or session capture keeps
62+
// the persistent path away for at least this long.
63+
const SESSION_RETRY_FLOOR_MS = 10_000;
64+
// …and for no longer than this, however long the failure took. The floor keeps a burst of commands
6565
// from re-paying an instant failure; the ceiling keeps a host whose helper is simply broken from
6666
// being written off for longer than a working session would have lasted.
67-
const SESSION_START_RETRY_CEILING_MS = 60_000;
67+
const SESSION_RETRY_CEILING_MS = 60_000;
6868

6969
export type AndroidSnapshotHelperSessionHelperIdentity = {
7070
packageName: string;
@@ -93,8 +93,11 @@ export type AndroidSnapshotHelperSessionAcquisition = {
9393
};
9494

9595
const sessions = new Map<string, AndroidSnapshotHelperSession>();
96-
/** Capture identity → when this process may spawn that helper build again after a failed start. */
97-
const failedStarts = new Map<string, number>();
96+
/**
97+
* Capture identity → when this process may spawn that helper build again after a failed start or a
98+
* failed session capture.
99+
*/
100+
const failedSessions = new Map<string, number>();
98101

99102
/**
100103
* Starts (or reuses) the session without capturing, so a helper-backed read that is not a snapshot
@@ -150,7 +153,7 @@ async function resolveAndroidSnapshotHelperSession(params: {
150153
resolved: AndroidSnapshotHelperResolvedCaptureOptions;
151154
startBudgetMs: number;
152155
}): Promise<AndroidSnapshotHelperSession | undefined> {
153-
if (isAndroidSnapshotHelperStartBackedOff(params.identity)) return undefined;
156+
if (isAndroidSnapshotHelperSessionBackedOff(params.identity)) return undefined;
154157
await retireUnusableAndroidSnapshotHelperSession(params.deviceKey, params.identity);
155158
return sessions.get(params.deviceKey) ?? (await tryStartAndroidSnapshotHelperSession(params));
156159
}
@@ -186,9 +189,9 @@ async function tryStartAndroidSnapshotHelperSession(params: {
186189
return await startAndroidSnapshotHelperSession(params);
187190
} catch (error) {
188191
params.options.signal?.throwIfAborted();
189-
failedStarts.set(
192+
failedSessions.set(
190193
params.identity,
191-
Date.now() + androidSnapshotHelperStartRetryAfterMs(Date.now() - startedAtMs),
194+
Date.now() + androidSnapshotHelperSessionRetryAfterMs(Date.now() - startedAtMs),
192195
);
193196
emitDiagnostic({
194197
level: 'warn',
@@ -203,24 +206,52 @@ async function tryStartAndroidSnapshotHelperSession(params: {
203206
}
204207
}
205208

206-
/** A helper build whose last start failed is left alone until that start's backoff has run out. */
207-
function isAndroidSnapshotHelperStartBackedOff(identity: string): boolean {
208-
const retryAtMs = failedStarts.get(identity);
209+
/** A helper build whose last start or capture failed is left alone until its backoff has run out. */
210+
function isAndroidSnapshotHelperSessionBackedOff(identity: string): boolean {
211+
const retryAtMs = failedSessions.get(identity);
209212
if (retryAtMs === undefined) return false;
210213
if (retryAtMs > Date.now()) return true;
211-
failedStarts.delete(identity);
214+
failedSessions.delete(identity);
212215
return false;
213216
}
214217

215218
/**
216-
* How long a failed start earns: as long as it spent failing, because a start that burned half a
217-
* minute on a wedged device would burn another half minute on the next command, bounded so a burst
219+
* How long a failed start or capture earns: as long as it spent failing, because a start that burned
220+
* half a minute on a wedged device would burn another half minute on the next command, bounded so a burst
218221
* of commands neither re-pays an instant failure nor writes a device off for the rest of the run.
219222
*/
220-
function androidSnapshotHelperStartRetryAfterMs(startDurationMs: number): number {
221-
return Math.min(
222-
Math.max(startDurationMs, SESSION_START_RETRY_FLOOR_MS),
223-
SESSION_START_RETRY_CEILING_MS,
223+
function androidSnapshotHelperSessionRetryAfterMs(failureDurationMs: number): number {
224+
return Math.min(Math.max(failureDurationMs, SESSION_RETRY_FLOOR_MS), SESSION_RETRY_CEILING_MS);
225+
}
226+
227+
/**
228+
* Tears down a session whose capture failed and keeps its helper build off the persistent path for
229+
* as long as a failed start would, counted from when the one-shot capture that answers instead has
230+
* had its whole budget. A session that timed out once times out again on the same screen, so
231+
* restarting it on the next attempt or command only spends its budget again before the one-shot
232+
* transport answers. A cancellation is the caller's deadline, not evidence about the helper, so it
233+
* earns no backoff.
234+
*/
235+
export async function retireFailedAndroidSnapshotHelperSession(params: {
236+
deviceKey: string;
237+
failedAfterMs: number;
238+
/** Budget of the one-shot capture that answers in place of the failed session. */
239+
fallbackBudgetMs: number;
240+
signal?: AbortSignal;
241+
cause: unknown;
242+
}): Promise<void> {
243+
const identity = sessions.get(params.deviceKey)?.identity;
244+
await stopAndroidSnapshotHelperSession(params.deviceKey, {
245+
force: true,
246+
signal: params.signal,
247+
cause: params.cause,
248+
});
249+
if (identity === undefined || params.signal?.aborted) return;
250+
failedSessions.set(
251+
identity,
252+
Date.now() +
253+
params.fallbackBudgetMs +
254+
androidSnapshotHelperSessionRetryAfterMs(params.failedAfterMs),
224255
);
225256
}
226257

@@ -285,7 +316,7 @@ async function startAndroidSnapshotHelperSession(params: {
285316
params.options.signal,
286317
);
287318
sessions.set(params.deviceKey, session);
288-
failedStarts.delete(params.identity);
319+
failedSessions.delete(params.identity);
289320
// `am instrument` force-stops whatever is already instrumenting this package, so a helper that
290321
// reported itself ready is the only helper process the device has left, and the release the
291322
// previous teardown could not prove went away with the process that owed it. Leaving the entry
@@ -530,7 +561,7 @@ export async function resetAndroidSnapshotHelperSessions(): Promise<void> {
530561
// One teardown that throws must not leave the next caller believing a session, a pending
531562
// retirement, or a failed start is still standing.
532563
sessions.clear();
533-
failedStarts.clear();
564+
failedSessions.clear();
534565
resetAndroidSnapshotHelperRetirements();
535566
resetAndroidAdbShellProtocolProbes();
536567
}

0 commit comments

Comments
 (0)