Skip to content

Commit 4f18a1a

Browse files
thymikeeclaude
andcommitted
fix(perf): keep Android clipboard and restore failures out of timings
The API 35 and 36 emulator images ship no shell clipboard service, so clipboard is an Android exclusion. A daemon restart that reopens the session clears the stuck flag, and one that cannot reopen it skips the rest of the scenario as restore_failed. The one-shot step warms the restarted daemon's helper install check untimed. latest and date references resolve to the newest run with the platform's artifact. The tour-only --n alias is gone. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent ea9af01 commit 4f18a1a

9 files changed

Lines changed: 94 additions & 41 deletions

File tree

‎scripts/perf/README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ Some shared steps use other screen coordinates or waits on Android. The `on.andr
5555
| --- | --- | --- |
5656
| `action-button` | lifecycle | Not timed: Android has no key event for the Action button. |
5757
| `prepare` | lifecycle (`prepare ios-runner`) | Not timed: `prepare` has only the `ios-runner` action. |
58+
| `clipboard` | lifecycle ×2 | Not timed: the API 35 and 36 emulator images have no shell clipboard service. |
5859
| `hardware-button` | Not timed: the simulator refuses hardware-button presses. | lifecycle (`hardware-button wake`) |
5960
| `push` | Not timed: the fixture has no notification authorization. | lifecycle (adb broadcast) |
6061
| `telephony` | Not timed: the simulator refuses calls and SMS. | lifecycle (`telephony sms`) |
@@ -95,7 +96,7 @@ All times are milliseconds. Statistics use only the successful samples of a step
9596
| `producer` | The number of samples that each snapshot producer answered: `simulator-ax-bridge` and `apple-runner` on iOS; `android-helper/persistent-session` and `android-helper/instrumentation` (one-shot `am instrument`) on Android. |
9697
| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected` or `android-helper/healthy`. |
9798
| fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. |
98-
| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that the forced path did not serve the capture. |
99+
| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `restore_failed` means that the daemon restart after a forced-path step could not reopen the session. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth, or that the forced path did not serve the capture. |
99100

100101
## Nightly results
101102

‎scripts/perf/compare-cli.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -134,14 +134,17 @@ function main(): void {
134134
const load = (ref: string): Loaded => {
135135
if (fs.existsSync(ref))
136136
return { summary: readLocalSummary(ref), label: path.basename(path.resolve(ref)) };
137-
const run = resolveRunRef(ref, allRuns());
137+
const summaries = new Map<string, PerfSummary>();
138+
const run = resolveRunRef(ref, allRuns(), (candidate) => {
139+
const summary = fetchSummary(options.source, candidate);
140+
if (summary) summaries.set(candidate.id, summary);
141+
return summary !== undefined;
142+
});
138143
if (!run)
139144
throw new Error(
140-
`No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref}.`,
145+
`No completed ${options.source.workflow} run on ${options.source.branch} matches ${ref} with a ${options.source.artifact} artifact.`,
141146
);
142-
const summary = fetchSummary(options.source, run);
143-
if (!summary) throw new Error(`Run ${run.id} has no ${options.source.artifact} artifact.`);
144-
return { summary, label: runLabel(run), run };
147+
return { summary: summaries.get(run.id)!, label: runLabel(run), run };
145148
};
146149
const loadRuns = (candidates: WorkflowRun[]): Loaded[] =>
147150
candidates.flatMap((run) => {

‎scripts/perf/config.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ test('a run measures at least three rounds, so every step can get a timing verdi
66
assert.equal(parseConfig(['--udid', 'sim', '--rounds', '3']).rounds, 3);
77
assert.throws(() => parseConfig(['--udid', 'sim', '--rounds', '2']), /--rounds must be .* >= 3/);
88
assert.throws(
9-
() => parseConfig(['--platform', 'android', '--serial', 'emulator-5554', '--n', '1']),
10-
/--n must be .* >= 3/,
9+
() => parseConfig(['--platform', 'android', '--serial', 'emulator-5554', '--rounds', '1']),
10+
/--rounds must be .* >= 3/,
1111
);
1212
});
1313

‎scripts/perf/config.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ export function parseConfig(argv: string[]): PerfConfig {
6666
cfg.platform = v;
6767
break;
6868
}
69-
case '--n':
7069
case '--rounds':
7170
cfg.rounds = readIntValue(argv, i++, a, MIN_MEASURED_ROUNDS);
7271
break;

‎scripts/perf/nightly-artifacts.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ test('resolves latest, run ids, and dates to the newest matching run', () => {
3030
assert.equal(resolveRunRef('2026-10-19', RUNS)?.id, '103');
3131
assert.equal(resolveRunRef('2026-10-18', RUNS), undefined);
3232
assert.throws(() => resolveRunRef('yesterday', RUNS), /Unknown run reference/);
33+
// A run without the platform artifact gives way to the newest run that has it.
34+
const withArtifact = (run: WorkflowRun) => run.id !== '104' && run.id !== '103';
35+
assert.equal(resolveRunRef('latest', RUNS, withArtifact)?.id, '102');
36+
assert.equal(resolveRunRef('2026-10-19', RUNS, withArtifact)?.id, '102');
37+
assert.equal(resolveRunRef('104', RUNS, withArtifact), undefined);
3338
});
3439

3540
const baselineIds = (

‎scripts/perf/nightly-artifacts.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,20 @@ function sortNewestFirst(runs: WorkflowRun[]): WorkflowRun[] {
8181

8282
/**
8383
* Resolves one run reference: `latest`, a numeric run id, or a `YYYY-MM-DD` date (the newest run
84-
* created that UTC day). Returns undefined when no listed run matches.
84+
* created that UTC day). Only runs that `accept` takes count, so a run that skipped a platform
85+
* does not hide an older run of that platform. Returns undefined when no listed run matches.
8586
*/
86-
export function resolveRunRef(ref: string, runs: WorkflowRun[]): WorkflowRun | undefined {
87+
export function resolveRunRef(
88+
ref: string,
89+
runs: WorkflowRun[],
90+
accept: (run: WorkflowRun) => boolean = () => true,
91+
): WorkflowRun | undefined {
8792
const newestFirst = sortNewestFirst(runs);
88-
if (ref === 'latest') return newestFirst[0];
89-
if (/^\d+$/.test(ref)) return newestFirst.find((run) => run.id === ref);
90-
if (DATE_REF.test(ref)) return newestFirst.find((run) => run.createdAt.slice(0, 10) === ref);
93+
const first = (match: (run: WorkflowRun) => boolean) =>
94+
newestFirst.find((run) => match(run) && accept(run));
95+
if (ref === 'latest') return first(() => true);
96+
if (/^\d+$/.test(ref)) return first((run) => run.id === ref);
97+
if (DATE_REF.test(ref)) return first((run) => run.createdAt.slice(0, 10) === ref);
9198
throw new Error(
9299
`Unknown run reference ${JSON.stringify(ref)}: use latest, a run id, a YYYY-MM-DD date, or a path.`,
93100
);

‎scripts/perf/suite-harness.test.ts‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import path from 'node:path';
44
import { mkdtempForTestSync } from '@agent-device/test-kit';
55
import { test, vi } from 'vitest';
66
import type { PerfConfig } from './config.ts';
7+
import type { PlatformStep } from './suite.ts';
78

89
// Stands in for the CLI: records each teardown call, and whether the logs were already saved,
910
// then appends to the daemon log and fails like a teardown that went wrong.
@@ -72,14 +73,26 @@ const args = process.argv.slice(2);
7273
const command = args.slice(0, args.indexOf('--state-dir')).join(' ');
7374
const session = process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION ?? 'default';
7475
fs.appendFileSync(process.env.PERF_FAKE_CALLS, command + ' session=' + session + '\\n');
76+
if (process.env.PERF_FAKE_FAIL_RESTORE && command === 'open app --serial emulator-5554' && session === 'default') {
77+
process.stdout.write(JSON.stringify({ success: false, error: { code: 'DEVICE_IN_USE', message: 'claimed' } }));
78+
process.exit(1);
79+
}
7580
const transport = session === '0' ? 'instrumentation' : 'persistent-session';
7681
const data = args[0] === 'snapshot'
7782
? { nodes: [{ depth: 0 }], androidSnapshot: { helperTransport: transport } }
7883
: {};
7984
process.stdout.write(JSON.stringify({ success: true, data }));
8085
`;
8186

82-
test('a step with daemon variables runs against a restarted daemon and restores it after', async () => {
87+
const ONE_SHOT: PlatformStep = {
88+
id: 'one-shot',
89+
command: 'snapshot',
90+
args: ['snapshot', '--raw'],
91+
daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' },
92+
expect: { transport: 'instrumentation' },
93+
};
94+
95+
async function runFakeDaemonSuite(steps: PlatformStep[]) {
8396
const root = mkdtempForTestSync('perf-daemon-env-');
8497
const fakeCli = path.join(root, 'fake-cli.mjs');
8598
fs.writeFileSync(fakeCli, FAKE_DAEMON_CLI);
@@ -88,8 +101,7 @@ test('a step with daemon variables runs against a restarted daemon and restores
88101
vi.stubEnv('PERF_FAKE_CALLS', calls);
89102
vi.resetModules();
90103
const { runSuite } = await import('./suite-harness.ts');
91-
92-
const [measurement] = await runSuite(
104+
const measurements = await runSuite(
93105
{
94106
platform: 'android',
95107
stateDir: path.join(root, 'state'),
@@ -104,21 +116,21 @@ test('a step with daemon variables runs against a restarted daemon and restores
104116
id: 'heavy-feed',
105117
title: 'heavy feed',
106118
setup: [],
107-
steps: [
108-
{
109-
id: 'one-shot',
110-
command: 'snapshot',
111-
args: ['snapshot', '--raw'],
112-
daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' },
113-
expect: { transport: 'instrumentation' },
114-
},
115-
],
119+
steps,
116120
},
117121
],
118122
);
123+
return { measurements, calls: fs.readFileSync(calls, 'utf8') };
124+
}
125+
126+
test('a step with daemon variables runs against a restarted daemon and restores it after', async () => {
127+
const {
128+
measurements: [measurement],
129+
calls,
130+
} = await runFakeDaemonSuite([ONE_SHOT]);
119131

120132
assert.equal(
121-
fs.readFileSync(calls, 'utf8'),
133+
calls,
122134
[
123135
'open app --relaunch --serial emulator-5554 session=default',
124136
'daemon stop session=default',
@@ -132,3 +144,13 @@ test('a step with daemon variables runs against a restarted daemon and restores
132144
assert.equal(measurement!.failures, 0);
133145
assert.equal(measurement!.samples[0]!.snapshot?.transport, 'instrumentation');
134146
});
147+
148+
test('a failed daemon restore skips the rest of the scenario as restore_failed', async () => {
149+
vi.stubEnv('PERF_FAKE_FAIL_RESTORE', '1');
150+
const { measurements } = await runFakeDaemonSuite([
151+
ONE_SHOT,
152+
{ id: 'back', command: 'back', args: ['back'] },
153+
]);
154+
assert.equal(measurements[0]!.failures, 0);
155+
assert.equal(measurements[1]!.samples[0]!.errorCode, 'restore_failed');
156+
});

‎scripts/perf/suite-harness.ts‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ export type SuiteContext = {
3232
daemonStuck?: boolean;
3333
// Variables that the running daemon was started with, set by a step with `daemonEnv`.
3434
daemonEnv?: Readonly<Record<string, string>>;
35+
// Set when the daemon restart after a `daemonEnv` step could not reopen the session. The rest
36+
// of the scenario is skipped, and the next scenario reopens the session.
37+
restoreFailed?: string;
3538
};
3639

3740
function log(msg: string): void {
@@ -156,7 +159,10 @@ function restartDaemon(
156159
const stop = invokeCli(['daemon', 'stop'], ['--state-dir', ctx.stateDir]);
157160
if (!stop.ok) return `daemon stop: ${errorOf(stop).errorMessage}`;
158161
ctx.daemonEnv = env;
159-
return sessionError(openSession(ctx, { relaunch: false }));
162+
const failure = sessionError(openSession(ctx, { relaunch: false }));
163+
// A fresh daemon holds no request from before the restart.
164+
if (!failure) ctx.daemonStuck = false;
165+
return failure;
160166
}
161167

162168
async function runTimedStep(
@@ -167,8 +173,7 @@ async function runTimedStep(
167173
): Promise<Sample> {
168174
const id = `${scenario.id}/${step.id}`;
169175
const restoreDaemon = (): void => {
170-
// The rest of the scenario cannot run without its session; the next scenario reopens it.
171-
if (step.daemonEnv && restartDaemon(ctx, undefined)) ctx.daemonStuck = true;
176+
if (step.daemonEnv) ctx.restoreFailed = restartDaemon(ctx, undefined);
172177
};
173178
const before =
174179
(step.daemonEnv ? restartDaemon(ctx, step.daemonEnv) : undefined) ?? untimed(ctx, step.before);
@@ -252,6 +257,7 @@ function stopStuckDaemon(ctx: SuiteContext): void {
252257
invokeCli(['daemon', 'stop', '--clean'], ['--state-dir', ctx.stateDir]);
253258
ctx.daemonStuck = false;
254259
ctx.daemonEnv = undefined;
260+
ctx.restoreFailed = undefined;
255261
}
256262

257263
/** Opens the session with device selectors; later calls address it by session name alone. */
@@ -261,7 +267,7 @@ function openSession(ctx: SuiteContext, options: { relaunch: boolean }): CliResu
261267
}
262268

263269
function establishSession(ctx: SuiteContext): CliResult {
264-
if (ctx.daemonStuck) stopStuckDaemon(ctx);
270+
if (ctx.daemonStuck || ctx.restoreFailed) stopStuckDaemon(ctx);
265271
return openSession(ctx, { relaunch: true });
266272
}
267273

@@ -277,8 +283,10 @@ export async function runSuite(
277283
log(`round ${round + 1}/${totalRounds}${measured ? '' : ' (warmup, dropped)'}`);
278284
let sessionFailure = sessionError(establishSession(ctx));
279285
for (const scenario of scenarios) {
280-
// A request that outlived its timeout still holds the session; reopen before the next scenario.
281-
if (ctx.daemonStuck) sessionFailure = sessionError(establishSession(ctx));
286+
// A request that outlived its timeout still holds the session, and a failed daemon restore
287+
// lost it; reopen before the next scenario.
288+
if (ctx.daemonStuck || ctx.restoreFailed)
289+
sessionFailure = sessionError(establishSession(ctx));
282290
log(` scenario ${scenario.id}`);
283291
const setupFailure = sessionFailure ?? untimed(ctx, scenario.setup);
284292
if (setupFailure) log(` ✗ setup failed: ${setupFailure}`);
@@ -288,12 +296,14 @@ export async function runSuite(
288296
const id = `${scenario.id}/${step.id}`;
289297
const skipped = setupFailure
290298
? { errorCode: 'setup_failed', errorMessage: setupFailure }
291-
: ctx.daemonStuck
292-
? {
293-
errorCode: 'daemon_stuck',
294-
errorMessage: 'an earlier request still holds the session',
295-
}
296-
: undefined;
299+
: ctx.restoreFailed
300+
? { errorCode: 'restore_failed', errorMessage: ctx.restoreFailed }
301+
: ctx.daemonStuck
302+
? {
303+
errorCode: 'daemon_stuck',
304+
errorMessage: 'an earlier request still holds the session',
305+
}
306+
: undefined;
297307
const sample: Sample = skipped
298308
? { round, ok: false, wallClockMs: 0, ...skipped }
299309
: await runTimedStep(scenario, step, ctx, round);

‎scripts/perf/suite.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,10 @@ export const SUITE: readonly SuiteScenario<SuiteStep>[] = [
130130
step('orientation-landscape', C.orientation, ['orientation', 'landscape-left']),
131131
step('orientation-portrait', C.orientation, ['orientation', 'portrait']),
132132
step('settings-appearance', C.settings, ['settings', 'appearance', 'light']),
133-
step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf']),
134-
step('clipboard-read', C.clipboard, ['clipboard', 'read']),
133+
step('clipboard-write', C.clipboard, ['clipboard', 'write', 'agent-device perf'], {
134+
only: 'ios',
135+
}),
136+
step('clipboard-read', C.clipboard, ['clipboard', 'read'], { only: 'ios' }),
135137
step('action-button', C.actionButton, ['action-button'], { only: 'ios' }),
136138
// `wake` is the one hardware key that leaves an awake device as it was.
137139
step('hardware-button', C.hardwareButton, ['hardware-button', 'wake'], { only: 'android' }),
@@ -386,6 +388,8 @@ export const SUITE: readonly SuiteScenario<SuiteStep>[] = [
386388
step('snapshot-raw-one-shot', C.snapshot, ['snapshot', '--raw'], {
387389
only: 'android',
388390
daemonEnv: { AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION: '0' },
391+
// The restarted daemon verifies the helper install on its first capture; keep that out.
392+
before: [['snapshot', '-i']],
389393
expect: { maxDepthAtLeast: 56, transport: 'instrumentation' },
390394
}),
391395
step('back', C.back, ['back']),
@@ -446,6 +450,8 @@ export const SUITE_EXCLUSIONS: Record<Platform, Partial<Record<PublicCommand, st
446450
[C.prepare]:
447451
'prepare has only the ios-runner action; the snapshot helper installs on first use.',
448452
[C.actionButton]: 'Android has no key event for the Action button.',
453+
[C.clipboard]:
454+
'The API 35 and 36 emulator images ship no shell clipboard service, so adb cannot read or write the clipboard.',
449455
[C.audio]:
450456
'Host audio capture uses macOS ScreenCaptureKit; the Linux emulator runner has none.',
451457
[C.hover]: 'The Android runtime refuses hover; it is a web pointer contract.',

0 commit comments

Comments
 (0)