Skip to content

Commit b0b469b

Browse files
thymikeeclaude
andcommitted
feat(perf): time every iOS command against the fixture app
pnpm perf --platform ios now drives the test app through seven scenarios that time each catalog command the iOS simulator serves, including the heavy feed and a raw capture with the XCTest tree backend pinned. The suite records a reason for every command it does not time, and a unit test keeps that partition complete. Each sample keeps the client daemon round trip and a phase profile from its --debug request log: runner commands, host process spawns, snapshot sources, retries, stabilization, and fallbacks. A run writes summary.json, report.md, and the raw daemon and session logs. The iOS Settings tour is removed; the Android tour stays as it was. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d17b8ea commit b0b469b

20 files changed

Lines changed: 1763 additions & 282 deletions

‎.github/workflows/perf-nightly.yml‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,24 +12,24 @@ on:
1212
rounds:
1313
description: 'Measured rounds per command (samples)'
1414
required: false
15-
default: '5'
15+
default: '3'
1616

1717
permissions:
1818
contents: read
19+
actions: read
1920

2021
concurrency:
2122
group: ci-${{ github.workflow }}-${{ github.ref }}
2223
cancel-in-progress: true
2324

2425
env:
25-
AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts'
26-
PERF_ROUNDS: ${{ github.event.inputs.rounds || '5' }}
26+
PERF_ROUNDS: ${{ github.event.inputs.rounds || '3' }}
2727

2828
jobs:
2929
perf-ios:
3030
name: iOS Command Perf
3131
runs-on: macos-26
32-
timeout-minutes: 80
32+
timeout-minutes: 100
3333
env:
3434
IOS_RUNTIME_VERSION: '26.2'
3535
AGENT_DEVICE_XCUITEST_ARCHS: 'arm64'
@@ -42,6 +42,15 @@ jobs:
4242

4343
- name: Setup toolchain
4444
uses: ./.github/actions/setup-node-pnpm
45+
with:
46+
cache-dependency-path: |
47+
pnpm-lock.yaml
48+
examples/test-app/pnpm-lock.yaml
49+
50+
- name: Build the shared CLI
51+
uses: ./.github/actions/run-gate
52+
with:
53+
gate: build
4554

4655
- name: Restore and build iOS XCTest runner
4756
uses: ./.github/actions/setup-apple-runner-build
@@ -54,22 +63,35 @@ jobs:
5463
xcuitest-destination: generic/platform=iOS Simulator
5564

5665
- name: Boot iOS test simulator
66+
id: ios-simulator
5767
uses: ./.github/actions/boot-ios-test-simulator
5868
with:
5969
runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
6070
preferred-device-name: iPhone 17 Pro
6171

72+
- name: Install the fixture app
73+
id: fixture-app
74+
uses: ./.github/actions/setup-fixture-app
75+
6276
- name: Preflight iOS runner through public CLI
77+
env:
78+
UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
6379
run: |
80+
# The keyboard must stay expanded for keyboard dismiss to find its Done key.
81+
xcrun simctl spawn "$UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false
6482
pnpm clean:daemon
65-
node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS"
83+
node bin/agent-device.mjs prepare ios-runner --platform ios --udid "$UDID" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS"
6684
pnpm clean:daemon
6785
68-
- name: Run iOS command perf benchmark
86+
- name: Run iOS command perf suite
87+
env:
88+
UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
89+
APP_ID: ${{ steps.fixture-app.outputs.app-id }}
90+
APP_PATH: ${{ steps.fixture-app.outputs.app-path }}
6991
run: |
70-
pnpm perf --platform ios \
71-
--device "iPhone 17 Pro" \
72-
--n "$PERF_ROUNDS" --warmup 1 \
92+
pnpm perf --platform ios --udid "$UDID" \
93+
--app-id "$APP_ID" --app-path "$APP_PATH" \
94+
--rounds "$PERF_ROUNDS" --warmup 1 \
7395
--out-dir "$GITHUB_WORKSPACE/perf-results"
7496
7597
- name: Upload iOS perf report
@@ -84,6 +106,8 @@ jobs:
84106
name: Android Command Perf
85107
runs-on: ubuntu-latest
86108
timeout-minutes: 80
109+
env:
110+
AGENT_DEVICE_PERF_CLI: '--experimental-strip-types src/bin.ts'
87111
steps:
88112
- name: Checkout
89113
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

‎scripts/perf/README.md‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Command performance suite
2+
3+
`pnpm perf --platform ios` times every interaction command against the fixture app (`examples/test-app`).
4+
The steps are in `ios-suite.ts`. Each step belongs to one scenario.
5+
Each scenario starts from an untimed setup in every round.
6+
The heavy-feed scenario uses a screen whose accessibility tree is deeper than 65 levels.
7+
The suite records a reason for each catalog command that it does not time. `ios-suite.test.ts` keeps that list complete.
8+
`pnpm perf --platform android` runs the older report-only Settings tour.
9+
10+
## Run locally
11+
12+
1. Run `pnpm build`. The suite drives the built CLI and the built SDK.
13+
2. Use your own booted simulator. Install a current Release fixture on it. For the cached CI build and the JavaScript repack, see `examples/test-app/README.md`.
14+
3. Set `AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH` to a path in your worktree.
15+
4. Run the suite:
16+
17+
```sh
18+
pnpm perf --platform ios --udid <udid> --rounds 3 --warmup 1 --out-dir .tmp/perf/local
19+
```
20+
21+
`--scenarios lifecycle,heavy-feed` runs a subset. `--app-id` changes the fixture bundle id.
22+
The suite uses its own temporary state directory and daemon. It stops them at the end.
23+
24+
## Output
25+
26+
- `summary.json`: the machine-readable run (`agent-device-perf-summary/v1`). It holds the metadata, the coverage, and every sample.
27+
- `report.md`: one table per scenario, the backends and fallbacks, the failures, and the coverage.
28+
- `raw/state/`: the daemon log, and for each session `runner.log`, `events.ndjson`, and the `--debug` request log of each request. `requestId` in a sample names its request log.
29+
- `raw/failures/`: the JSON response and stderr of each failed sample.
30+
31+
## Metrics
32+
33+
All times are milliseconds. Statistics use only the successful samples of a step.
34+
35+
| Metric | Meaning |
36+
| --- | --- |
37+
| `wall` | The CLI process time, from spawn to exit. |
38+
| `daemon` | The daemon request round trip that the client measures (`daemon_request`). For the pinned-backend capture, the span of its request log. Comparisons use this metric. |
39+
| `mean ± sd`, `cv` | The mean, the sample standard deviation, and the standard deviation divided by the mean. |
40+
| `median`, `p95`, `min`, `max` | The median (the mean of the two middle values for an even count), the nearest-rank 95th percentile, and the extremes. |
41+
| `runner` | The median sum of XCTest runner command times in one request. |
42+
| `spawns` | The median count and time of host processes that one request started. |
43+
| `snapshot source` | The median time of host accessibility-bridge acquisitions in one request. |
44+
| `settle wait` | The median `settle.waitedMs` that a `--settle` response reports. |
45+
| `stabilization` | The median time of post-gesture stabilization phases in one request. |
46+
| `retries` | The failed attempts of retried operations in all successful samples. |
47+
| `top phases` | The five phases with the largest median time per request. |
48+
| `producer` | The number of samples that each snapshot producer answered (`simulator-ax-bridge`, `apple-runner`). |
49+
| backends | The `snapshotQuality` backend, state, and reason code, for example `private-ax/recovered/ax-rejected`. |
50+
| fallbacks | Degraded paths: snapshot-source errors, truncated recovery, and warn-level phases. |
51+
| `fail` | The number of failed samples. `setup_failed` and `before_failed` mean that the untimed preparation failed. `daemon_stuck` means that an earlier request outlived its timeout; the suite restarts the daemon before the next scenario. `expectation_failed` means that the screen no longer has the shape that the step requires, for example the heavy-feed depth. |

‎scripts/perf/config.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,16 @@ export type PerfConfig = {
1313
warmup: number; // leading rounds dropped from stats
1414
keepArtifacts: boolean; // keep temp state dir + leave device booted
1515
outDir: string;
16-
udid?: string; // iOS device override (UDID)
17-
device?: string; // device override by name (e.g. "iPhone 17 Pro"); preferred over udid
16+
udid?: string; // iOS simulator UDID (required for iOS)
17+
device?: string; // Android AVD name for boot
1818
serial?: string; // Android device override
19+
appId: string; // iOS fixture bundle id
20+
appPath?: string; // iOS fixture bundle path, recorded in the run metadata
21+
scenarios?: string[]; // iOS scenario ids; all when omitted
1922
};
2023

24+
export const DEFAULT_IOS_APP_ID = 'com.callstack.agentdevicelab';
25+
2126
// How to invoke the CLI. Defaults to the built dist binary (bin/agent-device.mjs).
2227
// Set AGENT_DEVICE_PERF_CLI to run from source instead, e.g. on CI:
2328
// AGENT_DEVICE_PERF_CLI="--experimental-strip-types src/bin.ts"
@@ -50,10 +55,11 @@ function readIntValue(argv: string[], i: number, flag: string, min: number): num
5055
export function parseConfig(argv: string[]): PerfConfig {
5156
const cfg: PerfConfig = {
5257
platform: 'ios',
53-
rounds: 5,
58+
rounds: 3,
5459
warmup: 1,
5560
keepArtifacts: false,
56-
outDir: DEFAULT_OUT_DIR,
61+
outDir: path.join(DEFAULT_OUT_DIR, new Date().toISOString().replaceAll(/[:.]/g, '-')),
62+
appId: DEFAULT_IOS_APP_ID,
5763
};
5864
for (let i = 0; i < argv.length; i++) {
5965
const a = argv[i];
@@ -86,9 +92,22 @@ export function parseConfig(argv: string[]): PerfConfig {
8692
case '--serial':
8793
cfg.serial = readValue(argv, i++, a);
8894
break;
95+
case '--app-id':
96+
cfg.appId = readValue(argv, i++, a);
97+
break;
98+
case '--app-path':
99+
cfg.appPath = path.resolve(readValue(argv, i++, a));
100+
break;
101+
case '--scenarios':
102+
cfg.scenarios = readValue(argv, i++, a)
103+
.split(',')
104+
.map((id) => id.trim())
105+
.filter(Boolean);
106+
break;
89107
default:
90108
throw new Error(`Unknown flag: ${a}`);
91109
}
92110
}
111+
if (cfg.platform === 'ios' && !cfg.udid) throw new Error('--udid is required for --platform ios');
93112
return cfg;
94113
}

‎scripts/perf/forced-backend.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import path from 'node:path';
2+
import { performance } from 'node:perf_hooks';
3+
import { pathToFileURL } from 'node:url';
4+
import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client';
5+
import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution';
6+
import { createRequestId } from '@agent-device/host-kit/diagnostics';
7+
import { captureHostEnvironment } from '@agent-device/host-kit/process';
8+
import type { SnapshotPreferredBackend } from '@agent-device/kernel/snapshot';
9+
import { readDaemonInfo } from '../../src/daemon-client/daemon-client-metadata.ts';
10+
import { sendRequest } from '../../src/daemon-client/daemon-client-transport.ts';
11+
import { REPO_ROOT } from './config.ts';
12+
import type { CliResult } from './types.ts';
13+
14+
type AgentDeviceSdk = typeof import('../../src/sdk/index.ts');
15+
16+
const FORCED_CAPTURE_TIMEOUT_MS = 90_000;
17+
18+
/**
19+
* Captures one raw snapshot with the internal backend pin that the CLI does not expose. The
20+
* transport talks to the daemon that the built CLI already started and never starts or replaces
21+
* it, so the measured daemon stays the daemon under test.
22+
*/
23+
export async function captureWithForcedBackend(options: {
24+
stateDir: string;
25+
session: string;
26+
udid: string;
27+
backend: SnapshotPreferredBackend;
28+
}): Promise<{ result: CliResult; requestId: string }> {
29+
const requestId = createRequestId();
30+
const sdk = (await import(
31+
pathToFileURL(path.join(REPO_ROOT, 'dist/src/index.js')).href
32+
)) as AgentDeviceSdk;
33+
const client = sdk.createAgentDeviceClient(
34+
{ session: options.session, stateDir: options.stateDir },
35+
{ transport: forcedBackendTransport(options.backend, requestId) },
36+
);
37+
const t0 = performance.now();
38+
try {
39+
const snapshot = await client.capture.snapshot({
40+
raw: true,
41+
platform: 'ios',
42+
udid: options.udid,
43+
timeoutMs: FORCED_CAPTURE_TIMEOUT_MS,
44+
});
45+
const json = { success: true, data: snapshot };
46+
const wallClockMs = performance.now() - t0;
47+
return {
48+
result: { exitCode: 0, wallClockMs, stdout: '', stderr: '', json, ok: true },
49+
requestId,
50+
};
51+
} catch (error) {
52+
const normalized = sdk.normalizeAgentDeviceError(error);
53+
const json = { success: false, error: normalized };
54+
const wallClockMs = performance.now() - t0;
55+
return {
56+
result: { exitCode: 1, wallClockMs, stdout: '', stderr: normalized.message, json, ok: false },
57+
requestId,
58+
};
59+
}
60+
}
61+
62+
function forcedBackendTransport(
63+
backend: SnapshotPreferredBackend,
64+
requestId: string,
65+
): AgentDeviceDaemonTransport {
66+
const host = captureHostEnvironment(process);
67+
return async (request, context) => {
68+
const stateDir = request.flags?.stateDir;
69+
if (request.session === undefined || typeof stateDir !== 'string') {
70+
throw new Error('A forced-backend capture requires an explicit session and state dir.');
71+
}
72+
const paths = resolveDaemonPaths(stateDir, { env: process.env });
73+
const info = readDaemonInfo(paths.infoPath);
74+
if (!info) throw new Error(`No daemon is running for state dir ${stateDir}.`);
75+
return await sendRequest(
76+
info,
77+
{
78+
...request,
79+
session: request.session,
80+
token: info.token,
81+
flags: { ...request.flags, snapshotPreferredBackend: backend },
82+
meta: { ...request.meta, debug: true, requestId },
83+
},
84+
'auto',
85+
paths,
86+
FORCED_CAPTURE_TIMEOUT_MS,
87+
{ host, signal: context?.signal, timeoutRecovery: 'none' },
88+
);
89+
};
90+
}

0 commit comments

Comments
 (0)