Skip to content

Commit 9bf62c7

Browse files
committed
fix(settle): resolve the default quiet window from the capture it settles
--settle asked every capture the same 500ms, and the iOS simulator snapshot bridge answers well inside it: on the app's own screens (iPhone 17 sim) a text entry settled at 680ms instead of 836ms and a tap-to-detail press at 619ms instead of 813ms, each at two captures instead of three, with no later change for the text entry at either window. The window is now resolved from the baseline capture's producer rather than being one constant: the bridge gets 300ms, and the Apple runner's own read, the Android helper, and any capture arriving without provenance keep 500ms, because none of them carry the measurement. The baseline carries the channel-and-producer pair it arrived on, so the decision reads a fact the capture published instead of rediscovering the path from node shape. --settle-quiet still outranks both, and `wait stable` keeps the general default of its own.
1 parent 70e3e8f commit 9bf62c7

9 files changed

Lines changed: 216 additions & 7 deletions

File tree

‎packages/command-registry/src/flag-definitions-action.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
211211
min: 0,
212212
usageLabel: '--settle-quiet <ms>',
213213
usageDescription:
214-
'Settle: quiet window the UI must hold to count as settled (default 500ms). On iOS, dismissing a presented surface (alert or sheet) holds at least 1500ms, within the --timeout settle deadline',
214+
'Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms on an iOS simulator interactive tree). On iOS, dismissing a presented surface (alert or sheet) holds at least 1500ms, within the --timeout settle deadline',
215215
projectConfig: true,
216216
recorded: false,
217217
},

‎packages/contracts/src/client-gesture.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ export type RepeatedPressOptions = {
2525
* Opt-in (#1101): after the action, wait for the UI to go quiet and return the
2626
* settled diff vs the pre-action tree (`settle` on the result) in the same
2727
* response. Best-effort — never fails the action. `settleQuietMs` tunes the
28-
* quiet window (default 500ms); `timeoutMs` bounds the settle wait (default
28+
* quiet window (default 500ms, or 300ms after an action on a healthy iOS
29+
* interactive tree); `timeoutMs` bounds the settle wait (default
2930
* 10s) when `settle` is true. A bare `timeoutMs` without `settle` is ignored
3031
* for compatibility; `settleQuietMs` still requires `settle`.
3132
*/

‎packages/contracts/src/interaction.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1-
import type { Point, SnapshotKeyboardBandFact, SnapshotNode } from '@agent-device/kernel/snapshot';
1+
import type {
2+
Point,
3+
SnapshotKeyboardBandFact,
4+
SnapshotNode,
5+
SnapshotStateProvenance,
6+
} from '@agent-device/kernel/snapshot';
27
import type { ResponseCost } from '@agent-device/kernel/contracts';
38
import type { ClickButton } from './click-button.ts';
49
import type { FillUnconfirmedVerification } from './fill-evidence.ts';
@@ -148,6 +153,12 @@ export type SurfaceScopedNodes = {
148153
nodes: SnapshotNode[];
149154
/** Bundle id of the in-place iOS system surface; absent for ordinary app content. */
150155
surfaceBundleId?: string;
156+
/**
157+
* The channel and producer that delivered `nodes`, when the capture declared them. Consumers that
158+
* key on the CAPTURE PATH rather than on the tree — the post-action settle choosing its default
159+
* quiet window (#3470 follow-up) — read this instead of rediscovering the path from node shape.
160+
*/
161+
captureSource?: SnapshotStateProvenance;
151162
};
152163

153164
export type ResolvedInteractionTarget =

‎src/commands/interaction/runtime/post-action-surface.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { SnapshotState } from '@agent-device/kernel/snapshot';
1+
import { snapshotStateProvenance, type SnapshotState } from '@agent-device/kernel/snapshot';
22
import { summarizeAxEvidence } from '@agent-device/capture-kit/snapshot-evidence';
33
import type {
44
InteractionEvidence,
@@ -24,11 +24,13 @@ import {
2424

2525
/** Mints the one carried value from a capture: the nodes together with the surface they describe. */
2626
export function surfaceScopedNodes(snapshot: SnapshotState): SurfaceScopedNodes {
27+
const captureSource = snapshotStateProvenance(snapshot);
2728
return {
2829
nodes: snapshot.nodes,
2930
...(snapshot.iosSystemSurfaceBundleId
3031
? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId }
3132
: {}),
33+
...(captureSource.backend ? { captureSource } : {}),
3234
};
3335
}
3436

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import type { SnapshotProvenance, SnapshotState } from '@agent-device/kernel/snapshot';
4+
import { ref } from './selector-read-utils.ts';
5+
import {
6+
buttonSnapshot,
7+
createSettleDevice,
8+
welcomeSnapshot,
9+
} from './__tests__/settle-device-fixtures.ts';
10+
11+
// The `--settle` DEFAULT quiet window comes from the capture the action was performed against
12+
// (#3470 follow-up): the iOS simulator snapshot bridge goes quiet well inside the general 500 ms
13+
// default, and only that read carries the measurement. A window the caller asks for with
14+
// `--settle-quiet` always outranks it, and every other capture — the Apple runner's own tree read,
15+
// the Android helper, a capture that arrived without provenance — keeps the general default.
16+
//
17+
// Budgets are injected (fake clock) — no real waiting.
18+
19+
function withProvenance<P extends SnapshotProvenance>(state: SnapshotState, provenance: P) {
20+
return { ...state, ...provenance };
21+
}
22+
23+
function pressWithBaseline(
24+
baseline: SnapshotState,
25+
settle?: { quietMs?: number; timeoutMs?: number },
26+
) {
27+
let captures = 0;
28+
const device = createSettleDevice({
29+
stored: baseline,
30+
captureSnapshot: () => {
31+
captures += 1;
32+
// Capture 1 is the selector resolution (the pre-action tree); captures 2+ are the loop.
33+
return { snapshot: captures === 1 ? baseline : welcomeSnapshot() };
34+
},
35+
});
36+
return device.interactions.press(ref('@e1'), { session: 'default', settle: settle ?? {} });
37+
}
38+
39+
const BRIDGE = { backend: 'xctest', producer: 'simulator-ax-bridge' } as const;
40+
41+
test('a simulator snapshot-bridge baseline settles on the shorter default window', async () => {
42+
const result = await pressWithBaseline(withProvenance(buttonSnapshot(), BRIDGE));
43+
44+
const settle = result.settle;
45+
assert.ok(settle);
46+
assert.equal(settle.settled, true);
47+
assert.equal(settle.quietMs, 300);
48+
});
49+
50+
test('an explicit --settle-quiet outranks the per-capture default window', async () => {
51+
const result = await pressWithBaseline(withProvenance(buttonSnapshot(), BRIDGE), {
52+
quietMs: 700,
53+
timeoutMs: 5_000,
54+
});
55+
56+
const settle = result.settle;
57+
assert.ok(settle);
58+
assert.equal(settle.quietMs, 700);
59+
});
60+
61+
// Each of these is a capture path the 300 ms default was NOT measured on, so shortening it there
62+
// would be an unmeasured change of behaviour rather than the measured one. The Apple runner shares
63+
// the `xctest` CHANNEL with the bridge, which is why the rule keys on the producer.
64+
const UNMEASURED_CAPTURES: Array<[string, SnapshotProvenance?]> = [
65+
[
66+
'the Apple runner tree read (same channel, different producer)',
67+
{ backend: 'xctest', producer: 'apple-runner' },
68+
],
69+
['an Appium-sourced tree on the same channel', { backend: 'xctest', producer: 'appium-source' }],
70+
['the Android helper', { backend: 'android', producer: 'android-uiautomator' }],
71+
['a capture that declares no provenance at all', undefined],
72+
];
73+
74+
for (const [label, provenance] of UNMEASURED_CAPTURES) {
75+
test(`${label} keeps the 500ms default window`, async () => {
76+
const result = await pressWithBaseline(
77+
provenance ? withProvenance(buttonSnapshot(), provenance) : buttonSnapshot(),
78+
);
79+
80+
const settle = result.settle;
81+
assert.ok(settle);
82+
assert.equal(settle.quietMs, 500);
83+
});
84+
}
85+
86+
test('the shorter default window never shortens a presented-surface dismissal confirmation', async () => {
87+
// The pre-action tree is a presented surface (a tiny tree over app content), so the dismissal
88+
// confirmation owns the wait: the 300 ms default must still hold for its 1.5 s floor (#3470).
89+
let captures = 0;
90+
const presented = withProvenance(buttonSnapshot(), BRIDGE);
91+
const device = createSettleDevice({
92+
stored: presented,
93+
captureSnapshot: () => {
94+
captures += 1;
95+
return { snapshot: captures === 1 ? presented : welcomeSnapshot() };
96+
},
97+
});
98+
99+
const result = await device.interactions.press(ref('@e1'), {
100+
session: 'default',
101+
settle: { timeoutMs: 5_000 },
102+
});
103+
104+
const settle = result.settle;
105+
assert.ok(settle);
106+
assert.equal(settle.settled, true);
107+
assert.equal(settle.quietMs, 300);
108+
assert.ok(
109+
settle.waitedMs >= 1_500,
110+
`dismissal confirmation did not outlast the shorter default: waitedMs=${settle.waitedMs}`,
111+
);
112+
});

‎src/commands/interaction/runtime/settle.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ async function settleAfterAction(
126126
options: CommandContext,
127127
params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point },
128128
): Promise<SettleOutcome> {
129-
const quietMs = params.quietMs ?? DEFAULT_STABLE_QUIET_MS;
129+
const quietMs = params.quietMs ?? defaultSettleQuietMs(params.baseline);
130130
const timeoutMs = params.timeoutMs ?? DEFAULT_STABLE_TIMEOUT_MS;
131131
const base: SettleObservation = { settled: false, waitedMs: 0, captures: 0, quietMs, timeoutMs };
132132
try {
@@ -149,6 +149,34 @@ async function settleAfterAction(
149149
}
150150
}
151151

152+
/**
153+
* The default quiet window for the post-action settle, resolved from the capture the action was
154+
* performed against (#3470 follow-up). The iOS simulator snapshot bridge answers well inside the
155+
* general 500 ms default, so the loop spends a whole extra capture holding a window the tree has
156+
* already satisfied. Measured on the app's own screens (iPhone 17 simulator), one run at each window
157+
* per rep with the order alternating so host load falls on both arms alike: a text entry into the
158+
* Catalog search settled at 664 ms instead of 859 ms, and a press on a product's Add to cart button
159+
* at 647 ms instead of 844 ms, each at two captures instead of three. The tree behind the answer
160+
* stayed correct: a `diff snapshot -i` two seconds after the settled response found no further change
161+
* for the text entry at either window (0/6 both), and for the press it found one in three of six runs
162+
* at either window — that screen keeps animating after the press whichever window is asked for, so the
163+
* shorter window changes the wait, not the verdict's accuracy.
164+
*
165+
* Every other capture keeps {@link DEFAULT_STABLE_QUIET_MS}: the Apple runner's own read, the
166+
* Android helper, and any capture arriving without provenance, none of which carry that measurement.
167+
* An explicit `--settle-quiet` outranks both, and `wait stable` keeps the general default of its
168+
* own — only the post-action settle resolves its window from the baseline it diffs.
169+
*/
170+
const SIMULATOR_BRIDGE_SETTLE_QUIET_MS = 300;
171+
172+
function defaultSettleQuietMs(baseline: SurfaceScopedNodes): number {
173+
// Keyed on the producer, not the channel: the bridge and the Apple runner share the `xctest`
174+
// channel and read at different costs, and only the bridge is the path measured above.
175+
return baseline.captureSource?.producer === 'simulator-ax-bridge'
176+
? SIMULATOR_BRIDGE_SETTLE_QUIET_MS
177+
: DEFAULT_STABLE_QUIET_MS;
178+
}
179+
152180
/** Turns a finished stable-capture loop into the settled observation payload. */
153181
async function readSettledOutcome(
154182
runtime: AgentDeviceRuntime,

‎src/commands/post-action-observation-grammar.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ const settleFields = () => ({
3434
'After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response. Best-effort; never fails the action.',
3535
),
3636
settleQuietMs: integerField(
37-
'Settle: quiet window in milliseconds (default 500). On iOS, dismissing a presented surface (alert or sheet) holds at least 1500, within the timeoutMs settle deadline.',
37+
'Settle: quiet window in milliseconds (default 500; 300 on an iOS simulator interactive tree). On iOS, dismissing a presented surface (alert or sheet) holds at least 1500, within the timeoutMs settle deadline.',
3838
{ min: 0 },
3939
),
4040
timeoutMs: integerField('Settle: wait deadline in milliseconds (default 10000).', { min: 1 }),

‎src/daemon/__tests__/generic-settle.test.ts‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,13 +150,14 @@ function seedSession(
150150
type GenericSettleTree = {
151151
nodes: RawSnapshotNode[];
152152
systemSurface?: IosSystemSurfaceProvenance;
153+
producer?: 'apple-runner' | 'simulator-ax-bridge';
153154
};
154155

155156
function snapshotPayload(tree: GenericSettleTree) {
156157
return {
157158
nodes: tree.nodes,
158159
backend: 'xctest' as const,
159-
producer: 'apple-runner' as const,
160+
producer: tree.producer ?? ('apple-runner' as const),
160161
...(tree.systemSurface ? { systemSurface: tree.systemSurface } : {}),
161162
};
162163
}
@@ -712,3 +713,54 @@ for (const change of ['rebuild', 'retire'] as const) {
712713
}
713714
});
714715
}
716+
717+
// The generic route builds its own baseline from the session's stored tree, so the capture-source
718+
// fact the default quiet window keys on (#3470 follow-up) has to be threaded through that
719+
// construction too; the targeted route asserts the window itself in
720+
// `src/commands/interaction/runtime/settle-default-quiet.test.ts`.
721+
test('scroll --settle without --settle-quiet uses the shorter default on a simulator bridge tree', async () => {
722+
const sessionStore = makeSessionStore();
723+
const sessionName = 'generic-settle-default-quiet';
724+
const session = seedSession(sessionName, sessionStore, {
725+
nodes: BEFORE_NODES,
726+
producer: 'simulator-ax-bridge',
727+
});
728+
mockCommandDispatch([AFTER_NODES]);
729+
730+
const response = await dispatchGeneric({
731+
sessionName,
732+
sessionStore,
733+
session,
734+
command: 'scroll',
735+
positionals: ['down'],
736+
// No settleQuietMs: the route's own default window is what is under test.
737+
flags: { settle: true, timeoutMs: 2_000 },
738+
});
739+
740+
const settle = expectOkData(response).settle as SettlePayload;
741+
expect(settle.settled).toBe(true);
742+
expect(settle.quietMs).toBe(300);
743+
});
744+
745+
test('scroll --settle keeps the 500ms default when the stored tree came from the Apple runner', async () => {
746+
const sessionStore = makeSessionStore();
747+
const sessionName = 'generic-settle-default-quiet-runner';
748+
const session = seedSession(sessionName, sessionStore, {
749+
nodes: BEFORE_NODES,
750+
producer: 'apple-runner',
751+
});
752+
mockCommandDispatch([AFTER_NODES]);
753+
754+
const response = await dispatchGeneric({
755+
sessionName,
756+
sessionStore,
757+
session,
758+
command: 'scroll',
759+
positionals: ['down'],
760+
flags: { settle: true, timeoutMs: 2_000 },
761+
});
762+
763+
const settle = expectOkData(response).settle as SettlePayload;
764+
expect(settle.settled).toBe(true);
765+
expect(settle.quietMs).toBe(500);
766+
});

‎src/daemon/generic-settle.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type { CommandFlags } from '@agent-device/contracts/command';
22
import type { CreateDaemonCommandSurface } from '@agent-device/contracts/daemon-command-surface';
3+
import { snapshotStateProvenance } from '@agent-device/kernel/snapshot';
34
import type {
45
SettleObservation,
56
SettleParams,
@@ -82,11 +83,13 @@ export function planGenericSettleObservation(
8283
const settle = readSettleRequest(params.flags);
8384
if (!settle) return {};
8485
const snapshot = params.session.snapshot;
86+
const captureSource = snapshot ? snapshotStateProvenance(snapshot) : {};
8587
const baseline: SurfaceScopedNodes = {
8688
nodes: snapshot?.nodes ?? [],
8789
...(snapshot?.iosSystemSurfaceBundleId
8890
? { surfaceBundleId: snapshot.iosSystemSurfaceBundleId }
8991
: {}),
92+
...(captureSource.backend ? { captureSource } : {}),
9093
};
9194
return { observe: async () => await observeSettled(params, settle, baseline) };
9295
}

0 commit comments

Comments
 (0)