Skip to content

Commit e5bba1b

Browse files
committed
fix(daemon): the interaction outcome weighs the states stateMarkers prints, and a scroll's container check matches content by identity
Review follow-up. The interaction surface key folded enabled and selected but not checked, so a tap whose only effect was a toggle read as a no-op on the outcome lane while the diff called it a change, and a no-change retry would tap the switch straight back. The key now spreads stateMarkers, so the outcome lane, the unchanged-snapshot comparison, and the diff weigh one list. With checked in the key, a flip inside a scroll container beside an unrelated change elsewhere (the status clock) read as content moving within the container. discriminatingSurfaceChangedWithinRect now matches entries on the flip-tolerant identity where one exists, told apart by document order when repeated, so a state flip at the same rect is not movement. Tests: a checked-only flip is a change on the outcome lane; a flip at the same rect is not movement and a moved row is; the scroll claim is withheld for a flipped toggle inside the container; checked in both answers invalidates snapshot reuse and survives the selector digest.
1 parent d2d4d35 commit e5bba1b

6 files changed

Lines changed: 125 additions & 15 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@
99
node Android reports as checkable, with both answers, so an unchecked switch reads `false`, a
1010
node that cannot be checked reads nothing, and so does a helper older than the attribute. The
1111
parser, the Android hierarchy node, and the published snapshot node carry it through to
12-
`get attrs` and to the `[checked]` and `[unchecked]` markers in snapshot text; `--settle` and
13-
`diff` compare it, so a toggle tap reads as a changed pair.
12+
`get attrs` and to the `[checked]` and `[unchecked]` markers in snapshot text; `--settle`, `diff`,
13+
and the interaction outcome weigh the same states, so a toggle tap reads as a changed pair rather
14+
than a no-op to retry. A scroll's movement claim now matches content by identity inside the
15+
container, so a state flip the swipe caused is not movement.
1416
- Fixed (ios): `perf cpu profile report --kind xctrace` on Xcode 27 no longer fails with
1517
`Apple xctrace CPU report contained no samples` on a trace that holds thousands of samples. Xcode
1618
27 exports each `time-profile` sample stack as `<tagged-backtrace>` instead of `<backtrace>`, and

‎src/commands/capture/runtime/snapshot-unchanged.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ test.each<Partial<SnapshotNode>>([
7272
{ identifier: 'create' },
7373
{ enabled: false },
7474
{ selected: true },
75+
{ checked: true },
76+
{ checked: false },
7577
{ focused: true },
7678
{ hittable: false },
7779
{ bundleId: 'com.example.app' },

‎src/daemon/__tests__/interaction-outcome-policy.test.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,3 +467,48 @@ test('discriminatingSurfaceChangedWithinRect counts content appearing inside the
467467

468468
assert.equal(discriminatingSurfaceChangedWithinRect(before, after, LIST_RECT), true);
469469
});
470+
471+
// Android is the only producer of `checked`, and a tap whose only effect is a toggle changes nothing
472+
// else on a screen without a mirrored label. The outcome lane has to read the flip as a change, or a
473+
// no-change retry taps the switch straight back.
474+
test('classifyInteractionSurfaceChange reads a checked-only flip as a change', () => {
475+
const before = buildInteractionSurfaceSignature(makeToggleSnapshot(false).nodes);
476+
const after = buildInteractionSurfaceSignature(makeToggleSnapshot(true).nodes);
477+
478+
assert.equal(classifyInteractionSurfaceChange(before, after), 'changed');
479+
});
480+
481+
test('discriminatingSurfaceChangedWithinRect reads a flip at the same rect as no movement, and a moved row as movement', () => {
482+
const rect = { x: 0, y: 0, width: 390, height: 844 };
483+
const signature = (checked: boolean, y?: number) =>
484+
buildInteractionSurfaceSignature(makeToggleSnapshot(checked, y).nodes);
485+
486+
assert.equal(
487+
discriminatingSurfaceChangedWithinRect(signature(false), signature(true), rect),
488+
false,
489+
);
490+
assert.equal(
491+
discriminatingSurfaceChangedWithinRect(signature(false, 300), signature(false, 200), rect),
492+
true,
493+
);
494+
});
495+
496+
function makeToggleSnapshot(checked: boolean, y = 300): SnapshotState {
497+
const base = makeSnapshot('Inbox');
498+
return {
499+
...base,
500+
nodes: [
501+
...base.nodes,
502+
{
503+
ref: 'e3',
504+
index: 2,
505+
parentIndex: 0,
506+
type: 'android.widget.Switch',
507+
identifier: 'wifi-switch',
508+
label: 'Wi-Fi switch',
509+
checked,
510+
rect: { x: 300, y, width: 60, height: 40 },
511+
},
512+
],
513+
};
514+
}

‎src/daemon/__tests__/response-views.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,18 @@ test('attrs digest keeps explicit false/zero/empty field facts; unavailable ones
335335
}
336336
});
337337

338+
test('attrs digest carries both checked answers, and none for a node that cannot be checked', () => {
339+
for (const checked of [true, false]) {
340+
const digest = getView!({ ref: 'e7', node: { ...MATCHED_NODE, checked } }, 'digest');
341+
expect(digest.node).toEqual({ ...COMPACT_NODE, checked });
342+
}
343+
const plain = getView!({ ref: 'e7', node: MATCHED_NODE }, 'digest').node as Record<
344+
string,
345+
unknown
346+
>;
347+
expect('checked' in plain).toBe(false);
348+
});
349+
338350
test('find/get default and full return today’s shape unchanged (same reference)', () => {
339351
const data: DaemonResponseData = { ref: '@e7', text: 'Sign in', node: MATCHED_NODE };
340352
expect(findView!(data, 'default')).toBe(data);

‎src/daemon/__tests__/scroll-movement.test.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,34 @@ test('a difference outside the scrolled container does not buy the movement clai
388388
assert.equal(spy.calls(), 1);
389389
});
390390

391+
// Android reports `checked`, and a swipe that starts on a switch can flip it. On its own the flip is
392+
// identity-invariant and reads as an unchanged surface. Beside a change elsewhere, such as the status
393+
// clock ticking, the pair reads as changed, and the within-container check then sees a switch whose
394+
// key differs at the same rect: a state flip, not the list moving, so it buys no movement claim.
395+
test('a toggle the swipe flipped inside the container does not buy the movement claim', async () => {
396+
const chrome = {
397+
type: 'Image',
398+
identifier: 'status-clock',
399+
label: '2:40',
400+
rect: { x: 20, y: 20, width: 60, height: 20 },
401+
} as SnapshotNode;
402+
const toggle = (checked: boolean) =>
403+
({
404+
type: 'android.widget.Switch',
405+
identifier: 'wifi-switch',
406+
label: 'Wi-Fi switch',
407+
checked,
408+
rect: { x: 300, y: 320, width: 60, height: 40 },
409+
}) as SnapshotNode;
410+
const { observation } = observe({
411+
baseline: baselineOf([...screen(0), chrome, toggle(false)]),
412+
screens: [[...screen(0), { ...chrome, label: '2:41' } as SnapshotNode, toggle(true)]],
413+
});
414+
415+
assert.equal(await observation, 'unobserved');
416+
assertWithheld('change-outside-container');
417+
});
418+
391419
test('a surface with no container to confine the claim to keeps the whole-surface answer', async () => {
392420
const rowsOnly = (offset: number) => screen(offset).filter((node) => node.type !== 'ScrollView');
393421
const { observation } = observe({

‎src/daemon/interaction-outcome-policy.ts‎

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type { CommandFlags } from '@agent-device/contracts/command';
22
import { isMobilePlatform } from '@agent-device/kernel/device';
33
import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
44
import { collectKeyboardChromeRefs } from '@agent-device/capture-kit/snapshot-chrome';
5+
import { stateMarkers } from '@agent-device/capture-kit/snapshot-lines';
56
import { emitDiagnostic } from '@agent-device/host-kit/diagnostics';
67
import { isViewportRootNode } from '@agent-device/contracts/snapshot';
78
import { contextFromFlags, type DaemonCommandContext } from './context.ts';
@@ -465,8 +466,11 @@ export function summarizeDiscriminatingSurfaceDivergence(
465466
}
466467

467468
/**
468-
* Whether the DISCRIMINATING entries inside `rect` differ across a gesture, on the same key-matched,
469-
* rect-tolerant view `haveIdenticalDiscriminatingSurfaces` uses — restricted to one region.
469+
* Whether the DISCRIMINATING entries inside `rect` moved across a gesture: one left or entered the
470+
* region, or its rect moved beyond tolerance. Entries match on the flip-tolerant `identity` where
471+
* they have one, told apart by document order when repeated, and on `key` otherwise. A scroll moves
472+
* content, while a state flip inside the container (a switch the swipe brushed, a row it selected)
473+
* changes the key at the same rect and is not movement.
470474
*
471475
* A whole-surface difference is not automatically the gesture's doing. A captured tree carries system
472476
* chrome with it, and on Android the status bar clocks and icons change on their own while the app's
@@ -478,17 +482,30 @@ export function discriminatingSurfaceChangedWithinRect(
478482
after: InteractionSurfaceSignature,
479483
rect: Rect,
480484
): boolean {
481-
const beforeInRect = discriminatingEntriesWithinRect(before, rect);
482-
const afterByKey = new Map(
483-
discriminatingEntriesWithinRect(after, rect).map((entry) => [entry.key, entry]),
484-
);
485-
for (const entry of beforeInRect) {
486-
const other = afterByKey.get(entry.key);
485+
const beforeInRect = contentKeyed(discriminatingEntriesWithinRect(before, rect));
486+
const afterInRect = contentKeyed(discriminatingEntriesWithinRect(after, rect));
487+
for (const [content, entry] of beforeInRect) {
488+
const other = afterInRect.get(content);
487489
if (!other) return true;
488490
if (!rectsWithinTolerance(entry, other)) return true;
489-
afterByKey.delete(entry.key);
491+
afterInRect.delete(content);
490492
}
491-
return afterByKey.size > 0;
493+
return afterInRect.size > 0;
494+
}
495+
496+
/** Entries by what they are rather than the state they are in; repeated content is told apart by document order. */
497+
function contentKeyed(
498+
entries: InteractionSurfaceSignature,
499+
): Map<string, InteractionSurfaceSignature[number]> {
500+
const occurrences = new Map<string, number>();
501+
const keyed = new Map<string, InteractionSurfaceSignature[number]>();
502+
for (const entry of entries) {
503+
const content = entry.identity ?? entry.key;
504+
const occurrence = occurrences.get(content) ?? 0;
505+
occurrences.set(content, occurrence + 1);
506+
keyed.set(`${content}|#${occurrence}`, entry);
507+
}
508+
return keyed;
492509
}
493510

494511
function discriminatingEntriesWithinRect(
@@ -552,7 +569,7 @@ function buildInteractionSurfaceEntry(
552569
/**
553570
* What the element IS — never where it sits, and never volatile state a gesture
554571
* is expected to change. `interactionSurfaceSemanticKey` deliberately folds in
555-
* `hittable`/`enabled`/`selected` and an occurrence index, which is right for
572+
* the states `stateMarkers` prints, `hittable`, and an occurrence index, which is right for
556573
* "did these two back-to-back captures agree" and wrong for "is this the same
557574
* element as before the gesture": scrolling flips `hittable` the moment a
558575
* node's centre leaves the viewport, so keying on it evicts precisely the
@@ -584,15 +601,19 @@ function isNonDiscriminatingSurfaceNode(
584601
return isViewportRootNode(node) || (node.ref !== undefined && keyboardChromeRefs.has(node.ref));
585602
}
586603

604+
/**
605+
* What the element is and the state it is in. The states are the ones `stateMarkers` prints, so the
606+
* outcome lane, the unchanged-snapshot comparison, and the diff weigh one list: a tap whose only
607+
* effect is a toggle is a change here, not a no-op to retry.
608+
*/
587609
function interactionSurfaceSemanticKey(node: SnapshotNode): string | undefined {
588610
const semanticKey = [
589611
node.identifier,
590612
node.label,
591613
node.value,
592614
node.type,
593615
node.role,
594-
node.enabled === false ? 'disabled' : 'enabled',
595-
node.selected === true ? 'selected' : 'unselected',
616+
...stateMarkers(node),
596617
node.hittable === true ? 'hittable' : 'not-hittable',
597618
]
598619
.map((value) => (typeof value === 'string' ? value.trim() : ''))

0 commit comments

Comments
 (0)